diff --git a/.binny.yaml b/.binny.yaml index 197d52ede..561c3a792 100644 --- a/.binny.yaml +++ b/.binny.yaml @@ -16,7 +16,7 @@ tools: # used to upload test fixture cache - name: oras version: - want: v1.3.3 + want: v1.3.4 method: github-release with: repo: oras-project/oras diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 000000000..2bcde6f9d --- /dev/null +++ b/.gitattributes @@ -0,0 +1,7 @@ +# check out text files with lf on every platform (regardless of core.autocrlf), letting git detect binary files +* text=auto eol=lf + +# fixtures that must keep their crlf line endings byte for byte +syft/linux/testdata/unprintable -text +syft/pkg/cataloger/bitnami/testdata/no-rel/opt/bitnami/redis/.spdx-redis.spdx -text +syft/pkg/cataloger/java/testdata/jar-metadata/**/META-INF/MANIFEST.MF -text diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index edc571c35..72ec17297 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -86,7 +86,7 @@ jobs: # dockers_v2 builds multi-platform manifests in a single buildx invocation, which requires a # docker-container driver builder (the default builder on the runner cannot build multi-platform). - name: Set up Docker Buildx - uses: docker/setup-buildx-action@37fe631027851001ddb9b187196cc803df7f5f0e # v4.3.0 + uses: docker/setup-buildx-action@f87e5991a6d7451dcb8d9637bfbc97413f497069 # v4.4.1 - name: Login to Docker Hub uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f #v4.6.0 diff --git a/.github/workflows/validations.yaml b/.github/workflows/validations.yaml index 2656ab1f1..95092a476 100644 --- a/.github/workflows/validations.yaml +++ b/.github/workflows/validations.yaml @@ -27,7 +27,7 @@ jobs: # enough; a 4 vCPU instance can't pull much more than that anyway. # note: s3-cache intentionally omitted -- PR runs are untrusted and must not write to the # shared cache backend that the trusted release workflow reads from (cache poisoning). - runs-on: &test-runner "runs-on=${{ github.run_id }}/cpu=4+8/ram=16+32/family=m5+m6+m7+m8+c5+c6+c7+c8/spot=price-capacity-optimized/volume=100gb" + runs-on: &test-runner "runs-on=${{ github.run_id }}/cpu=4+8/ram=16+32/family=m6+m7+m8+c6+c7+c8/spot=price-capacity-optimized/volume=100gb" permissions: contents: read steps: @@ -49,6 +49,13 @@ jobs: Unit-Test: # Note: changing this job name requires making the same update in the .github/workflows/release.yaml pipeline name: "Unit tests" + runs-on: ubuntu-latest + needs: [Unit-Test-Linux, Unit-Test-Windows] + steps: + - run: echo All unit tests passed! + + Unit-Test-Linux: + name: "Unit tests (Linux)" runs-on: *test-runner permissions: contents: read @@ -68,6 +75,27 @@ jobs: - name: Check for capability drift run: make check-capability-drift + Unit-Test-Windows: + name: "Unit tests (Windows)" + runs-on: windows-latest + permissions: + contents: read + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 #v7.0.1 + with: + persist-credentials: false + + # go-make's setup only (not ./.github/actions/bootstrap): no apt, binny tools, or fixture cache are needed + - name: Setup go + go-make tooling + uses: anchore/go-make/.github/actions/setup@fa5421b3bf24c9d18f11a736c5eb4172f2b7d683 # v0.8.1 + with: + # keep in sync with the go-version default in ./.github/actions/bootstrap + go-version: "1.26.2" + + # the full unit suite leans on linux docker fixtures, so this runs the windows-relevant subset + - name: Run unit tests + run: make unit:windows + Integration-Test: # Note: changing this job name requires making the same update in the .github/workflows/release.yaml pipeline name: "Integration tests" diff --git a/.golangci.yaml b/.golangci.yaml index 6b9bf74b0..5aa2a4905 100644 --- a/.golangci.yaml +++ b/.golangci.yaml @@ -7,6 +7,7 @@ linters: - asciicheck - bodyclose - copyloopvar + - depguard - dogsled - dupl - errcheck @@ -29,6 +30,13 @@ linters: - unused - whitespace settings: + depguard: + rules: + yaml: + deny: + # gopkg.in/yaml.v3 is unmaintained and panics on some malformed input (see #5327) + - pkg: gopkg.in/yaml.v3 + desc: use go.yaml.in/yaml/v3 instead funlen: lines: 70 statements: 50 @@ -71,7 +79,6 @@ linters: # do not enable... # - deadcode # The owner seems to have abandoned the linter. Replaced by "unused". -# - depguard # We don't have a configuration for this yet # - goprintffuncname # does not catch all cases and there are exceptions # - nakedret # does not catch all cases and should not fail a build # - gochecknoglobals diff --git a/.make/main.go b/.make/main.go index f0f08700d..58db41a96 100644 --- a/.make/main.go +++ b/.make/main.go @@ -1,6 +1,7 @@ package main import ( + "os" "path/filepath" "runtime" "strconv" @@ -52,7 +53,7 @@ func main() { raceFlag = " -race" } Run( - "go test -count=1 -timeout=30m -v"+raceFlag+" ./cmd/syft/internal/test/integration/...", + "go test -count=1 -timeout=30m"+raceFlag+" ./cmd/syft/internal/test/integration/...", run.Env("GODEBUG", "dontfreezetheworld=1"), ) }, @@ -96,6 +97,23 @@ func main() { }, }, + // windows unit tests: deliberately not hooked into "test" (no RunsOn). The full unit suite leans on + // linux docker fixture images and posix-only assumptions, so this covers the packages that own + // host path handling (where windows bugs live) and skips the tests that need docker image fixtures. + // ponytail: curated package list, widen it as more of the suite is made windows-safe. + Task{ + Name: "unit:windows", + Description: "run the windows-relevant subset of unit tests", + Run: func() { + Run("go test -count=1"+ + ` -skip "Image|Squash|AllLayers|MixFileTypes|GivenCoordinates"`+ + " ./syft/internal/windows/... ./syft/internal/fileresolver/... ./syft/source/directorysource/... ./syft/source/filesource/..."+ + " ./syft/file ./syft/file/cataloger/filedigest/... ./syft/file/cataloger/filecontent/...", + run.Stdout(os.Stderr), + ) + }, + }, + // default validation pipeline (replaces Taskfile `default`/`pr-validations`/`validations`). Task{ Name: "default", diff --git a/cmd/syft/cli/ui/handle_attestation.go b/cmd/syft/cli/ui/handle_attestation.go index 838d77c89..4c44410b5 100644 --- a/cmd/syft/cli/ui/handle_attestation.go +++ b/cmd/syft/cli/ui/handle_attestation.go @@ -158,6 +158,11 @@ func (l *backgroundLineReader) read(reader io.Reader, stage *progress.Stage) { l.lock.Unlock() } + if err := s.Err(); err != nil { + log.WithFields("error", err).Debug("unable to read attestation output") + // keep draining so the writer never blocks on a full pipe + _, _ = io.Copy(io.Discard, reader) + } } func (l backgroundLineReader) Lines() []string { diff --git a/cmd/syft/internal/test/integration/multiplatform_oci_test.go b/cmd/syft/internal/test/integration/multiplatform_oci_test.go index 3373d8f36..411095849 100644 --- a/cmd/syft/internal/test/integration/multiplatform_oci_test.go +++ b/cmd/syft/internal/test/integration/multiplatform_oci_test.go @@ -16,19 +16,21 @@ import ( "github.com/anchore/syft/syft/source" ) +// multiplatformImage is busybox:1.38.0 pinned by index digest, since docker hub periodically rebuilds and +// re-pushes official image tags, which changes the per-platform digests asserted below. +const multiplatformImage = "docker.io/library/busybox@sha256:fd7dc98638c8e305f4dc34e979f1c0fdfdcaeb0fbf8fcff77ae834b6da3d7e6e" + // TestMultiPlatformOCIImageSelection verifies that syft can load a single platform out of a // multi-platform OCI image on disk (both --from oci-dir and --from oci-archive) by selecting the // image that matches the requested platform. func TestMultiPlatformOCIImageSelection(t *testing.T) { - remoteImage := "docker.io/library/busybox:1.38.0" - // Per-platform image config digests within the multi-platform index, obtained from the OCI layout // (see anchore/stereoscope integration tests: TestPlatformSelectionWithOciLocalSources). expectedDigest := map[string]string{ - "arm64": "sha256:e0e8b3cbfed68a90084781e2962f9c0deead51c5a3f11a488eef0283a4284bc2", - "s390x": "sha256:0cf160e720a8e4f20883b72276a6eaded83b79539f3f1d39e35a3336154b9960", - "amd64": "sha256:c6348fa86ba0fb2108c9334f5fe913ddc6d853313e655891f133a0127c30099f", - "ppc64le": "sha256:b144fc0e06537d07956cde340b878a81a717e394edb736d3110858a12a6635cb", + "arm64": "sha256:d2482869a6b838d3b4c9cad0c8085c06277909482771eb525d298fc3a7d07927", + "s390x": "sha256:e55a0fa55032cceac2d983f4b4a25b28e42303b46a2f1915fc0fa45051e6ca70", + "amd64": "sha256:aaef90e065235eb0b2d9938be85d1b81add2902cf89affe2cdfa19be33476ac0", + "ppc64le": "sha256:8158829b015d1c5285d63d54d37e95998ef0af4177e3f416678decbcaff3170c", } // syft --from source tag -> stereoscope OCI source used to prepare the local fixture @@ -39,7 +41,7 @@ func TestMultiPlatformOCIImageSelection(t *testing.T) { for from, imageSource := range sources { t.Run(from, func(t *testing.T) { - localPath := imagetest.PrepareMultiplatformFixtureImage(t, imageSource, remoteImage) + localPath := imagetest.PrepareMultiplatformFixtureImage(t, imageSource, multiplatformImage) for _, arch := range []string{"amd64", "arm64", "s390x", "ppc64le"} { t.Run(fmt.Sprintf("linux/%s", arch), func(t *testing.T) { platform, err := image.NewPlatform("linux/" + arch) @@ -73,8 +75,6 @@ func TestMultiPlatformOCIImageSelection(t *testing.T) { // TestMultiPlatformOCIImageSelection_UnavailablePlatform verifies that requesting a platform not present // in the multi-platform OCI image results in an error rather than silently selecting the wrong image. func TestMultiPlatformOCIImageSelection_UnavailablePlatform(t *testing.T) { - remoteImage := "docker.io/library/busybox:1.38.0" - // windows/amd64 is not present in this linux-only multi-platform image platform, err := image.NewPlatform("windows/amd64") require.NoError(t, err) @@ -86,7 +86,7 @@ func TestMultiPlatformOCIImageSelection_UnavailablePlatform(t *testing.T) { for from, imageSource := range sources { t.Run(from, func(t *testing.T) { - localPath := imagetest.PrepareMultiplatformFixtureImage(t, imageSource, remoteImage) + localPath := imagetest.PrepareMultiplatformFixtureImage(t, imageSource, multiplatformImage) _, err := syft.GetSource( context.Background(), @@ -101,8 +101,6 @@ func TestMultiPlatformOCIImageSelection_UnavailablePlatform(t *testing.T) { // TestMultiPlatformOCIImageSelection_DefaultPlatform verifies that not specifying a platform results // in the current platform being selected. func TestMultiPlatformOCIImageSelection_DefaultPlatform(t *testing.T) { - remoteImage := "docker.io/library/busybox:1.38.0" - sources := map[string]image.Source{ "oci-dir": image.OciDirectorySource, "oci-archive": image.OciTarballSource, @@ -110,7 +108,7 @@ func TestMultiPlatformOCIImageSelection_DefaultPlatform(t *testing.T) { for from, imageSource := range sources { t.Run(from, func(t *testing.T) { - localPath := imagetest.PrepareMultiplatformFixtureImage(t, imageSource, remoteImage) + localPath := imagetest.PrepareMultiplatformFixtureImage(t, imageSource, multiplatformImage) src, err := syft.GetSource( context.Background(), diff --git a/cmd/syft/internal/ui/ui.go b/cmd/syft/internal/ui/ui.go index b426fd867..878558367 100644 --- a/cmd/syft/internal/ui/ui.go +++ b/cmd/syft/internal/ui/ui.go @@ -1,6 +1,7 @@ package ui import ( + "bytes" "fmt" "io" "os" @@ -9,6 +10,7 @@ import ( "time" tea "github.com/charmbracelet/bubbletea" + "github.com/charmbracelet/lipgloss" "github.com/wagoodman/go-partybus" "github.com/anchore/bubbly" @@ -53,7 +55,7 @@ func New(out io.Writer, quiet bool, handlers ...bubbly.EventHandler) *UI { func (m *UI) Setup(subscription partybus.Unsubscribable) error { // we still want to collect log messages, however, we also the logger shouldn't write to the screen directly if logWrapper, ok := log.Get().(logger.Controller); ok { - logWrapper.SetOutput(m.frame.(*frame.Frame).Footer()) + logWrapper.SetOutput(&logFooter{w: m.frame.(*frame.Frame).Footer()}) } m.subscription = subscription @@ -193,6 +195,48 @@ func (m UI) View() string { return v } +var ( + logTitleStyle = lipgloss.NewStyle().Foreground(lipgloss.Color("8")) // dark grey + // the first log line gets a branch pointing at it from the title, the rest are indented to match + logBranch = logTitleStyle.Render("└─▶") + " " + logIndent = " " +) + +// logFooter decorates log entries drawn below the TUI with a one-time title (written when the first entry arrives) +// and a branch from the title to the first log line. Writes are serialized by the logger. +type logFooter struct { + w io.Writer + started bool +} + +func (l *logFooter) Write(p []byte) (int, error) { + var buf bytes.Buffer + first := !l.started + if first { + buf.WriteString(logTitleStyle.Render("Logs (non-fatal, see -v for more):") + "\n") + l.started = true + } + for i, line := range bytes.SplitAfter(p, []byte("\n")) { + if len(line) == 0 { + continue + } + if first && i == 0 { + buf.WriteString(logBranch) + } else { + buf.WriteString(logIndent) + } + buf.Write(line) + } + if _, err := l.w.Write(buf.Bytes()); err != nil { + return 0, err + } + return len(p), nil +} + +// IsTerminal marks the footer as terminal-bound (it is drawn by the TUI) so the logger keeps colors. NO_COLOR is +// still honored by the formatter. +func (*logFooter) IsTerminal() bool { return true } + func runWithTimeout(timeout time.Duration, fn func() error) (err error) { c := make(chan struct{}, 1) go func() { diff --git a/go.mod b/go.mod index 8c49b84ff..09b8346f3 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module github.com/anchore/syft -go 1.26.3 +go 1.26.8 require ( github.com/BurntSushi/toml v1.6.0 @@ -11,12 +11,12 @@ require ( github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d github.com/acobaugh/osrelease v0.1.0 github.com/adrg/xdg v0.5.3 - github.com/anchore/bubbly v0.2.1 + github.com/anchore/bubbly v0.2.2 github.com/anchore/clio v0.1.1 github.com/anchore/fangs v0.1.1 github.com/anchore/go-collections v0.1.1 github.com/anchore/go-homedir v0.1.1 - github.com/anchore/go-logger v0.1.1 + github.com/anchore/go-logger v0.2.0 github.com/anchore/go-macholibre v0.1.1 github.com/anchore/go-rpmdb v0.2.0 github.com/anchore/go-sync v0.1.2 @@ -36,7 +36,7 @@ require ( github.com/deitch/magic v0.0.0-20230404182410-1ff89d7342da github.com/diskfs/go-diskfs v1.9.4 github.com/distribution/reference v0.6.0 - github.com/dustin/go-humanize v1.0.1 + github.com/dustin/go-humanize v1.1.0 github.com/elliotchance/phpserialize v1.4.0 github.com/facebookincubator/nvdtools v0.1.5 github.com/github/go-spdx/v2 v2.7.0 @@ -56,18 +56,18 @@ require ( github.com/hashicorp/go-cleanhttp v0.5.2 github.com/hashicorp/go-getter v1.8.9 github.com/hashicorp/go-multierror v1.1.1 - github.com/hashicorp/hcl/v2 v2.24.0 + github.com/hashicorp/hcl/v2 v2.25.0 github.com/iancoleman/strcase v0.3.0 github.com/invopop/jsonschema v0.14.0 github.com/jedib0t/go-pretty/v6 v6.8.3 github.com/jinzhu/copier v0.4.0 github.com/kastenhq/goversion v0.0.0-20230811215019-93b2f8823953 github.com/klauspost/compress v1.20.0 - github.com/magiconair/properties v1.18.11 + github.com/magiconair/properties v1.18.12 github.com/mholt/archives v0.1.5 github.com/moby/sys/mountinfo v0.7.2 github.com/nix-community/go-nix v0.0.0-20250101154619-4bdde671e0a1 - github.com/olekukonko/tablewriter v1.1.4 + github.com/olekukonko/tablewriter v1.1.5 github.com/opencontainers/go-digest v1.0.0 github.com/pelletier/go-toml v1.9.5 github.com/quasilyte/go-ruleguard/dsl v0.3.23 @@ -97,8 +97,7 @@ require ( golang.org/x/net v0.59.0 golang.org/x/time v0.16.0 golang.org/x/tools v0.50.0 - gopkg.in/yaml.v3 v3.0.1 - modernc.org/sqlite v1.58.0 + modernc.org/sqlite v1.59.0 ) require ( @@ -131,6 +130,7 @@ require ( github.com/anchore/go-struct-converter v0.2.0-rc2 // indirect github.com/andybalholm/brotli v1.2.0 // indirect github.com/apparentlymart/go-textseg/v15 v15.0.0 // indirect + github.com/apparentlymart/go-textseg/v17 v17.0.1 // indirect github.com/aquasecurity/go-version v0.0.1 // indirect github.com/atotto/clipboard v0.1.4 // indirect github.com/aws/aws-sdk-go-v2 v1.44.0 // indirect @@ -170,7 +170,7 @@ require ( github.com/cncf/xds/go v0.0.0-20260202195803-dba9d589def2 // indirect github.com/containerd/cgroups/v3 v3.1.3 // indirect github.com/containerd/containerd/api v1.11.1 // indirect - github.com/containerd/containerd/v2 v2.3.5 // indirect + github.com/containerd/containerd/v2 v2.3.6 // indirect github.com/containerd/continuity v0.5.0 // indirect github.com/containerd/errdefs v1.0.0 // indirect github.com/containerd/errdefs/pkg v0.3.0 // indirect @@ -248,7 +248,7 @@ require ( github.com/muesli/cancelreader v0.2.2 // indirect github.com/muesli/termenv v0.16.0 // indirect github.com/ncruces/go-strftime v1.0.0 // indirect - github.com/nwaples/rardecode/v2 v2.2.0 // indirect + github.com/nwaples/rardecode/v2 v2.4.1 // indirect github.com/olekukonko/cat v0.0.0-20250911104152-50322a0618f6 // indirect github.com/olekukonko/errors v1.2.0 // indirect github.com/olekukonko/ll v0.1.6 // indirect @@ -293,7 +293,7 @@ require ( github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 // indirect github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8 // indirect github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e // indirect - github.com/zclconf/go-cty v1.16.3 // indirect + github.com/zclconf/go-cty v1.19.0 // indirect go.opencensus.io v0.24.0 // indirect go.opentelemetry.io/auto/sdk v1.2.1 // indirect go.opentelemetry.io/contrib/detectors/gcp v1.44.0 // indirect @@ -321,7 +321,7 @@ require ( google.golang.org/grpc v1.83.2 // indirect google.golang.org/protobuf v1.36.12 // indirect gopkg.in/warnings.v0 v0.1.2 // indirect - modernc.org/libc v1.75.6 // indirect + modernc.org/libc v1.75.7 // indirect modernc.org/mathutil v1.7.1 // indirect modernc.org/memory v1.12.1 // indirect ) diff --git a/go.sum b/go.sum index 99c006c5d..b66b9b51a 100644 --- a/go.sum +++ b/go.sum @@ -120,8 +120,8 @@ github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuy github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= -github.com/anchore/bubbly v0.2.1 h1:b3ZNBefY8ks5DI4SQ7BXSbBIsN6eoaTD6s1su7NuCBo= -github.com/anchore/bubbly v0.2.1/go.mod h1:o6a9aH3/fCn+mb5g1T1tBbQZ4hGR0yO5/uD1d2Vtago= +github.com/anchore/bubbly v0.2.2 h1:53JHRlpaNZjMcUYdz7n1U7+kOhy0zsJv9gpGEXvTcz8= +github.com/anchore/bubbly v0.2.2/go.mod h1:M4twoFkczlfuNdU8QV7oQjVJwPbxdUWRMfKydcLba0U= github.com/anchore/clio v0.1.1 h1:9bkYnaFbpT0J3G0qSHyQqa/lpK5zV0C9Gtgh82H/jj0= github.com/anchore/clio v0.1.1/go.mod h1:CFPIopJlakJbBpKtAL/EPpj4W9hgBO7PfFc8N2FAtlE= github.com/anchore/fangs v0.1.1 h1:NBTHocmWM2/0gNhebpoE/wEB0Z0nxh2lESww2o/Uf5M= @@ -130,8 +130,8 @@ github.com/anchore/go-collections v0.1.1 h1:kGlmSehs+ctTBZr+98F2+taHeolHy+XDx1Nc github.com/anchore/go-collections v0.1.1/go.mod h1:zgDbPSNDpfU47tVo7aW+2suG7+vXB1Sq1qtDhRJknQ0= github.com/anchore/go-homedir v0.1.1 h1:KpaO5BPrVT7cVMcEZ9KfFfEi0iuo3DtagPGl0C744X0= github.com/anchore/go-homedir v0.1.1/go.mod h1:9B0DGhbmMAJVGGEbrlb+PkORM5eDFCEOtZ3xQ22qaLA= -github.com/anchore/go-logger v0.1.1 h1:HPBNmQVBdYliwaFVUL7oJ3UvSlaoXAMj/ZagAIYaCpA= -github.com/anchore/go-logger v0.1.1/go.mod h1:ekWuh5BkZVwyXnyEd4fwL29N4pNLssgRsqh2+kFN/b8= +github.com/anchore/go-logger v0.2.0 h1:V8Gazz+F8xwWEWz+whqt8/5163IkO4XrAie+1qbK10k= +github.com/anchore/go-logger v0.2.0/go.mod h1:omqLWBwT/B5/aTwIH6pl5WBylBzeH3BGHUTS6OFeFvY= github.com/anchore/go-lzo v0.1.1 h1:IwL/fvkdtlIrYIXck6WxZ3nb8WjjHziYYmGxlooyOnM= github.com/anchore/go-lzo v0.1.1/go.mod h1:3kLx0bve2oN1iDwgM1U5zGku1Tfbdb0No5qp1eL1fIk= github.com/anchore/go-macholibre v0.1.1 h1:sTBHi3S1WVIOCamc6NyT8Sn9mBmD30DCTfjfVm4k6/M= @@ -156,6 +156,8 @@ github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be/go.mod h1:ySMOLuW github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= github.com/apparentlymart/go-textseg/v15 v15.0.0 h1:uYvfpb3DyLSCGWnctWKGj857c6ew1u1fNQOlOtuGxQY= github.com/apparentlymart/go-textseg/v15 v15.0.0/go.mod h1:K8XmNZdhEBkdlyDdvbmmsvpAG721bKi0joRfFdHIWJ4= +github.com/apparentlymart/go-textseg/v17 v17.0.1 h1:bpMXRgQ5cEoRNuQke1a80/Nl6w3G5eoIbWo9f3gXkAs= +github.com/apparentlymart/go-textseg/v17 v17.0.1/go.mod h1:fa8X4jgGeevslICIY6LcdjkSecWnXmYd9Lk34z/VxZs= github.com/aquasecurity/go-pep440-version v0.0.1 h1:8VKKQtH2aV61+0hovZS3T//rUF+6GDn18paFTVS0h0M= github.com/aquasecurity/go-pep440-version v0.0.1/go.mod h1:3naPe+Bp6wi3n4l5iBFCZgS0JG8vY6FT0H4NGhFJ+i4= github.com/aquasecurity/go-version v0.0.1 h1:4cNl516agK0TCn5F7mmYN+xVs1E3S45LkgZk3cbaW2E= @@ -294,8 +296,8 @@ github.com/containerd/cgroups/v3 v3.1.3 h1:eUNflyMddm18+yrDmZPn3jI7C5hJ9ahABE5q6 github.com/containerd/cgroups/v3 v3.1.3/go.mod h1:PKZ2AcWmSBsY/tJUVhtS/rluX0b1uq1GmPO1ElCmbOw= github.com/containerd/containerd/api v1.11.1 h1:h8nfoDW9+fNsC/9TwiAHj8B1GzXKtR4eFtkhi/X5RLU= github.com/containerd/containerd/api v1.11.1/go.mod h1:CaQFRu+N1MtbgL6JDOJLUB1hCKESU1lD6MuTJhgtdlw= -github.com/containerd/containerd/v2 v2.3.5 h1:9MYlI81gUcOZ0WsCkSMtvOU7rTR3hqAoa2eCzhoLlkA= -github.com/containerd/containerd/v2 v2.3.5/go.mod h1:RXDyLPaI3zoO7dFdAW9/54W4cix+z3A6larieufC9mg= +github.com/containerd/containerd/v2 v2.3.6 h1:huoqZXaDW1x1kgPvFC4rMwxv92vUJnqgq7I5b1rSddM= +github.com/containerd/containerd/v2 v2.3.6/go.mod h1:qaUSWKk9EchqbudSbhv1MxI8dXKGYHLpZtMQdkEkVZ4= github.com/containerd/continuity v0.5.0 h1:7a85HZpCSs+1Zps0Ee3DPSuAWY+0SJM1JNM51nlEVDg= github.com/containerd/continuity v0.5.0/go.mod h1:/lNJvtJKUQStBzpVQ1+rasXO1LAWtUQssk28EZvJ3nE= github.com/containerd/errdefs v1.0.0 h1:tg5yIfIlQIrxYtu9ajqY42W3lpS19XqdxRQeEwYG8PI= @@ -348,8 +350,8 @@ github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDD github.com/dsnet/compress v0.0.2-0.20230904184137-39efe44ab707 h1:2tV76y6Q9BB+NEBasnqvs7e49aEBFI8ejC89PSnWH+4= github.com/dsnet/compress v0.0.2-0.20230904184137-39efe44ab707/go.mod h1:qssHWj60/X5sZFNxpG4HBPDHVqxNm4DfnCKgrbZOT+s= github.com/dsnet/golib v0.0.0-20171103203638-1ea166775780/go.mod h1:Lj+Z9rebOhdfkVLjJ8T6VcRQv3SXugXy999NBtR9aFY= -github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= -github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= +github.com/dustin/go-humanize v1.1.0 h1:dbKTrvD0klcbBV/h4AWJdMuZogJACoMlvWIWZ5b2xWg= +github.com/dustin/go-humanize v1.1.0/go.mod h1:hc1CvRkJMsgxqjmjMQF3QNRAZBwY8AXBAzKYoSX9sFI= github.com/elazarl/goproxy v1.7.2 h1:Y2o6urb7Eule09PjlhQRGNsqRfPmYI3KKQLFpCAV3+o= github.com/elazarl/goproxy v1.7.2/go.mod h1:82vkLNir0ALaW14Rc399OTTjyNREgmdL2cVoIbS6XaE= github.com/elliotchance/phpserialize v1.4.0 h1:cAp/9+KSnEbUC8oYCE32n2n84BeW8HOY3HMDI8hG2OY= @@ -595,8 +597,8 @@ github.com/hashicorp/golang-lru v0.5.4/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uG github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k= github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM= github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= -github.com/hashicorp/hcl/v2 v2.24.0 h1:2QJdZ454DSsYGoaE6QheQZjtKZSUs9Nh2izTWiwQxvE= -github.com/hashicorp/hcl/v2 v2.24.0/go.mod h1:oGoO1FIQYfn/AgyOhlg9qLC6/nOJPX3qGbkZpYAcqfM= +github.com/hashicorp/hcl/v2 v2.25.0 h1:HmmQVYRny4MaBo4b20TjmL46wyuUxpnMWkPZ4+NTbWk= +github.com/hashicorp/hcl/v2 v2.25.0/go.mod h1:vR+FKETxoZAmRlHgFfKmuqivj+C4Izm/c66XkmZ3r7M= github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO+LraFDTW64= github.com/hashicorp/mdns v1.0.1/go.mod h1:4gW7WsVCke5TE7EPeYliwHlRUyBtfCwuFwuMg2DmyNY= github.com/hashicorp/mdns v1.0.4/go.mod h1:mtBihi+LeNXGtG8L9dX59gAEa12BDtBQSp4v/YAJqrc= @@ -668,8 +670,8 @@ github.com/lucasb-eyer/go-colorful v1.3.0 h1:2/yBRLdWBZKrf7gB40FoiKfAWYQ0lqNcbuQ github.com/lucasb-eyer/go-colorful v1.3.0/go.mod h1:R4dSotOR9KMtayYi1e77YzuveK+i7ruzyGqttikkLy0= github.com/lyft/protoc-gen-star v0.5.3/go.mod h1:V0xaHgaf5oCCqmcxYcWiDfTiKsZsRc87/1qhoTACD8w= github.com/magiconair/properties v1.8.5/go.mod h1:y3VJvCyxH9uVvJTWEGAELF3aiYNyPKd5NZ3oSwXrF60= -github.com/magiconair/properties v1.18.11 h1:j5ozYZl0zCjG7ahMDH0GWIobOvvUzT0BdAguG0ViKy0= -github.com/magiconair/properties v1.18.11/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3vdS329zhj2hYo0= +github.com/magiconair/properties v1.18.12 h1:sT9zQpvTB3B4gzrX0tmZNTEaGyg8Zw55MFYRE32Mr9I= +github.com/magiconair/properties v1.18.12/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3vdS329zhj2hYo0= github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= github.com/maruel/natural v1.3.0 h1:VsmCsBmEyrR46RomtgHs5hbKADGRVtliHTyCOLFBpsg= github.com/maruel/natural v1.3.0/go.mod h1:v+Rfd79xlw1AgVBjbO0BEQmptqb5HvL/k9GRHB7ZKEg= @@ -756,16 +758,16 @@ github.com/nix-community/go-nix v0.0.0-20250101154619-4bdde671e0a1 h1:kpt9ZfKcm+ github.com/nix-community/go-nix v0.0.0-20250101154619-4bdde671e0a1/go.mod h1:qgCw4bBKZX8qMgGeEZzGFVT3notl42dBjNqO2jut0M0= github.com/nsf/jsondiff v0.0.0-20210926074059-1e845ec5d249 h1:NHrXEjTNQY7P0Zfx1aMrNhpgxHmow66XQtm0aQLY0AE= github.com/nsf/jsondiff v0.0.0-20210926074059-1e845ec5d249/go.mod h1:mpRZBD8SJ55OIICQ3iWH0Yz3cjzA61JdqMLoWXeB2+8= -github.com/nwaples/rardecode/v2 v2.2.0 h1:4ufPGHiNe1rYJxYfehALLjup4Ls3ck42CWwjKiOqu0A= -github.com/nwaples/rardecode/v2 v2.2.0/go.mod h1:7uz379lSxPe6j9nvzxUZ+n7mnJNgjsRNb6IbvGVHRmw= +github.com/nwaples/rardecode/v2 v2.4.1 h1:F7zNW2LdAuuBThHWXQaiFUGVD/sef299NfWSB1nHAl4= +github.com/nwaples/rardecode/v2 v2.4.1/go.mod h1:7uz379lSxPe6j9nvzxUZ+n7mnJNgjsRNb6IbvGVHRmw= github.com/olekukonko/cat v0.0.0-20250911104152-50322a0618f6 h1:zrbMGy9YXpIeTnGj4EljqMiZsIcE09mmF8XsD5AYOJc= github.com/olekukonko/cat v0.0.0-20250911104152-50322a0618f6/go.mod h1:rEKTHC9roVVicUIfZK7DYrdIoM0EOr8mK1Hj5s3JjH0= github.com/olekukonko/errors v1.2.0 h1:10Zcn4GeV59t/EGqJc8fUjtFT/FuUh5bTMzZ1XwmCRo= github.com/olekukonko/errors v1.2.0/go.mod h1:ppzxA5jBKcO1vIpCXQ9ZqgDh8iwODz6OXIGKU8r5m4Y= github.com/olekukonko/ll v0.1.6 h1:lGVTHO+Qc4Qm+fce/2h2m5y9LvqaW+DCN7xW9hsU3uA= github.com/olekukonko/ll v0.1.6/go.mod h1:NVUmjBb/aCtUpjKk75BhWrOlARz3dqsM+OtszpY4o88= -github.com/olekukonko/tablewriter v1.1.4 h1:ORUMI3dXbMnRlRggJX3+q7OzQFDdvgbN9nVWj1drm6I= -github.com/olekukonko/tablewriter v1.1.4/go.mod h1:+kedxuyTtgoZLwif3P1Em4hARJs+mVnzKxmsCL/C5RY= +github.com/olekukonko/tablewriter v1.1.5 h1:4LoZSfMySpMQY3PT8RWJsJeuEuMIoo9xGRgvmqjg6IQ= +github.com/olekukonko/tablewriter v1.1.5/go.mod h1:+kedxuyTtgoZLwif3P1Em4hARJs+mVnzKxmsCL/C5RY= github.com/onsi/gomega v1.34.1 h1:EUMJIKUjM8sKjYbtxQI9A4z2o+rruxnzNvpknOXie6k= github.com/onsi/gomega v1.34.1/go.mod h1:kU1QgUvBDLXBJq618Xvm2LUX6rSAfRaFRTcdOeDLwwY= github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= @@ -976,8 +978,8 @@ github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9de github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= -github.com/zclconf/go-cty v1.16.3 h1:osr++gw2T61A8KVYHoQiFbFd1Lh3JOCXc/jFLJXKTxk= -github.com/zclconf/go-cty v1.16.3/go.mod h1:VvMs5i0vgZdhYawQNq5kePSpLAoz8u1xvZgrPIxfnZE= +github.com/zclconf/go-cty v1.19.0 h1:IV8WdqYZc2c5rLX9bEoLNXKojBAp0MZPBHMIrCoa/s4= +github.com/zclconf/go-cty v1.19.0/go.mod h1:12W89jGn3JCOIQi7infWr9m80rOkb5RNYJqXMZcN4c8= github.com/zclconf/go-cty-debug v0.0.0-20240509010212-0d6042c53940 h1:4r45xpDWB6ZMSMNJFMOjqrGHynW3DIBuR2H9j0ug+Mo= github.com/zclconf/go-cty-debug v0.0.0-20240509010212-0d6042c53940/go.mod h1:CmBdvvj3nqzfzJ6nTCIwDTPZ56aVGvDrmztiO5g3qrM= github.com/zyedidia/generic v1.2.2-0.20230320175451-4410d2372cb1 h1:V+UsotZpAVvfj3X/LMoEytoLzSiP6Lg0F7wdVyu9gGg= @@ -1526,8 +1528,8 @@ modernc.org/gc/v3 v3.1.5 h1:21ldfPfRYE31Tb7B3mwAK8gy1AxP4+dKjrOQPfqakoc= modernc.org/gc/v3 v3.1.5/go.mod h1:HFK/6AGESC7Ex+EZJhJ2Gni6cTaYpSMmU/cT9RmlfYY= modernc.org/goabi0 v0.2.0 h1:HvEowk7LxcPd0eq6mVOAEMai46V+i7Jrj13t4AzuNks= modernc.org/goabi0 v0.2.0/go.mod h1:CEFRnnJhKvWT1c1JTI3Avm+tgOWbkOu5oPA8eH8LnMI= -modernc.org/libc v1.75.6 h1:yKk8qo+Di4gkmvRboK8ocCqH22FiUCR6jRy2OwtCRus= -modernc.org/libc v1.75.6/go.mod h1:bO5o2ztHxBb2rjz0PgdHN0sSMw57CgxGFLZ3Qd/QpVQ= +modernc.org/libc v1.75.7 h1:o3DTP9/0p9pKmY2WCKQaySW6wIiZhNM7wc2lUoyhfew= +modernc.org/libc v1.75.7/go.mod h1:bO5o2ztHxBb2rjz0PgdHN0sSMw57CgxGFLZ3Qd/QpVQ= modernc.org/mathutil v1.7.1 h1:GCZVGXdaN8gTqB1Mf/usp1Y/hSqgI2vAGGP4jZMCxOU= modernc.org/mathutil v1.7.1/go.mod h1:4p5IwJITfppl0G4sUEDtCr4DthTaT47/N3aT6MhfgJg= modernc.org/memory v1.12.1 h1:nFMiWrpStgZczNl6XI9GnIk/rWhYIyHGUaR04pGbp9g= @@ -1536,8 +1538,8 @@ modernc.org/opt v0.2.0 h1:tGyef5ApycA7FSEOMraay9SaTk5zmbx7Tu+cJs4QKZg= modernc.org/opt v0.2.0/go.mod h1:03fq9lsNfvkYSfxrfUhZCWPk1lm4cq4N+Bh//bEtgns= modernc.org/sortutil v1.2.1 h1:+xyoGf15mM3NMlPDnFqrteY07klSFxLElE2PVuWIJ7w= modernc.org/sortutil v1.2.1/go.mod h1:7ZI3a3REbai7gzCLcotuw9AC4VZVpYMjDzETGsSMqJE= -modernc.org/sqlite v1.58.0 h1:38u40/bwkfM7f0Myhosl+SEMltSDxnGdQf8o6Kjmys0= -modernc.org/sqlite v1.58.0/go.mod h1:rsD2CckafgObKC4DhBlGBf+RiHxkc3hINGt1Xw32tVY= +modernc.org/sqlite v1.59.0 h1:X1es1GpqBlS/5T+vbM4HLUdaa8OtQx468DF2vrx+38A= +modernc.org/sqlite v1.59.0/go.mod h1:+paeT2A3iPRHkQDwG7oA6Tk0zQd5woMEI8q7orfry8k= modernc.org/strutil v1.2.1 h1:UneZBkQA+DX2Rp35KcM69cSsNES9ly8mQWD71HKlOA0= modernc.org/strutil v1.2.1/go.mod h1:EHkiggD70koQxjVdSBM3JKM7k6L0FbGE5eymy9i3B9A= modernc.org/token v1.1.0 h1:Xl7Ap9dKaEs5kLoOQeQmPWevfnk/DM5qcLcYlA8ys6Y= diff --git a/internal/capabilities/capabilities.go b/internal/capabilities/capabilities.go index 492cdd2c4..2844f7136 100644 --- a/internal/capabilities/capabilities.go +++ b/internal/capabilities/capabilities.go @@ -9,7 +9,7 @@ import ( "sort" "github.com/scylladb/go-set/strset" - "gopkg.in/yaml.v3" + "go.yaml.in/yaml/v3" "github.com/anchore/syft/internal/task" ) diff --git a/internal/capabilities/generate/io.go b/internal/capabilities/generate/io.go index 3d660cf32..2eb839a6e 100644 --- a/internal/capabilities/generate/io.go +++ b/internal/capabilities/generate/io.go @@ -1,4 +1,4 @@ -// this file handles YAML file reading and writing with comment preservation, using gopkg.in/yaml.v3's node tree to maintain all existing comments during regeneration. +// this file handles YAML file reading and writing with comment preservation, using go.yaml.in/yaml/v3's node tree to maintain all existing comments during regeneration. package main import ( @@ -6,7 +6,7 @@ import ( "os" "regexp" - "gopkg.in/yaml.v3" + "go.yaml.in/yaml/v3" "github.com/anchore/syft/internal/capabilities" "github.com/anchore/syft/internal/capabilities/internal" diff --git a/internal/capabilities/generate/io_test.go b/internal/capabilities/generate/io_test.go index 0d6e790b4..65189c75f 100644 --- a/internal/capabilities/generate/io_test.go +++ b/internal/capabilities/generate/io_test.go @@ -4,7 +4,7 @@ import ( "testing" "github.com/stretchr/testify/require" - "gopkg.in/yaml.v3" + "go.yaml.in/yaml/v3" ) func TestFindFieldValue(t *testing.T) { diff --git a/internal/capabilities/internal/load_capabilities.go b/internal/capabilities/internal/load_capabilities.go index 43c12be27..b07230def 100644 --- a/internal/capabilities/internal/load_capabilities.go +++ b/internal/capabilities/internal/load_capabilities.go @@ -5,7 +5,7 @@ import ( "os" "path/filepath" - "gopkg.in/yaml.v3" + "go.yaml.in/yaml/v3" "github.com/anchore/syft/internal/capabilities" ) diff --git a/internal/capabilities/model_test.go b/internal/capabilities/model_test.go index 12ddb8994..f658e5c27 100644 --- a/internal/capabilities/model_test.go +++ b/internal/capabilities/model_test.go @@ -5,7 +5,7 @@ import ( "github.com/google/go-cmp/cmp" "github.com/stretchr/testify/require" - "gopkg.in/yaml.v3" + "go.yaml.in/yaml/v3" ) func TestConfigSchemaExtensions(t *testing.T) { diff --git a/internal/line_scanner.go b/internal/line_scanner.go new file mode 100644 index 000000000..ad4da5e29 --- /dev/null +++ b/internal/line_scanner.go @@ -0,0 +1,19 @@ +package internal + +import ( + "bufio" + "io" +) + +// maxScannedLineSize is the longest line NewLineScanner will return before Scan stops with bufio.ErrTooLong. +// The buffer only grows to this size when a line needs it. +const maxScannedLineSize = 1024 * 1024 + +// NewLineScanner returns a bufio.Scanner that accepts lines up to 1MB rather than the 64KB default, since +// package metadata files carry long single-line fields (descriptions, license text, dependency lists). Callers +// must still check Err() after scanning. +func NewLineScanner(r io.Reader) *bufio.Scanner { + s := bufio.NewScanner(r) + s.Buffer(nil, maxScannedLineSize) + return s +} diff --git a/internal/line_scanner_test.go b/internal/line_scanner_test.go new file mode 100644 index 000000000..f29f0928d --- /dev/null +++ b/internal/line_scanner_test.go @@ -0,0 +1,24 @@ +package internal + +import ( + "bufio" + "strings" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestNewLineScanner(t *testing.T) { + long := strings.Repeat("a", 100*1024) + s := NewLineScanner(strings.NewReader(long + "\nb\n")) + require.True(t, s.Scan()) + require.Equal(t, long, s.Text()) + require.True(t, s.Scan()) + require.Equal(t, "b", s.Text()) + require.False(t, s.Scan()) + require.NoError(t, s.Err()) + + s = NewLineScanner(strings.NewReader(strings.Repeat("a", maxScannedLineSize+1))) + require.False(t, s.Scan()) + require.ErrorIs(t, s.Err(), bufio.ErrTooLong) +} diff --git a/internal/os/feature_detection.go b/internal/os/feature_detection.go index 1aaf3f23c..a836a2191 100644 --- a/internal/os/feature_detection.go +++ b/internal/os/feature_detection.go @@ -6,6 +6,7 @@ import ( "errors" "fmt" "regexp" + "slices" "github.com/anchore/syft/internal" "github.com/anchore/syft/internal/log" @@ -191,12 +192,9 @@ func hasInstalledESMPackage(packages *pkg.Collection) bool { if packages == nil { return false } - for p := range packages.Enumerate(pkg.DebPkg) { - if esmVersionPattern.MatchString(p.Version) { - return true - } - } - return false + return slices.ContainsFunc(packages.Sorted(pkg.DebPkg), func(p pkg.Package) bool { + return esmVersionPattern.MatchString(p.Version) + }) } func fileMatchesAny(resolver file.Resolver, location file.Location, patterns ...*regexp.Regexp) (bool, error) { diff --git a/internal/os/feature_detection_test.go b/internal/os/feature_detection_test.go index bb8a30de1..b4f2ed38d 100644 --- a/internal/os/feature_detection_test.go +++ b/internal/os/feature_detection_test.go @@ -8,6 +8,7 @@ import ( "path/filepath" "slices" "testing" + "time" "github.com/stretchr/testify/require" @@ -232,6 +233,33 @@ func Test_findUbuntuFeatures(t *testing.T) { } } +// returning early from Collection.Enumerate leaks its goroutine and read lock, so any later write blocks forever. +func Test_findUbuntuFeatures_releasesPackageCollectionLock(t *testing.T) { + resolver := fixtureResolverForDir(t, "testdata/ubuntu_plain") + + s := sbom.SBOM{} + s.Artifacts.LinuxDistribution = &linux.Release{ID: "ubuntu"} + s.Artifacts.Packages = pkg.NewCollection( + pkg.Package{Name: "openssl", Version: "1.1.1f-1ubuntu2.19+esm1", Type: pkg.DebPkg}, + pkg.Package{Name: "libcap2", Version: "1:2.32-1ubuntu0.1~esm1", Type: pkg.DebPkg}, + ) + + require.NoError(t, os.DetectFeatures(context.Background(), resolver, sbomsync.NewBuilder(&s))) + require.True(t, s.Artifacts.LinuxDistribution.ExtendedSupport) + + added := make(chan struct{}) + go func() { + s.Artifacts.Packages.Add(pkg.Package{Name: "zlib1g", Version: "1:1.2.11.dfsg-2ubuntu1", Type: pkg.DebPkg}) + close(added) + }() + + select { + case <-added: + case <-time.After(5 * time.Second): + t.Fatal("package collection is still read-locked after feature detection") + } +} + // a read error in the apt-source signal must not suppress the ubuntu-advantage status signal; the three // signals are meant to be independent fallbacks. func Test_findUbuntuFeatures_signalErrorDoesNotSuppressOthers(t *testing.T) { diff --git a/internal/packagemetadata/untyped_fields_test.go b/internal/packagemetadata/untyped_fields_test.go new file mode 100644 index 000000000..c4bde3718 --- /dev/null +++ b/internal/packagemetadata/untyped_fields_test.go @@ -0,0 +1,74 @@ +package packagemetadata + +import ( + "reflect" + "strings" + "testing" +) + +// untypedFieldAllowlist holds package metadata fields that are allowed to carry untyped (interface) values. +// Values in these fields come straight from whatever a parser decoded, so they are not shaped by syft +// and can hold anything an encoder or hasher chokes on. Every entry is part of existing SBOM output and +// cannot be retyped without breaking it; new entries need a reason and a review. +var untypedFieldAllowlist = map[string]string{ + "GGUFFileHeader.RemainingKeyValues": "raw GGUF header key-values; retyping changes the header output and metadataHash", +} + +// TestNoUntypedMetadataFields fails when a package metadata type gains a field typed any, []any, +// map[string]any (or any other interface) that is not on the allowlist. +func TestNoUntypedMetadataFields(t *testing.T) { + var hits []string + for _, ty := range AllTypes() { + rt := reflect.TypeOf(ty) + collectUntyped(rt, rt.Name(), map[reflect.Type]bool{}, &hits) + } + + seen := map[string]bool{} + for _, h := range hits { + seen[h] = true + if _, ok := untypedFieldAllowlist[h]; !ok { + t.Errorf("untyped field %s: use a concrete type, or add it to untypedFieldAllowlist with a reason", h) + } + } + for k := range untypedFieldAllowlist { + if !seen[k] { + t.Errorf("stale untypedFieldAllowlist entry %s: field no longer exists or is now typed", k) + } + } +} + +// collectUntyped records "Struct.Field" for every field whose type reaches an interface, descending +// through pointers, slices, arrays, maps and nested syft-owned structs. +func collectUntyped(rt reflect.Type, path string, visited map[reflect.Type]bool, hits *[]string) { + switch rt.Kind() { + case reflect.Interface: + *hits = append(*hits, path) + case reflect.Pointer, reflect.Slice, reflect.Array: + collectUntyped(rt.Elem(), path, visited, hits) + case reflect.Map: + collectUntyped(rt.Key(), path, visited, hits) + collectUntyped(rt.Elem(), path, visited, hits) + case reflect.Struct: + if visited[rt] { + return + } + if rt.Name() != "" && !strings.HasPrefix(rt.PkgPath(), "github.com/anchore/syft/") { + // a third-party type in the model carries whatever shape the library gives it (the same leak as an + // interface field), so it counts as untyped. The standard library is left alone. + if strings.Contains(strings.Split(rt.PkgPath(), "/")[0], ".") { + *hits = append(*hits, path) + } + return + } + visited[rt] = true + for i := 0; i < rt.NumField(); i++ { + f := rt.Field(i) + // unexported embedded structs still promote their exported fields to the encoders and the hasher. A + // field is only out of reach when it is neither encoded nor hashed by SetID. + if (!f.IsExported() && !f.Anonymous) || (f.Tag.Get("json") == "-" && f.Tag.Get("hash") == "ignore") { + continue + } + collectUntyped(f.Type, rt.Name()+"."+f.Name, visited, hits) + } + } +} diff --git a/internal/safetensors.go b/internal/safetensors.go new file mode 100644 index 000000000..deae046f5 --- /dev/null +++ b/internal/safetensors.go @@ -0,0 +1,7 @@ +package internal + +// MaxSafeTensorsHeaderSize caps a safetensors header's JSON (excluding its +// 8-byte length prefix). Shared by the ai cataloger and the OCI model source so +// dir and OCI scans accept the same headers. Real-world shard headers are well +// under 1 MB. +const MaxSafeTensorsHeaderSize = 8 * 1024 * 1024 diff --git a/internal/spdxlicense/license.go b/internal/spdxlicense/license.go index ff8bd1265..d23491e5c 100644 --- a/internal/spdxlicense/license.go +++ b/internal/spdxlicense/license.go @@ -44,6 +44,7 @@ type LicenseInfo struct { // LicenseByURL returns the license ID and name for a given URL from the SPDX license list. // The URL should match one of the URLs in the seeAlso field of an SPDX license. // The scheme (http:// or https://) is stripped before lookup, so both schemes match. +// opensource.org URLs additionally match regardless of the form used (see normalizeOpenSourceOrgURL). func LicenseByURL(url string) (LicenseInfo, bool) { url = strings.TrimSpace(url) url = stripScheme(url) @@ -52,9 +53,51 @@ func LicenseByURL(url string) (LicenseInfo, bool) { ID: id, }, true } + if normalized, ok := normalizeOpenSourceOrgURL(url); ok { + if id, exists := openSourceOrgURLToLicense[normalized]; exists { + return LicenseInfo{ + ID: id, + }, true + } + } return LicenseInfo{}, false } +// openSourceOrgURLToLicense indexes the opensource.org entries of urlToLicense by normalized URL. +var openSourceOrgURLToLicense = buildOpenSourceOrgURLToLicense(urlToLicense) + +func buildOpenSourceOrgURLToLicense(urls map[string]string) map[string]string { + index := make(map[string]string) + ambiguous := make(map[string]bool) + for url, id := range urls { + normalized, ok := normalizeOpenSourceOrgURL(url) + if !ok || ambiguous[normalized] { + continue + } + if existing, exists := index[normalized]; exists && existing != id { + // don't guess between licenses that only differ by URL form + delete(index, normalized) + ambiguous[normalized] = true + continue + } + index[normalized] = id + } + return index +} + +// normalizeOpenSourceOrgURL reduces a scheme-less opensource.org URL to a canonical form. +// SPDX has changed these URLs over time (e.g. 3.29.0 replaced opensource.org/licenses/MIT and +// opensource.org/license/mit/ with opensource.org/license/MIT), while package metadata in the wild +// still uses the older forms. The www. prefix, /licenses/ vs /license/, case, and trailing slash are ignored. +func normalizeOpenSourceOrgURL(url string) (string, bool) { + url = strings.TrimPrefix(strings.ToLower(url), "www.") + if !strings.HasPrefix(url, "opensource.org/") { + return "", false + } + url = strings.Replace(url, "opensource.org/licenses/", "opensource.org/license/", 1) + return strings.TrimSuffix(url, "/"), true +} + // stripScheme removes http:// or https:// prefix from a URL. // This allows a single map entry to match both schemes. func stripScheme(url string) string { diff --git a/internal/spdxlicense/license_list.go b/internal/spdxlicense/license_list.go index 78e330244..1a4357378 100644 --- a/internal/spdxlicense/license_list.go +++ b/internal/spdxlicense/license_list.go @@ -2,7 +2,7 @@ // This file was generated using data from https://spdx.org/licenses/licenses.json package spdxlicense -const Version = "3.28.0" +const Version = "3.29.0" var licenseIDs = map[string]string{ "0bsd": "0BSD", @@ -116,6 +116,7 @@ var licenseIDs = map[string]string{ "aswfdigitalassets1.0.0": "ASWF-Digital-Assets-1.0", "aswfdigitalassets1.1": "ASWF-Digital-Assets-1.1", "aswfdigitalassets1.1.0": "ASWF-Digital-Assets-1.1", + "atcgame": "atc-game", "baekmuk": "Baekmuk", "bahyph": "Bahyph", "barr": "Barr", @@ -142,8 +143,11 @@ var licenseIDs = map[string]string{ "briangladman2.0clause": "Brian-Gladman-2-Clause", "briangladman2clause": "Brian-Gladman-2-Clause", "briangladman3.0.0clause": "Brian-Gladman-3-Clause", + "briangladman3.0.0clausenoconversion": "Brian-Gladman-3-Clause-no-conversion", "briangladman3.0clause": "Brian-Gladman-3-Clause", + "briangladman3.0clausenoconversion": "Brian-Gladman-3-Clause-no-conversion", "briangladman3clause": "Brian-Gladman-3-Clause", + "briangladman3clausenoconversion": "Brian-Gladman-3-Clause-no-conversion", "bsd1.0.0clause": "BSD-1-Clause", "bsd1.0clause": "BSD-1-Clause", "bsd1clause": "BSD-1-Clause", @@ -154,6 +158,7 @@ var licenseIDs = map[string]string{ "bsd2.0.0clausenetbsd": "BSD-2-Clause", "bsd2.0.0clausepatent": "BSD-2-Clause-Patent", "bsd2.0.0clausepkgconfdisclaimer": "BSD-2-Clause-pkgconf-disclaimer", + "bsd2.0.0clauseposunchanged": "BSD-2-Clause-pos-unchanged", "bsd2.0.0clauseviews": "BSD-2-Clause-Views", "bsd2.0clause": "BSD-2-Clause", "bsd2.0clausedarwin": "BSD-2-Clause-Darwin", @@ -162,6 +167,7 @@ var licenseIDs = map[string]string{ "bsd2.0clausenetbsd": "BSD-2-Clause", "bsd2.0clausepatent": "BSD-2-Clause-Patent", "bsd2.0clausepkgconfdisclaimer": "BSD-2-Clause-pkgconf-disclaimer", + "bsd2.0clauseposunchanged": "BSD-2-Clause-pos-unchanged", "bsd2.0clauseviews": "BSD-2-Clause-Views", "bsd2clause": "BSD-2-Clause", "bsd2clausedarwin": "BSD-2-Clause-Darwin", @@ -170,6 +176,7 @@ var licenseIDs = map[string]string{ "bsd2clausenetbsd": "BSD-2-Clause", "bsd2clausepatent": "BSD-2-Clause-Patent", "bsd2clausepkgconfdisclaimer": "BSD-2-Clause-pkgconf-disclaimer", + "bsd2clauseposunchanged": "BSD-2-Clause-pos-unchanged", "bsd2clauseviews": "BSD-2-Clause-Views", "bsd3.0.0clause": "BSD-3-Clause", "bsd3.0.0clauseacpica": "BSD-3-Clause-acpica", @@ -184,6 +191,7 @@ var licenseIDs = map[string]string{ "bsd3.0.0clausenonuclearlicense2014": "BSD-3-Clause-No-Nuclear-License-2014", "bsd3.0.0clausenonuclearwarranty": "BSD-3-Clause-No-Nuclear-Warranty", "bsd3.0.0clauseopenmpi": "BSD-3-Clause-Open-MPI", + "bsd3.0.0clauseopenwebui": "BSD-3-Clause-OpenWebUI", "bsd3.0.0clausesun": "BSD-3-Clause-Sun", "bsd3.0.0clausetso": "BSD-3-Clause-Tso", "bsd3.0clause": "BSD-3-Clause", @@ -199,6 +207,7 @@ var licenseIDs = map[string]string{ "bsd3.0clausenonuclearlicense2014": "BSD-3-Clause-No-Nuclear-License-2014", "bsd3.0clausenonuclearwarranty": "BSD-3-Clause-No-Nuclear-Warranty", "bsd3.0clauseopenmpi": "BSD-3-Clause-Open-MPI", + "bsd3.0clauseopenwebui": "BSD-3-Clause-OpenWebUI", "bsd3.0clausesun": "BSD-3-Clause-Sun", "bsd3.0clausetso": "BSD-3-Clause-Tso", "bsd3clause": "BSD-3-Clause", @@ -214,6 +223,7 @@ var licenseIDs = map[string]string{ "bsd3clausenonuclearlicense2014": "BSD-3-Clause-No-Nuclear-License-2014", "bsd3clausenonuclearwarranty": "BSD-3-Clause-No-Nuclear-Warranty", "bsd3clauseopenmpi": "BSD-3-Clause-Open-MPI", + "bsd3clauseopenwebui": "BSD-3-Clause-OpenWebUI", "bsd3clausesun": "BSD-3-Clause-Sun", "bsd3clausetso": "BSD-3-Clause-Tso", "bsd4.0.0clause": "BSD-4-Clause", @@ -232,12 +242,15 @@ var licenseIDs = map[string]string{ "bsd4reno": "BSD-4.3RENO", "bsd4tahoe": "BSD-4.3TAHOE", "bsdadvertisingacknowledgement": "BSD-Advertising-Acknowledgement", + "bsdasktoendorse": "BSD-ask-to-endorse", "bsdattributionhpnddisclaimer": "BSD-Attribution-HPND-disclaimer", "bsdinfernonettverk": "BSD-Inferno-Nettverk", "bsdmarkmodifications": "BSD-Mark-Modifications", "bsdprotection": "BSD-Protection", + "bsdsourcealtgpl": "BSD-Source-alt-GPL", "bsdsourcebeginningfile": "BSD-Source-beginning-file", "bsdsourcecode": "BSD-Source-Code", + "bsdsourcecodenodisclaimer": "BSD-Source-Code-no-disclaimer", "bsdsystemics": "BSD-Systemics", "bsdsystemicsw3.0.0works": "BSD-Systemics-W3Works", "bsdsystemicsw3.0works": "BSD-Systemics-W3Works", @@ -246,6 +259,7 @@ var licenseIDs = map[string]string{ "bsl1.0": "BSL-1.0", "bsl1.0.0": "BSL-1.0", "buddy": "Buddy", + "bugroff": "Bugroff", "busl1": "BUSL-1.1", "busl1.1": "BUSL-1.1", "busl1.1.0": "BUSL-1.1", @@ -316,8 +330,11 @@ var licenseIDs = map[string]string{ "ccbync3.0": "CC-BY-NC-3.0", "ccbync3.0.0": "CC-BY-NC-3.0", "ccbync3.0.0de": "CC-BY-NC-3.0-DE", + "ccbync3.0.0igo": "CC-BY-NC-3.0-IGO", "ccbync3.0de": "CC-BY-NC-3.0-DE", + "ccbync3.0igo": "CC-BY-NC-3.0-IGO", "ccbync3de": "CC-BY-NC-3.0-DE", + "ccbync3igo": "CC-BY-NC-3.0-IGO", "ccbync4": "CC-BY-NC-4.0", "ccbync4.0": "CC-BY-NC-4.0", "ccbync4.0.0": "CC-BY-NC-4.0", @@ -592,6 +609,7 @@ var licenseIDs = map[string]string{ "fair": "Fair", "fbm": "FBM", "fdkaac": "FDK-AAC", + "fdkmpegh": "FDK-MPEG-H", "fergusontwofish": "Ferguson-Twofish", "frameworx1": "Frameworx-1.0", "frameworx1.0": "Frameworx-1.0", @@ -767,6 +785,9 @@ var licenseIDs = map[string]string{ "hippocratic2": "Hippocratic-2.1", "hippocratic2.1": "Hippocratic-2.1", "hippocratic2.1.0": "Hippocratic-2.1", + "hippocratic3.0.0core": "Hippocratic-3.0-core", + "hippocratic3.0core": "Hippocratic-3.0-core", + "hippocratic3core": "Hippocratic-3.0-core", "hp1986": "HP-1986", "hp1986.0": "HP-1986", "hp1986.0.0": "HP-1986", @@ -813,6 +834,7 @@ var licenseIDs = map[string]string{ "imlib2": "Imlib2", "imlib2.0": "Imlib2", "imlib2.0.0": "Imlib2", + "informatica": "Informatica", "infozip": "Info-ZIP", "innernet2": "Inner-Net-2.0", "innernet2.0": "Inner-Net-2.0", @@ -1011,6 +1033,9 @@ var licenseIDs = map[string]string{ "mulanpsl2.0.0": "MulanPSL-2.0", "multics": "Multics", "mup": "Mup", + "mvt1": "MVT-1.1", + "mvt1.1": "MVT-1.1", + "mvt1.1.0": "MVT-1.1", "naist2003": "NAIST-2003", "naist2003.0": "NAIST-2003", "naist2003.0.0": "NAIST-2003", @@ -1489,6 +1514,8 @@ var urlToLicense = map[string]string{ "cdla.io/permissive-1-0": "CDLA-Permissive-1.0", "cdla.io/sharing-1-0": "CDLA-Sharing-1.0", "celestrak.org/publications/AIAA/2006-6753/faq.php": "SGP4", + "cgit.freebsd.org/src/plain/sys/dev/aic7xxx/aic7xxx_osm.c": "BSD-Source-alt-GPL", + "cgit.freebsd.org/src/tree/usr.bin/sockstat/sockstat.c": "BSD-2-Clause-pos-unchanged", "cgit.freedesktop.org/libbsd/tree/man/setproctitle.3bsd": "libutil-David-Nugent", "coil.apotheon.org/plaintext/01.0.txt": "COIL-1.0", "core.tcl-lang.org/tk/file?name=compat/unistd.h": "HPND-UC", @@ -1512,6 +1539,7 @@ var urlToLicense = map[string]string{ "creativecommons.org/licenses/by-nc/2.0/legalcode": "CC-BY-NC-2.0", "creativecommons.org/licenses/by-nc/2.5/legalcode": "CC-BY-NC-2.5", "creativecommons.org/licenses/by-nc/3.0/de/legalcode": "CC-BY-NC-3.0-DE", + "creativecommons.org/licenses/by-nc/3.0/igo/legalcode": "CC-BY-NC-3.0-IGO", "creativecommons.org/licenses/by-nc/3.0/legalcode": "CC-BY-NC-3.0", "creativecommons.org/licenses/by-nc/4.0/legalcode": "CC-BY-NC-4.0", "creativecommons.org/licenses/by-nd-nc/1.0/legalcode": "CC-BY-NC-ND-1.0", @@ -1564,6 +1592,7 @@ var urlToLicense = map[string]string{ "directory.fsf.org/wiki/License:Fdk": "FDK-AAC", "doc.qt.io/archives/3.3/license.html": "QPL-1.0", "doc.qt.nokia.com/3.3/license.html": "QPL-1.0", + "docs.mvt.re/en/latest/license/": "MVT-1.1", "docs.omniverse.nvidia.com/install-guide/latest/common/licenses.html": "AML-glslang", "docs.oracle.com/cd/E81115_01/html/E81116/licenses.html": "man2html", "docs.python.org/3/license.html": "Python-2.0.1", @@ -1669,6 +1698,7 @@ var urlToLicense = map[string]string{ "fedoraproject.org/wiki/Licensing:Mplus?rd=Licensing/mplus": "mplus", "fedoraproject.org/wiki/Licensing:ThorPublicLicense": "TPL-1.0", "firstdonoharm.dev/version/2/1/license.html": "Hippocratic-2.1", + "firstdonoharm.dev/version/3/0/core.txt": "Hippocratic-3.0-core", "forge.gouv.qc.ca/licence/fr/liliq-v1-1/": "LiLiQ-P-1.1", "forge.gouv.qc.ca/licence/liliq-p": "LiLiQ-R-1.1", "forge.gouv.qc.ca/licence/liliq-p/": "LiLiQ-P-1.1", @@ -1686,68 +1716,70 @@ var urlToLicense = map[string]string{ "ftp://ftp.tin.org/pub/news/utils/newsx/newsx-1.6.tar.gz": "Zeeff", "gianluca.dellavedova.org/2011/01/03/clarified-artistic-license/": "ClArtistic", "git.denx.de/?p=u-boot.git;a=blob;f=arch/powerpc/cpu/ppc4xx/miiphy.c;h=297155fdafa064b955e53e9832de93bfb0cfb85b;hb=9fab4bf4cc077c21e43941866f3f2c196f28670d": "IBM-pibs", - "git.enlightenment.org/legacy/imlib2.git/tree/COPYING": "Imlib2", - "git.kernel.org/pub/scm/docs/man-pages/man-pages.git/tree/man2/getcpu.2#n4": "Linux-man-pages-1-para", - "git.kernel.org/pub/scm/docs/man-pages/man-pages.git/tree/man2/migrate_pages.2#n8": "Linux-man-pages-copyleft-2-para", - "git.kernel.org/pub/scm/docs/man-pages/man-pages.git/tree/man2/move_pages.2#n5": "Linux-man-pages-copyleft-2-para", - "git.kernel.org/pub/scm/docs/man-pages/man-pages.git/tree/man2/set_mempolicy.2#n5": "Linux-man-pages-copyleft-var", - "git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/tree/drivers/infiniband/core/sa.h": "Linux-OpenIB", - "git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/tree/net/sunrpc/auth_gss/gss_generic_token.c?h=v4.19": "HPND-sell-variant", - "git.openldap.org/openldap/openldap/-/blob/master/COPYRIGHT#L55-63": "BSD-4.3RENO", - "git.openldap.org/openldap/openldap/-/blob/master/COPYRIGHT?ref_type=heads#L39-51": "Furuseth", - "git.openldap.org/openldap/openldap/-/blob/master/libraries/libldap/os-local.c?ref_type=heads#L19-23": "PADL", - "git.savannah.gnu.org/cgit/freetype/freetype2.git/tree/docs/FTL.TXT": "FTL", - "git.savannah.gnu.org/cgit/gcl.git/tree/gcl/lsp/gcl_loop.lsp?h=Version_2_6_13pre": "LOOP", - "git.savannah.gnu.org/cgit/gnulib.git/tree/modules/COPYING?id=7b08932179d0d6b017f7df01a2ddf6e096b038e3": "FSFULLRSD", - "git.savannah.gnu.org/cgit/indent.git/tree/doc/indent.texi?id=a74c6b4ee49397cf330b333da1042bffa60ed14f#n1788": "BSD-4.3TAHOE", - "git.savannah.gnu.org/cgit/indent.git/tree/doc/indent.texi?id=a74c6b4ee49397cf330b333da1042bffa60ed14f#n74": "Latex2e-translated-notice", - "git.savannah.gnu.org/cgit/kazlib.git/tree/except.c?id=0062df360c2d17d57f6af19b0e444c51feb99036": "Kazlib", - "git.savannah.gnu.org/cgit/wget.git/tree/util/trunc.c?h=v1.21.3&id=40747a11e44ced5a8ac628a41f879ced3e2ebce9#n6": "FSFAP-no-warranty-disclaimer", - "git.sr.ht/~nabijaczleweli/ossp-var": "OSSP", - "github.com/389ds/389-ds-base/blob/main/ldap/include/sysexits-compat.h#L15": "BSD-4.3TAHOE", - "github.com/AcademySoftwareFoundation/foundation/blob/main/digital_assets/aswf_digital_assets_license_v1.0.txt": "ASWF-Digital-Assets-1.0", - "github.com/AcademySoftwareFoundation/foundation/blob/main/digital_assets/aswf_digital_assets_license_v1.1.txt": "ASWF-Digital-Assets-1.1", - "github.com/AdaCore/gnatcoll-core/blob/master/docs/index.rst": "AdaCore-doc", - "github.com/AdaCore/gnatcoll-db/blob/master/docs/index.rst": "AdaCore-doc", - "github.com/AdaCore/xmlada/blob/master/docs/index.rst": "AdaCore-doc", - "github.com/BuildCraft/BuildCraft/blob/623d323b1868712f29f4a8b0979a02e8d1835131/LICENSE": "MMPL-1.0.1", - "github.com/CommunitySpecification/1.0/blob/master/1._Community_Specification_License-v1.md": "Community-Spec-1.0", - "github.com/CorsixTH/deps/blob/fd339a9f526d1d9c9f01ccf39e438a015da50035/licences/libgsm.txt": "TU-Berlin-2.0", - "github.com/Cyan4973/xxHash/blob/dev/doc/xxhash_spec.md": "LPD-document", - "github.com/DISIC/politique-de-contribution-open-source/blob/master/LICENSE.pdf": "etalab-2.0", - "github.com/Distrotech/hdparm/blob/4517550db29a91420fb2b020349523b1b4512df2/LICENSE.TXT": "hdparm", - "github.com/Distrotech/transfig/blob/master/transfig/transfig.c": "Xfig", - "github.com/DrItanium/maya/blob/master/LICENSE.CLIPS": "Clips", - "github.com/Dual-Life/mime-base64/blob/master/Base64.xs#L12": "metamail", - "github.com/EthicalSource/hippocratic-license/blob/58c0e646d64ff6fbee275bfe2b9492f914e3ab2a/LICENSE.txt": "Hippocratic-2.1", - "github.com/FreeRADIUS/freeradius-client/blob/master/COPYRIGHT#L32": "HPND-Fenneberg-Livingston", - "github.com/GNOME/gcr/blob/master/docs/COPYING": "GCR-docs", - "github.com/GNOME/libxml2/blob/master/dict.c": "ISC-Veillard", - "github.com/HDFGroup/hdf5/?tab=License-1-ov-file#readme": "HDF5", - "github.com/KhronosGroup/SPIRV-Cross/blob/main/LICENSES/LicenseRef-KhronosFreeUse.txt": "MIT-Khronos-old", - "github.com/KhronosGroup/glslang/blob/main/LICENSE.txt#L949": "AML-glslang", - "github.com/MariaDB/server/blob/11.6/libmysqld/lib_sql.cc": "Boehm-GC-without-fee", - "github.com/Neo23x0/sigma/blob/master/LICENSE.Detection.Rules.md": "DRL-1.0", - "github.com/OGRECave/ogre/blob/master/OgreMain/include/OgreSingleton.h#L28C3-L35C46": "Game-Programming-Gems", - "github.com/Perl/perl5/blob/blead/util.c#L6136": "Martin-Birgmeier", - "github.com/PixarAnimationStudios/OpenSubdiv/blob/v3_5_0/opensubdiv/version.cpp#L2-L22": "Pixar", - "github.com/PixarAnimationStudios/OpenSubdiv/raw/v3_5_0/LICENSE.txt": "Pixar", - "github.com/RTimothyEdwards/magic/blob/master/LICENSE": "HPND-UC-export-US", - "github.com/SELinuxProject/selinux/blob/master/libselinux/LICENSE": "libselinux-1.0", - "github.com/SWI-Prolog/packages-clib/blob/master/sha1/brg_endian.h": "Brian-Gladman-3-Clause", - "github.com/SWI-Prolog/packages-xpce/blob/161a40cd82004f731ba48024f9d30af388a7edf5/src/img/gifwrite.c#L21-L26": "FBM", - "github.com/SWI-Prolog/swipl-devel/blob/master/src/os/dtoa.c": "dtoa", - "github.com/SigmaHQ/Detection-Rule-License/blob/6ec7fbde6101d101b5b5d1fcb8f9b69fbc76c04a/LICENSE.Detection.Rules.md": "DRL-1.1", - "github.com/Slicer/Slicer/blob/main/License.txt": "3D-Slicer-1.0", - "github.com/Unidata/UDUNITS-2/blob/master/COPYRIGHT": "UCAR", - "github.com/acpica/acpica/blob/master/source/common/acfileio.c#L119": "BSD-3-Clause-acpica", - "github.com/adaptivecomputing/torque/blob/master/PBS_License.txt": "OpenPBS-2.3", - "github.com/adversary-org/wtfnmf/raw/3f2cd8235a64350a57a51b9739715edaea63ec1a/COPYING.WTFNMFPL-utf8": "WTFNMFPL", - "github.com/adversary-org/wtfnmf/raw/refs/tags/1.0/COPYING.WTFNMFPL": "WTFNMFPL", - "github.com/ap/Text-Tabs/blob/master/lib.modern/Text/Tabs.pm#L148": "TTWL", - "github.com/apache/apr/blob/trunk/LICENSE#L298C6-L298C29": "Caldera-no-preamble", - "github.com/audacious-media-player/audacious/blob/master/COPYING": "BSD-2-Clause-pkgconf-disclaimer", - "github.com/audacious-media-player/audacious/blob/master/src/audacious/main.cc": "BSD-2-Clause-pkgconf-disclaimer", + "git.enlightenment.org/legacy/imlib2.git/tree/COPYING": "Imlib2", + "git.kernel.org/pub/scm/docs/man-pages/man-pages.git/tree/man2/getcpu.2#n4": "Linux-man-pages-1-para", + "git.kernel.org/pub/scm/docs/man-pages/man-pages.git/tree/man2/migrate_pages.2#n8": "Linux-man-pages-copyleft-2-para", + "git.kernel.org/pub/scm/docs/man-pages/man-pages.git/tree/man2/move_pages.2#n5": "Linux-man-pages-copyleft-2-para", + "git.kernel.org/pub/scm/docs/man-pages/man-pages.git/tree/man2/set_mempolicy.2#n5": "Linux-man-pages-copyleft-var", + "git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/tree/drivers/infiniband/core/sa.h": "Linux-OpenIB", + "git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/tree/net/sunrpc/auth_gss/gss_generic_token.c?h=v4.19": "HPND-sell-variant", + "git.openldap.org/openldap/openldap/-/blob/master/COPYRIGHT#L55-63": "BSD-4.3RENO", + "git.openldap.org/openldap/openldap/-/blob/master/COPYRIGHT?ref_type=heads#L39-51": "Furuseth", + "git.openldap.org/openldap/openldap/-/blob/master/libraries/libldap/os-local.c?ref_type=heads#L19-23": "PADL", + "git.savannah.gnu.org/cgit/freetype/freetype2.git/tree/docs/FTL.TXT": "FTL", + "git.savannah.gnu.org/cgit/gcl.git/tree/gcl/lsp/gcl_loop.lsp?h=Version_2_6_13pre": "LOOP", + "git.savannah.gnu.org/cgit/gnulib.git/tree/modules/COPYING?id=7b08932179d0d6b017f7df01a2ddf6e096b038e3": "FSFULLRSD", + "git.savannah.gnu.org/cgit/indent.git/tree/doc/indent.texi?id=a74c6b4ee49397cf330b333da1042bffa60ed14f#n1788": "BSD-4.3TAHOE", + "git.savannah.gnu.org/cgit/indent.git/tree/doc/indent.texi?id=a74c6b4ee49397cf330b333da1042bffa60ed14f#n74": "Latex2e-translated-notice", + "git.savannah.gnu.org/cgit/kazlib.git/tree/except.c?id=0062df360c2d17d57f6af19b0e444c51feb99036": "Kazlib", + "git.savannah.gnu.org/cgit/wget.git/tree/util/trunc.c?h=v1.21.3&id=40747a11e44ced5a8ac628a41f879ced3e2ebce9#n6": "FSFAP-no-warranty-disclaimer", + "git.sr.ht/~nabijaczleweli/ossp-var": "OSSP", + "github.com/389ds/389-ds-base/blob/main/ldap/include/sysexits-compat.h#L15": "BSD-4.3TAHOE", + "github.com/AcademySoftwareFoundation/foundation/blob/main/digital_assets/aswf_digital_assets_license_v1.0.txt": "ASWF-Digital-Assets-1.0", + "github.com/AcademySoftwareFoundation/foundation/blob/main/digital_assets/aswf_digital_assets_license_v1.1.txt": "ASWF-Digital-Assets-1.1", + "github.com/AdaCore/gnatcoll-core/blob/master/docs/index.rst": "AdaCore-doc", + "github.com/AdaCore/gnatcoll-db/blob/master/docs/index.rst": "AdaCore-doc", + "github.com/AdaCore/xmlada/blob/master/docs/index.rst": "AdaCore-doc", + "github.com/BuildCraft/BuildCraft/blob/623d323b1868712f29f4a8b0979a02e8d1835131/LICENSE": "MMPL-1.0.1", + "github.com/CommunitySpecification/1.0/blob/master/1._Community_Specification_License-v1.md": "Community-Spec-1.0", + "github.com/CorsixTH/deps/blob/fd339a9f526d1d9c9f01ccf39e438a015da50035/licences/libgsm.txt": "TU-Berlin-2.0", + "github.com/Cyan4973/xxHash/blob/dev/doc/xxhash_spec.md": "LPD-document", + "github.com/DISIC/politique-de-contribution-open-source/blob/master/LICENSE.pdf": "etalab-2.0", + "github.com/Distrotech/hdparm/blob/4517550db29a91420fb2b020349523b1b4512df2/LICENSE.TXT": "hdparm", + "github.com/Distrotech/transfig/blob/master/transfig/transfig.c": "Xfig", + "github.com/DrItanium/maya/blob/master/LICENSE.CLIPS": "Clips", + "github.com/Dual-Life/mime-base64/blob/master/Base64.xs#L12": "metamail", + "github.com/EthicalSource/hippocratic-license/blob/58c0e646d64ff6fbee275bfe2b9492f914e3ab2a/LICENSE.txt": "Hippocratic-2.1", + "github.com/Fraunhofer-IIS/mpeghdec/blob/main/LICENSE.txt": "FDK-MPEG-H", + "github.com/FreeRADIUS/freeradius-client/blob/master/COPYRIGHT#L32": "HPND-Fenneberg-Livingston", + "github.com/GNOME/gcr/blob/master/docs/COPYING": "GCR-docs", + "github.com/GNOME/libxml2/blob/master/dict.c": "ISC-Veillard", + "github.com/HDFGroup/hdf5/?tab=License-1-ov-file#readme": "HDF5", + "github.com/KhronosGroup/SPIRV-Cross/blob/main/LICENSES/LicenseRef-KhronosFreeUse.txt": "MIT-Khronos-old", + "github.com/KhronosGroup/glslang/blob/main/LICENSE.txt#L949": "AML-glslang", + "github.com/MariaDB/server/blob/11.6/libmysqld/lib_sql.cc": "Boehm-GC-without-fee", + "github.com/Neo23x0/sigma/blob/master/LICENSE.Detection.Rules.md": "DRL-1.0", + "github.com/OGRECave/ogre/blob/master/OgreMain/include/OgreSingleton.h#L28C3-L35C46": "Game-Programming-Gems", + "github.com/Perl/perl5/blob/blead/util.c#L6136": "Martin-Birgmeier", + "github.com/PicoEFI/PicoEFI/blob/2263ac1a2bc98e5534d64a4896d3cfbbae00745a/LICENSES/LicenseRef-scancode-bsd-no-disclaimer-unmodified.txt": "BSD-Source-Code-no-disclaimer", + "github.com/PixarAnimationStudios/OpenSubdiv/blob/v3_5_0/opensubdiv/version.cpp#L2-L22": "Pixar", + "github.com/PixarAnimationStudios/OpenSubdiv/raw/v3_5_0/LICENSE.txt": "Pixar", + "github.com/RTimothyEdwards/magic/blob/master/LICENSE": "HPND-UC-export-US", + "github.com/SELinuxProject/selinux/blob/master/libselinux/LICENSE": "libselinux-1.0", + "github.com/SWI-Prolog/packages-clib/blob/master/sha1/brg_endian.h": "Brian-Gladman-3-Clause", + "github.com/SWI-Prolog/packages-xpce/blob/161a40cd82004f731ba48024f9d30af388a7edf5/src/img/gifwrite.c#L21-L26": "FBM", + "github.com/SWI-Prolog/swipl-devel/blob/master/src/os/dtoa.c": "dtoa", + "github.com/SigmaHQ/Detection-Rule-License/blob/6ec7fbde6101d101b5b5d1fcb8f9b69fbc76c04a/LICENSE.Detection.Rules.md": "DRL-1.1", + "github.com/Slicer/Slicer/blob/main/License.txt": "3D-Slicer-1.0", + "github.com/Unidata/UDUNITS-2/blob/master/COPYRIGHT": "UCAR", + "github.com/acpica/acpica/blob/master/source/common/acfileio.c#L119": "BSD-3-Clause-acpica", + "github.com/adaptivecomputing/torque/blob/master/PBS_License.txt": "OpenPBS-2.3", + "github.com/adversary-org/wtfnmf/raw/3f2cd8235a64350a57a51b9739715edaea63ec1a/COPYING.WTFNMFPL-utf8": "WTFNMFPL", + "github.com/adversary-org/wtfnmf/raw/refs/tags/1.0/COPYING.WTFNMFPL": "WTFNMFPL", + "github.com/ap/Text-Tabs/blob/master/lib.modern/Text/Tabs.pm#L148": "TTWL", + "github.com/apache/apr/blob/trunk/LICENSE#L298C6-L298C29": "Caldera-no-preamble", + "github.com/audacious-media-player/audacious/blob/master/COPYING": "BSD-2-Clause-pkgconf-disclaimer", + "github.com/audacious-media-player/audacious/blob/master/src/audacious/main.cc": "BSD-2-Clause-pkgconf-disclaimer", "github.com/aws/mit-0": "MIT-0", "github.com/awsdocs/aws-cloud9-user-guide/blob/master/LICENSE-SAMPLECODE": "MIT-0", "github.com/bagder/curl/blob/master/COPYING": "curl", @@ -1772,6 +1804,7 @@ var urlToLicense = map[string]string{ "github.com/file/file/blob/master/COPYING": "BSD-2-Clause-Darwin", "github.com/freebsd/freebsd-src/blob/main/lib/msun/src/e_acosh.c": "SunPro", "github.com/freebsd/freebsd-src/blob/main/lib/msun/src/e_lgammal.c": "SunPro", + "github.com/freebsd/pkg/blob/main/src/add.c": "BSD-2-Clause-pos-unchanged", "github.com/gabegundy/atlserver/blob/master/License.txt": "MS-LPL", "github.com/greymass/swift-eosio/blob/master/LICENSE": "BSD-3-Clause-No-Military-License", "github.com/hamano/man2html/blob/master/man2html.c": "man2html", @@ -1784,6 +1817,7 @@ var urlToLicense = map[string]string{ "github.com/jsommers/pytricia/blob/master/patricia.c#L33-L67": "BSD-4-Clause", "github.com/kfish/xsel/blob/master/COPYING": "HPND-sell-variant", "github.com/kohler/t1utils/blob/master/LICENSE": "MIT-Click", + "github.com/krb5/krb5/blob/kfw-4.0.1-beta1/src/lib/crypto/builtin/aes/aesopt.h": "Brian-Gladman-3-Clause-no-conversion", "github.com/krb5/krb5/blob/krb5-1.21.2-final/NOTICE#L111-L133": "HPND-export2-US", "github.com/krb5/krb5/blob/krb5-1.21.2-final/NOTICE#L1157-L1182": "HPND-export-US-modify", "github.com/krb5/krb5/blob/krb5-1.21.2-final/NOTICE#L140-L156": "Brian-Gladman-2-Clause", @@ -1793,6 +1827,7 @@ var urlToLicense = map[string]string{ "github.com/krb5/krb5/blob/krb5-1.21.2-final/NOTICE#L831-L852": "HPND-export-US-acknowledgement", "github.com/krzyzanowskim/CryptoSwift/blob/main/LICENSE": "CryptoSwift", "github.com/lattera/freebsd/blob/master/sys/cam/cam.c#L4": "BSD-Source-beginning-file", + "github.com/libarchive/libarchive/blob/master/libarchive_fe/line_reader.c": "BSD-2-Clause-pos-unchanged", "github.com/libcheck/check/blob/master/checkmk/checkmk.in": "checkmk", "github.com/libjpeg-turbo/libjpeg-turbo/blob/main/README.ijg#L117-L161": "IJG", "github.com/lsof-org/lsof/blob/master/COPYING": "lsof", @@ -1805,6 +1840,7 @@ var urlToLicense = map[string]string{ "github.com/mm2/Little-CMS/blob/master/src/cmssm.c#L207": "softSurfer", "github.com/mruby/mruby/blob/83d12f8d52522cdb7c8cc46fad34821359f453e6/mrbgems/mruby-dir/src/Win/dirent.c#L127-L140": "HPND-Kevlin-Henney", "github.com/mtoyoda/sl/blob/master/LICENSE": "SL", + "github.com/mvt-project/mvt/blob/main/LICENSE": "MVT-1.1", "github.com/n8n-io/n8n/blob/master/LICENSE.md": "SUL-1.0", "github.com/ncbi/datasets/blob/0ea4cd16b61e5b799d9cc55aecfa016d6c9bd2bf/LICENSE.md": "NCBI-PD", "github.com/ncbi/datasets/blob/master/LICENSE.md": "NCBI-PD", @@ -1815,6 +1851,7 @@ var urlToLicense = map[string]string{ "github.com/novnc/noVNC/blob/master/core/crypto/des.js#L24": "Widget-Workshop", "github.com/oetiker/rrdtool-1.x/blob/master/src/plbasename.c#L8-L11": "gtkbook", "github.com/open-quantum-safe/liboqs/blob/40b01fdbb270f8614fde30e65d30e9da18c02393/src/common/rand/rand_nist.c#L1-L15": "NIST-Software", + "github.com/open-webui/open-webui/blob/main/LICENSE": "BSD-3-Clause-OpenWebUI", "github.com/openjdk/jdk/blob/6162e2c5213c5dd7c1127fd9616b543efa898962/LICENSE": "GPL-2.0-or-later", "github.com/openssh/openssh-portable/blob/1b11ea7c58cd5c59838b5fa574cd456d6047b2d4/LICENCE#L10": "SSH-OpenSSH", "github.com/openssh/openssh-portable/blob/1b11ea7c58cd5c59838b5fa574cd456d6047b2d4/pathnames.h": "SSH-short", @@ -1846,6 +1883,7 @@ var urlToLicense = map[string]string{ "github.com/sigmavirus24/x11-ssh-askpass/blob/master/dynlist.c": "HPND-sell-variant-MIT-disclaimer-rev", "github.com/signal11/hidapi/blob/master/LICENSE-orig.txt": "HIDAPI", "github.com/slogan621/gtkbook": "gtkbook", + "github.com/sudo-project/sudo/blob/c1307ea9ff340ce0538779f8e456501461fc44b7/plugins/sudoers/redblack.c#L24-L43": "BSD-ask-to-endorse", "github.com/swh/ladspa/blob/7bf6f3799fdba70fda297c2d8fd9f526803d9680/gsm/COPYRIGHT": "TU-Berlin-1.0", "github.com/tats/w3m/blob/master/COPYING": "w3m", "github.com/tcheneau/Routing/blob/f09f46fcfe636107f22f2c98348188a65a135d98/README.md": "NIST-PD", @@ -1856,6 +1894,7 @@ var urlToLicense = map[string]string{ "github.com/uim/libgcroots/blob/master/COPYING": "Boehm-GC", "github.com/usnistgov/fipy/blob/86aaa5c2ba2c6f1be19593c5986071cf6568cc34/LICENSE.rst": "NIST-PD-fallback", "github.com/usnistgov/jsip/blob/59700e6926cbe96c5cdae897d9a7d2656b42abe3/LICENSE": "NIST-PD-fallback", + "github.com/vattam/BSDGames/blob/master/atc/extern.c#L36": "atc-game", "github.com/vixie/cron/tree/545b3f5246824a9cda5905eeb7cf019c95e66995": "Vixie-Cron", "github.com/vstroebel/jpeg-encoder/blob/main/src/fdct.rs#L1-L72": "IJG", "github.com/wernerd/ZRTPCPP/blob/6b3cd8e6783642292bad0c21e3e5e5ce45ff3e03/cryptcommon/twofish.c#L113C3-L127": "Ferguson-Twofish", @@ -1869,6 +1908,7 @@ var urlToLicense = map[string]string{ "gitlab.com/embeddable-common-lisp/ecl/-/blob/develop/src/lsp/loop.lsp": "LOOP", "gitlab.com/libeigen/eigen/-/blob/master/COPYING.MINPACK": "Minpack", "gitlab.common-lisp.net/cmucl/cmucl/-/blob/master/src/code/loop.lisp": "LOOP", + "gitlab.freedesktop.org/dbus/dbus/-/blob/789d97ad53044f94351a958e785c2923643b346a/COPYING#L11-207": "AFL-2.1", "gitlab.freedesktop.org/freetype/freetype/-/raw/master/docs/FTL.TXT": "FTL", "gitlab.freedesktop.org/libopenraw/libopenraw/blob/master/lib/ljpegdecompressor.cpp#L32": "Cornell-Lossless-JPEG", "gitlab.freedesktop.org/mesa/glw/-/blob/master/README?ref_type=heads": "SGI-OpenGL", @@ -1892,6 +1932,7 @@ var urlToLicense = map[string]string{ "gitlab.freedesktop.org/xorg/lib/libxinerama/-/blob/cc22c2f60c3862482562955116d5455263b443dc/COPYING#L44-66": "X11-no-permit-persons", "gitlab.freedesktop.org/xorg/lib/libxtst/-/blob/master/COPYING?ref_type=heads#L108-117": "HPND-doc-sell", "gitlab.freedesktop.org/xorg/lib/libxtst/-/blob/master/COPYING?ref_type=heads#L70-77": "HPND-doc", + "gitlab.freedesktop.org/xorg/xserver/-/blob/dd5c2595a42d3ff0c4f18d9b53d1f6c3fd934fd4/COPYING#L365-389": "MIT", "gitlab.freedesktop.org/xorg/xserver/-/blob/master/COPYING?ref_type=heads#L1781": "HPND-sell-MIT-disclaimer-xserver", "gitlab.freedesktop.org/xorg/xserver/-/blob/master/COPYING?ref_type=heads#L239": "DEC-3-Clause", "gitlab.freedesktop.org/xorg/xserver/-/blob/master/COPYING?ref_type=heads#L752": "Adobe-Display-PostScript", @@ -1906,6 +1947,7 @@ var urlToLicense = map[string]string{ "heasarc.gsfc.nasa.gov/docs/software/ftools/fv/doc/license.html": "CFITSIO", "helixcommunity.org/content/rpsl": "RPSL-1.0", "hstechdocs.helpsystems.com/manuals/globalscape/archive/cuteftp6/open_ssl_license_agreement.htm": "OpenSSL-standalone", + "interoperable-europe.ec.europa.eu/sites/default/files/inline-files/EUPL%20v1_2%20EN(1).txt": "EUPL-1.2", "java.net/projects/javaeetutorial/pages/BerkeleyLicense": "BSD-3-Clause-No-Nuclear-License-2014", "javaee.github.io/glassfish/LICENSE": "CDDL-1.1", "jogamp.org/git/?p=gluegen.git;a=blob_plain;f=LICENSE.txt": "BSD-3-Clause-No-Nuclear-Warranty", @@ -1924,6 +1966,7 @@ var urlToLicense = map[string]string{ "library.netapp.com/ecm/ecm_download_file/ECMP1196395": "OpenSSL-standalone", "license.coscl.org.cn/MulanPSL/": "MulanPSL-1.0", "license.coscl.org.cn/MulanPSL2": "MulanPSL-2.0", + "license.mvt.re/1.1/": "MVT-1.1", "lists.gnu.org/archive/html/autoconf/2012-04/msg00061.html": "FSFULLRWD", "lists.opensource.org/pipermail/license-discuss_lists.opensource.org/2002-November/006304.html": "HPND", "mariadb.com/bsl11/": "BUSL-1.1", @@ -1965,99 +2008,105 @@ var urlToLicense = map[string]string{ "opensource.linux-mirror.org/licenses/afl-1.1.txt": "AFL-1.1", "opensource.linux-mirror.org/licenses/afl-1.2.txt": "AFL-1.2", "opensource.linux-mirror.org/licenses/afl-2.1.txt": "AFL-2.1", - "opensource.org/license/apache-2-0": "Apache-2.0", - "opensource.org/license/mit/": "MIT", - "opensource.org/license/olfl-1-3/": "OLFL-1.3", - "opensource.org/license/osc-license-1-0": "OSC-1.0", - "opensource.org/licenses/0BSD": "0BSD", - "opensource.org/licenses/AGPL-3.0": "AGPL-3.0-or-later", - "opensource.org/licenses/APL-1.0": "APL-1.0", - "opensource.org/licenses/Apache-1.1": "Apache-1.1", - "opensource.org/licenses/Apache-2.0": "Apache-2.0", - "opensource.org/licenses/Artistic-1.0": "Artistic-1.0-cl8", - "opensource.org/licenses/BSD-2-Clause": "BSD-2-Clause", - "opensource.org/licenses/BSD-3-Clause": "BSD-3-Clause", - "opensource.org/licenses/BSDplusPatent": "BSD-2-Clause-Patent", - "opensource.org/licenses/BSL-1.0": "BSL-1.0", - "opensource.org/licenses/CAL-1.0": "CAL-1.0-Combined-Work-Exception", - "opensource.org/licenses/CATOSL-1.1": "CATOSL-1.1", - "opensource.org/licenses/CNRI-Python": "CNRI-Python", - "opensource.org/licenses/CPAL-1.0": "CPAL-1.0", - "opensource.org/licenses/CPL-1.0": "CPL-1.0", - "opensource.org/licenses/CUA-OPL-1.0": "CUA-OPL-1.0", - "opensource.org/licenses/ECL-1.0": "ECL-1.0", - "opensource.org/licenses/ECL-2.0": "ECL-2.0", - "opensource.org/licenses/EFL-1.0": "EFL-1.0", - "opensource.org/licenses/EFL-2.0": "EFL-2.0", - "opensource.org/licenses/EPL-1.0": "EPL-1.0", - "opensource.org/licenses/EUDatagrid": "EUDatagrid", - "opensource.org/licenses/EUPL-1.1": "EUPL-1.1", - "opensource.org/licenses/EUPL-1.2": "EUPL-1.2", - "opensource.org/licenses/Entessa": "Entessa", - "opensource.org/licenses/Fair": "Fair", - "opensource.org/licenses/Frameworx-1.0": "Frameworx-1.0", - "opensource.org/licenses/GPL-2.0": "GPL-2.0-or-later", - "opensource.org/licenses/GPL-3.0": "GPL-3.0-or-later", - "opensource.org/licenses/HPND": "HPND", - "opensource.org/licenses/IPA": "IPA", - "opensource.org/licenses/IPL-1.0": "IPL-1.0", - "opensource.org/licenses/ISC": "ISC", - "opensource.org/licenses/Intel": "Intel", - "opensource.org/licenses/LGPL-2.1": "LGPL-2.1-or-later", - "opensource.org/licenses/LGPL-3.0": "LGPL-3.0-or-later", - "opensource.org/licenses/LPL-1.0": "LPL-1.0", - "opensource.org/licenses/LPL-1.02": "LPL-1.02", - "opensource.org/licenses/LPPL-1.3c": "LPPL-1.3c", - "opensource.org/licenses/LiLiQ-P-1.1": "LiLiQ-P-1.1", - "opensource.org/licenses/LiLiQ-R-1.1": "LiLiQ-R-1.1", - "opensource.org/licenses/LiLiQ-Rplus-1.1": "LiLiQ-Rplus-1.1", - "opensource.org/licenses/MIT": "MIT", - "opensource.org/licenses/MPL-1.0": "MPL-1.0", - "opensource.org/licenses/MPL-1.1": "MPL-1.1", - "opensource.org/licenses/MPL-2.0": "MPL-2.0-no-copyleft-exception", - "opensource.org/licenses/MS-PL": "MS-PL", - "opensource.org/licenses/MS-RL": "MS-RL", - "opensource.org/licenses/MirOS": "MirOS", - "opensource.org/licenses/Motosoto": "Motosoto", - "opensource.org/licenses/Multics": "Multics", - "opensource.org/licenses/NASA-1.3": "NASA-1.3", - "opensource.org/licenses/NCSA": "NCSA", - "opensource.org/licenses/NGPL": "NGPL", - "opensource.org/licenses/NOSL3.0": "NPOSL-3.0", - "opensource.org/licenses/NTP": "NTP", - "opensource.org/licenses/Naumen": "Naumen", - "opensource.org/licenses/OCLC-2.0": "OCLC-2.0", - "opensource.org/licenses/OFL-1.1": "OFL-1.1-no-RFN", - "opensource.org/licenses/OGTSL": "OGTSL", - "opensource.org/licenses/OPL-2.1": "OSET-PL-2.1", - "opensource.org/licenses/OSL-1.0": "OSL-1.0", - "opensource.org/licenses/OSL-2.1": "OSL-2.1", - "opensource.org/licenses/OSL-3.0": "OSL-3.0", - "opensource.org/licenses/PHP-3.0": "PHP-3.0", - "opensource.org/licenses/PostgreSQL": "PostgreSQL", - "opensource.org/licenses/Python-2.0": "Python-2.0", - "opensource.org/licenses/QPL-1.0": "QPL-1.0", - "opensource.org/licenses/RPL-1.1": "RPL-1.1", - "opensource.org/licenses/RPL-1.5": "RPL-1.5", - "opensource.org/licenses/RPSL-1.0": "RPSL-1.0", - "opensource.org/licenses/RSCPL": "RSCPL", - "opensource.org/licenses/SISSL": "SISSL", - "opensource.org/licenses/SPL-1.0": "SPL-1.0", - "opensource.org/licenses/SimPL-2.0": "SimPL-2.0", - "opensource.org/licenses/Sleepycat": "Sleepycat", - "opensource.org/licenses/UCL-1.0": "UCL-1.0", - "opensource.org/licenses/UPL": "UPL-1.0", - "opensource.org/licenses/VSL-1.0": "VSL-1.0", - "opensource.org/licenses/W3C": "W3C", - "opensource.org/licenses/Watcom-1.0": "Watcom-1.0", - "opensource.org/licenses/Xnet": "Xnet", - "opensource.org/licenses/ZPL-2.0": "ZPL-2.0", - "opensource.org/licenses/Zlib": "Zlib", - "opensource.org/licenses/afl-3.0": "AFL-3.0", - "opensource.org/licenses/artistic-license-2.0": "Artistic-2.0", - "opensource.org/licenses/attribution": "AAL", - "opensource.org/licenses/cddl1": "CDDL-1.0", - "opensource.org/licenses/nokia": "Nokia", + "opensource.org/license/0BSD": "0BSD", + "opensource.org/license/AAL": "AAL", + "opensource.org/license/AFL-3.0": "AFL-3.0", + "opensource.org/license/AGPL-3.0": "AGPL-3.0-or-later", + "opensource.org/license/AGPL-3.0-only": "AGPL-3.0-only", + "opensource.org/license/APL-1.0": "APL-1.0", + "opensource.org/license/Artistic-1.0": "Artistic-1.0-cl8", + "opensource.org/license/Artistic-2.0": "Artistic-2.0", + "opensource.org/license/BSD-2-Clause": "BSD-2-Clause", + "opensource.org/license/BSD-2-Clause-Patent": "BSD-2-Clause-Patent", + "opensource.org/license/BSD-3-Clause": "BSD-3-Clause", + "opensource.org/license/BSL-1.0": "BSL-1.0", + "opensource.org/license/CAL-1.0": "CAL-1.0-Combined-Work-Exception", + "opensource.org/license/CATOSL-1.1": "CATOSL-1.1", + "opensource.org/license/CDDL-1.0": "CDDL-1.0", + "opensource.org/license/CNRI-Python": "CNRI-Python", + "opensource.org/license/CNRI-Python-GPL-Compatible": "CNRI-Python-GPL-Compatible", + "opensource.org/license/CPAL-1.0": "CPAL-1.0", + "opensource.org/license/CPL-1.0": "CPL-1.0", + "opensource.org/license/CUA-OPL-1.0": "CUA-OPL-1.0", + "opensource.org/license/ECL-1.0": "ECL-1.0", + "opensource.org/license/ECL-2.0": "ECL-2.0", + "opensource.org/license/EFL-1.0": "EFL-1.0", + "opensource.org/license/EFL-2.0": "EFL-2.0", + "opensource.org/license/EPL-1.0": "EPL-1.0", + "opensource.org/license/EUDatagrid": "EUDatagrid", + "opensource.org/license/EUPL-1.1": "EUPL-1.1", + "opensource.org/license/EUPL-1.2": "EUPL-1.2", + "opensource.org/license/Entessa": "Entessa", + "opensource.org/license/Fair": "Fair", + "opensource.org/license/Frameworx-1.0": "Frameworx-1.0", + "opensource.org/license/GPL-2.0": "GPL-2.0-or-later", + "opensource.org/license/GPL-2.0-only": "GPL-2.0-only", + "opensource.org/license/GPL-3.0": "GPL-3.0-or-later", + "opensource.org/license/GPL-3.0-only": "GPL-3.0-only", + "opensource.org/license/HPND": "HPND", + "opensource.org/license/IPA": "IPA", + "opensource.org/license/IPL-1.0": "IPL-1.0", + "opensource.org/license/ISC": "ISC", + "opensource.org/license/Intel": "Intel", + "opensource.org/license/LGPL-2.1": "LGPL-2.1-or-later", + "opensource.org/license/LGPL-2.1-only": "LGPL-2.1-only", + "opensource.org/license/LGPL-3.0": "LGPL-3.0-or-later", + "opensource.org/license/LPL-1.0": "LPL-1.0", + "opensource.org/license/LPL-1.02": "LPL-1.02", + "opensource.org/license/LPPL-1.3c": "LPPL-1.3c", + "opensource.org/license/LiLiQ-P-1.1": "LiLiQ-P-1.1", + "opensource.org/license/LiLiQ-R-1.1": "LiLiQ-R-1.1", + "opensource.org/license/LiLiQ-Rplus-1.1": "LiLiQ-Rplus-1.1", + "opensource.org/license/MIT": "MIT", + "opensource.org/license/MPL-1.0": "MPL-1.0", + "opensource.org/license/MPL-1.1": "MPL-1.1", + "opensource.org/license/MPL-2.0": "MPL-2.0-no-copyleft-exception", + "opensource.org/license/MS-PL": "MS-PL", + "opensource.org/license/MS-RL": "MS-RL", + "opensource.org/license/MirOS": "MirOS", + "opensource.org/license/Motosoto": "Motosoto", + "opensource.org/license/Multics": "Multics", + "opensource.org/license/NASA-1.3": "NASA-1.3", + "opensource.org/license/NCSA": "NCSA", + "opensource.org/license/NGPL": "NGPL", + "opensource.org/license/NPOSL-3.0": "NPOSL-3.0", + "opensource.org/license/NTP": "NTP", + "opensource.org/license/Naumen": "Naumen", + "opensource.org/license/Nokia": "Nokia", + "opensource.org/license/OCLC-2.0": "OCLC-2.0", + "opensource.org/license/OFL-1.1": "OFL-1.1-no-RFN", + "opensource.org/license/OGTSL": "OGTSL", + "opensource.org/license/OLFL-1.3": "OLFL-1.3", + "opensource.org/license/OSC-1.0": "OSC-1.0", + "opensource.org/license/OSET-PL-2.1": "OSET-PL-2.1", + "opensource.org/license/OSL-1.0": "OSL-1.0", + "opensource.org/license/OSL-2.1": "OSL-2.1", + "opensource.org/license/OSL-3.0": "OSL-3.0", + "opensource.org/license/PHP-3.0": "PHP-3.0", + "opensource.org/license/PSF-2-0": "PSF-2.0", + "opensource.org/license/PostgreSQL": "PostgreSQL", + "opensource.org/license/Python-2.0": "Python-2.0", + "opensource.org/license/Python-2.0.1": "Python-2.0.1", + "opensource.org/license/QPL-1.0": "QPL-1.0", + "opensource.org/license/RPL-1.1": "RPL-1.1", + "opensource.org/license/RPL-1.5": "RPL-1.5", + "opensource.org/license/RPSL-1.0": "RPSL-1.0", + "opensource.org/license/RSCPL": "RSCPL", + "opensource.org/license/SISSL": "SISSL", + "opensource.org/license/SPL-1.0": "SPL-1.0", + "opensource.org/license/SiMPL-2.0": "SimPL-2.0", + "opensource.org/license/Sleepycat": "Sleepycat", + "opensource.org/license/UCL-1.0": "UCL-1.0", + "opensource.org/license/UPL-1.0": "UPL-1.0", + "opensource.org/license/VSL-1.0": "VSL-1.0", + "opensource.org/license/W3C": "W3C", + "opensource.org/license/Watcom-1.0": "Watcom-1.0", + "opensource.org/license/Xnet": "Xnet", + "opensource.org/license/ZPL-2.0": "ZPL-2.0", + "opensource.org/license/Zlib": "Zlib", + "opensource.org/license/apache-1.1": "Apache-1.1", + "opensource.org/license/apache-2.0": "Apache-2.0", + "opensource.org/license/curl": "curl", "oss.sgi.com/projects/FreeB/": "SGI-B-1.1", "oss.sgi.com/projects/FreeB/SGIFreeSWLicB.1.0.html": "SGI-B-1.0", "oss.sgi.com/projects/FreeB/SGIFreeSWLicB.2.0.pdf": "SGI-B-2.0", @@ -2082,37 +2131,40 @@ var urlToLicense = map[string]string{ "raw.githubusercontent.com/NTIA/scos-sensor/refs/heads/master/LICENSE.md": "NTIA-PD", "raw.githubusercontent.com/OpenMDW/OpenMDW/refs/heads/main/1.0/LICENSE.openmdw": "OpenMDW-1.0", "raw.githubusercontent.com/blackhole89/notekit/master/data/fonts/Charter%20license.txt": "Bitstream-Charter", - "rc.quest.com/topics/openssh/license.php#tcpwrappers": "TCP-wrappers", - "research.cs.wisc.edu/condor/license.html#condor": "Condor-1.1", - "research.scea.com/scea_shared_source_license.html": "SCEA", - "romanrm.net/mit-zero": "MIT-0", - "sam.zoy.org/wtfpl/COPYING": "WTFPL", - "scripts.sil.org/cms/scripts/page.php?item_id=OFL10_web": "OFL-1.0-no-RFN", - "scripts.sil.org/cms/scripts/page.php?item_id=OFL_web": "OFL-1.1-no-RFN", - "slicer.org/LICENSE": "3D-Slicer-1.0", - "solderpad.org/licenses/SHL-0.5/": "SHL-0.5", - "solderpad.org/licenses/SHL-0.51/": "SHL-0.51", - "source.icu-project.org/repos/icu/icu/trunk/license.html": "ICU", - "sourceforge.net/p/buddy/gitcode/ci/master/tree/README": "Buddy", - "sourceforge.net/p/ctrio/git/ci/master/tree/README": "ISC-Veillard", - "sourceforge.net/p/mcj/xfig/ci/master/tree/src/Makefile.am": "Xfig", - "sourceforge.net/p/netpbm/code/HEAD/tree/super_stable/converter/other/pnmtorle.c": "URT-RLE", - "sourceforge.net/p/netpbm/code/HEAD/tree/super_stable/converter/other/rletopnm.c": "URT-RLE", - "sourceforge.net/p/netpbm/code/HEAD/tree/super_stable/converter/ppm/ppmtompeg/jrevdct.c#l1189": "MPEG-SSG", - "sourceforge.net/p/netpbm/code/HEAD/tree/super_stable/editor/pnmstitch.c#l2": "pnmstitch", - "sourceforge.net/p/netpbm/code/HEAD/tree/super_stable/netpbm.c#l8": "HPND-Pbmplus", - "sourceforge.net/p/sbcl/sbcl/ci/master/tree/src/code/loop.lisp": "LOOP", - "sourceforge.net/p/trustedqsl/tqsl/ci/master/tree/LICENSE.txt": "TrustedQSL", - "sourceforge.net/p/xmedcon/code/ci/master/tree/libs/dicom/README": "OFFIS", - "sourceforge.net/p/xmedcon/code/ci/master/tree/libs/ljpg/": "IJG-short", - "sourceforge.net/projects/sevenzip/files/LZMA%20SDK/": "LZMA-SDK-9.22", - "sources.debian.org/copyright/license/debianutils/4.11.2/": "SMAIL-GPL", - "sources.debian.org/src/openmpi/4.1.0-10/ompi/debuggers/msgq_interface.h/?hl=19#L19": "mpi-permissive", - "sources.gentoo.org/cgi-bin/viewvc.cgi/gentoo-x86/licenses/BitTorrent?r1=1.1&r2=1.1.1.1&diff_format=s": "BitTorrent-1.0", - "sourceware.org/bzip2/1.0.5/bzip2-manual-1.0.5.html": "bzip2-1.0.6", - "sourceware.org/cgit/binutils-gdb/tree/include/coff/sym.h#n11": "MIPS", - "sourceware.org/cgit/binutils-gdb/tree/include/coff/symconst.h#n11": "ThirdEye", - "sourceware.org/cgit/valgrind/tree/mpi/libmpiwrap.c": "bzip2-1.0.6", + "raw.githubusercontent.com/landtuna/open-mtools/6c04c2882eca7c4313ec267766ea53ac9bfe129b/mdump.c": "Informatica", + "raw.githubusercontent.com/landtuna/open-mtools/6c04c2882eca7c4313ec267766ea53ac9bfe129b/msend.c": "Informatica", + "rc.quest.com/topics/openssh/license.php#tcpwrappers": "TCP-wrappers", + "research.cs.wisc.edu/condor/license.html#condor": "Condor-1.1", + "research.scea.com/scea_shared_source_license.html": "SCEA", + "romanrm.net/mit-zero": "MIT-0", + "sam.zoy.org/wtfpl/COPYING": "WTFPL", + "scancode-licensedb.aboutcode.org/bsd-unchanged": "BSD-2-Clause-pos-unchanged", + "scripts.sil.org/cms/scripts/page.php?item_id=OFL10_web": "OFL-1.0-no-RFN", + "scripts.sil.org/cms/scripts/page.php?item_id=OFL_web": "OFL-1.1-no-RFN", + "slicer.org/LICENSE": "3D-Slicer-1.0", + "solderpad.org/licenses/SHL-0.5/": "SHL-0.5", + "solderpad.org/licenses/SHL-0.51/": "SHL-0.51", + "source.icu-project.org/repos/icu/icu/trunk/license.html": "ICU", + "sourceforge.net/p/buddy/gitcode/ci/master/tree/README": "Buddy", + "sourceforge.net/p/ctrio/git/ci/master/tree/README": "ISC-Veillard", + "sourceforge.net/p/mcj/xfig/ci/master/tree/src/Makefile.am": "Xfig", + "sourceforge.net/p/netpbm/code/HEAD/tree/super_stable/converter/other/pnmtorle.c": "URT-RLE", + "sourceforge.net/p/netpbm/code/HEAD/tree/super_stable/converter/other/rletopnm.c": "URT-RLE", + "sourceforge.net/p/netpbm/code/HEAD/tree/super_stable/converter/ppm/ppmtompeg/jrevdct.c#l1189": "MPEG-SSG", + "sourceforge.net/p/netpbm/code/HEAD/tree/super_stable/editor/pnmstitch.c#l2": "pnmstitch", + "sourceforge.net/p/netpbm/code/HEAD/tree/super_stable/netpbm.c#l8": "HPND-Pbmplus", + "sourceforge.net/p/sbcl/sbcl/ci/master/tree/src/code/loop.lisp": "LOOP", + "sourceforge.net/p/trustedqsl/tqsl/ci/master/tree/LICENSE.txt": "TrustedQSL", + "sourceforge.net/p/xmedcon/code/ci/master/tree/libs/dicom/README": "OFFIS", + "sourceforge.net/p/xmedcon/code/ci/master/tree/libs/ljpg/": "IJG-short", + "sourceforge.net/projects/sevenzip/files/LZMA%20SDK/": "LZMA-SDK-9.22", + "sources.debian.org/copyright/license/debianutils/4.11.2/": "SMAIL-GPL", + "sources.debian.org/src/openmpi/4.1.0-10/ompi/debuggers/msgq_interface.h/?hl=19#L19": "mpi-permissive", + "sources.gentoo.org/cgi-bin/viewvc.cgi/gentoo-x86/licenses/BitTorrent?r1=1.1&r2=1.1.1.1&diff_format=s": "BitTorrent-1.0", + "sourceware.org/bzip2/1.0.5/bzip2-manual-1.0.5.html": "bzip2-1.0.6", + "sourceware.org/cgit/binutils-gdb/tree/include/coff/sym.h#n11": "MIPS", + "sourceware.org/cgit/binutils-gdb/tree/include/coff/symconst.h#n11": "ThirdEye", + "sourceware.org/cgit/valgrind/tree/mpi/libmpiwrap.c": "bzip2-1.0.6", "sourceware.org/git/?p=binutils-gdb.git;a=blob;f=libiberty/strcasecmp.c;h=131d81c2ce7881fa48c363dc5bf5fb302c61ce0b;hb=HEAD": "BSD-4.3RENO", "sourceware.org/git/?p=binutils-gdb.git;a=blob;f=readline/readline/support/wcwidth.c;h=0f5ec995796f4813abbcf4972aec0378ab74722a;hb=HEAD#l55": "HPND-Markus-Kuhn", "sourceware.org/git/?p=bzip2.git;a=blob;f=LICENSE;hb=bzip2-1.0.6": "bzip2-1.0.6", @@ -2129,6 +2181,7 @@ var urlToLicense = map[string]string{ "tahoe-lafs.org/trac/tahoe-lafs/browser/trunk/COPYING.TGPPL.rst": "TGPPL-1.0", "ti.arc.nasa.gov/opensource/nosa/": "NASA-1.3", "trac.enlightenment.org/e/browser/trunk/imlib2/COPYING": "Imlib2", + "tunes.org/legalese/bugroff.html": "Bugroff", "ubuntu.com/legal/font-licence": "Ubuntu-font-1.0", "unlicense.org/": "Unlicense", "vimdoc.sourceforge.net/htmldoc/uganda.html": "Vim", @@ -2139,6 +2192,7 @@ var urlToLicense = map[string]string{ "web.archive.org/web/20041020171434/http://www.rosenlaw.com/osl2.0.html": "OSL-2.0", "web.archive.org/web/20050212003940/http://www.rosenlaw.com/osl21.htm": "OSL-2.1", "web.archive.org/web/20060319014854/http://info.borland.com/devsupport/interbase/opensource/IPL.html": "Interbase-1.0", + "web.archive.org/web/20061013013632/http://opensource.linux-mirror.org/licenses/afl-2.1.txt": "AFL-2.1", "web.archive.org/web/20080207013128/http://www.gnome.org/fonts/": "Bitstream-Vera", "web.archive.org/web/20100302225219/http://www.zimbra.com/license/zimbra-public-license-1-3.html": "Zimbra-1.3", "web.archive.org/web/20111123062036/http://research.cs.wisc.edu/condor/license.html#condor": "Condor-1.1", @@ -2243,11 +2297,11 @@ var urlToLicense = map[string]string{ "www.jython.org/license.html": "CNRI-Jython", "www.kermitproject.org/ck90.html#source": "HPND-export-US", "www.kernel.org/doc/man-pages/licenses.html": "Linux-man-pages-copyleft", + "www.latex-project.org/lppl/LPPL-1.3c.txt": "LPPL-1.3c", "www.latex-project.org/lppl/lppl-1-0.txt": "LPPL-1.0", "www.latex-project.org/lppl/lppl-1-1.txt": "LPPL-1.1", "www.latex-project.org/lppl/lppl-1-2.txt": "LPPL-1.2", "www.latex-project.org/lppl/lppl-1-3a.txt": "LPPL-1.3a", - "www.latex-project.org/lppl/lppl-1-3c.txt": "LPPL-1.3c", "www.lemis.com/grog/UNIX/ancient-source-all.pdf": "Caldera", "www.libpng.org/pub/png/src/libpng-LICENSE.txt": "libpng-2.0", "www.mail-archive.com/debian-bugs-rc@lists.debian.org/msg11494.html": "Symlinks", @@ -2269,16 +2323,17 @@ var urlToLicense = map[string]string{ "www.netlib.org/lapack/LICENSE.txt": "BSD-3-Clause-Open-MPI", "www.netlib.org/minpack/disclaimer": "Minpack", "www.networkcultures.org/_uploads/%233notebook_telekommunist.pdf": "PPL", - "www.oclc.org/research/activities/software/license/v2final.htm": "OCLC-2.0", - "www.ogc.org/ogc/software/1.0": "OGC-1.0", - "www.ohwr.org/project/cernohl/wikis/Documents/CERN-OHL-version-2": "CERN-OHL-W-2.0", - "www.ohwr.org/project/licenses/wikis/cern-ohl-v1.1": "CERN-OHL-1.1", - "www.ohwr.org/project/licenses/wikis/cern-ohl-v1.2": "CERN-OHL-1.2", - "www.open-mpi.org/community/license.php": "BSD-3-Clause-Open-MPI", - "www.opencascade.com/content/occt-public-license": "OCCT-PL", - "www.opendatacommons.org/licenses/odbl/1.0/": "ODbL-1.0", - "www.opengroup.org/testing/downloads/The_Open_Group_TSL.txt": "OGTSL", - "www.openhub.net/licenses/mslpl": "MS-LPL", + "www.nist.gov/open/copyright-fair-use-and-licensing-statements-srd-data-software-and-technical-series-publications": "NIST-Software", + "www.oclc.org/research/activities/software/license/v2final.htm": "OCLC-2.0", + "www.ogc.org/ogc/software/1.0": "OGC-1.0", + "www.ohwr.org/project/cernohl/wikis/Documents/CERN-OHL-version-2": "CERN-OHL-W-2.0", + "www.ohwr.org/project/licenses/wikis/cern-ohl-v1.1": "CERN-OHL-1.1", + "www.ohwr.org/project/licenses/wikis/cern-ohl-v1.2": "CERN-OHL-1.2", + "www.open-mpi.org/community/license.php": "BSD-3-Clause-Open-MPI", + "www.opencascade.com/content/occt-public-license": "OCCT-PL", + "www.opendatacommons.org/licenses/odbl/1.0/": "ODbL-1.0", + "www.opengroup.org/testing/downloads/The_Open_Group_TSL.txt": "OGTSL", + "www.openhub.net/licenses/mslpl": "MS-LPL", "www.openldap.org/devel/gitweb.cgi?p=openldap.git;a=blob;f=LICENSE;hb=1cae062821881f41b73012ba816434897abf4205": "OLDAP-2.6", "www.openldap.org/devel/gitweb.cgi?p=openldap.git;a=blob;f=LICENSE;hb=37b4b3f6cc4bf34e1d3dec61e69914b9819d8894": "NBPL-1.0", "www.openldap.org/devel/gitweb.cgi?p=openldap.git;a=blob;f=LICENSE;hb=42b0383c50c299977b5893ee695cf4e486fb0dc7": "OLDAP-1.2", @@ -2301,7 +2356,7 @@ var urlToLicense = map[string]string{ "www.opensource.apple.com/license/apsl/": "APSL-2.0", "www.opensource.apple.com/source/IOSerialFamily/IOSerialFamily-7/APPLE_LICENSE": "APSL-1.1", "www.opensource.apple.com/source/tcl/tcl-5/tcl/generic/regfronts.c": "Spencer-99", - "www.opensource.org/licenses/EPL-2.0": "EPL-2.0", + "www.opensource.org/license/EPL-2.0": "EPL-2.0", "www.openssl.org/source/license.html": "OpenSSL", "www.osetfoundation.org/public-license": "OSET-PL-2.1", "www.parliament.uk/site-information/copyright-parliament/open-parliament-licence/": "OPL-UK-3.0", diff --git a/internal/spdxlicense/license_url_test.go b/internal/spdxlicense/license_url_test.go index 2ae646308..490976de3 100644 --- a/internal/spdxlicense/license_url_test.go +++ b/internal/spdxlicense/license_url_test.go @@ -123,6 +123,96 @@ func TestLicenseByURL_AlternateScheme(t *testing.T) { } } +func TestLicenseByURL_OpenSourceOrgVariants(t *testing.T) { + // SPDX has changed the form of opensource.org URLs across list versions (e.g. 3.29.0 moved + // opensource.org/licenses/MIT to opensource.org/license/MIT); every form should keep resolving. + tests := []struct { + name string + url string + wantID string + }{ + {name: "old plural path", url: "http://opensource.org/licenses/MIT", wantID: "MIT"}, + {name: "new singular path", url: "https://opensource.org/license/MIT", wantID: "MIT"}, + {name: "lowercase with trailing slash", url: "https://opensource.org/license/mit/", wantID: "MIT"}, + {name: "plural path with trailing slash", url: "https://opensource.org/licenses/MIT/", wantID: "MIT"}, + {name: "www prefix", url: "https://www.opensource.org/licenses/MIT", wantID: "MIT"}, + {name: "uppercase host", url: "https://WWW.OpenSource.org/licenses/MIT", wantID: "MIT"}, + {name: "old plural path BSD-3-Clause", url: "https://opensource.org/licenses/BSD-3-Clause", wantID: "BSD-3-Clause"}, + {name: "new singular path BSD-3-Clause", url: "https://opensource.org/license/BSD-3-Clause", wantID: "BSD-3-Clause"}, + {name: "lowercase BSD-3-Clause", url: "https://opensource.org/licenses/bsd-3-clause", wantID: "BSD-3-Clause"}, + {name: "old plural path Apache-2.0", url: "http://opensource.org/licenses/Apache-2.0", wantID: "Apache-2.0"}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + info, found := LicenseByURL(tt.url) + if !found { + t.Fatalf("LicenseByURL(%q) found = false, want true", tt.url) + } + if info.ID != tt.wantID { + t.Errorf("LicenseByURL(%q) ID = %v, want %v", tt.url, info.ID, tt.wantID) + } + }) + } +} + +func TestLicenseByURL_NonOpenSourceOrgStaysExact(t *testing.T) { + // the relaxed matching is scoped to opensource.org; other hosts may have case-sensitive paths + if _, found := LicenseByURL("https://www.apache.org/licenses/license-2.0"); found { + t.Errorf("LicenseByURL() found = true for case-altered non-opensource.org URL, want false") + } + if _, found := LicenseByURL("https://opensource.org/licenses/not-a-real-license"); found { + t.Errorf("LicenseByURL() found = true for unknown opensource.org URL, want false") + } +} + +func TestNormalizeOpenSourceOrgURL(t *testing.T) { + tests := []struct { + url string + want string + wantOK bool + }{ + {url: "opensource.org/licenses/MIT", want: "opensource.org/license/mit", wantOK: true}, + {url: "opensource.org/license/mit/", want: "opensource.org/license/mit", wantOK: true}, + {url: "www.opensource.org/licenses/EPL-2.0", want: "opensource.org/license/epl-2.0", wantOK: true}, + {url: "lists.opensource.org/pipermail/license-discuss", wantOK: false}, + {url: "www.apache.org/licenses/LICENSE-2.0", wantOK: false}, + } + + for _, tt := range tests { + t.Run(tt.url, func(t *testing.T) { + got, ok := normalizeOpenSourceOrgURL(tt.url) + if ok != tt.wantOK { + t.Fatalf("normalizeOpenSourceOrgURL(%q) ok = %v, want %v", tt.url, ok, tt.wantOK) + } + if got != tt.want { + t.Errorf("normalizeOpenSourceOrgURL(%q) = %q, want %q", tt.url, got, tt.want) + } + }) + } +} + +func TestBuildOpenSourceOrgURLToLicense(t *testing.T) { + index := buildOpenSourceOrgURLToLicense(map[string]string{ + "opensource.org/licenses/MIT": "MIT", + "opensource.org/license/mit/": "MIT", + "opensource.org/licenses/Foo": "Foo-1.0", + "opensource.org/license/foo": "Foo-2.0", + "opensource.org/license/FOO/": "Foo-1.0", + "www.apache.org/licenses/Bar": "Bar", + }) + + if got := index["opensource.org/license/mit"]; got != "MIT" { + t.Errorf("index[mit] = %q, want MIT", got) + } + if got, exists := index["opensource.org/license/foo"]; exists { + t.Errorf("index[foo] = %q, want no entry for URL forms mapping to different licenses", got) + } + if len(index) != 1 { + t.Errorf("len(index) = %d, want 1 (non-opensource.org URLs excluded): %v", len(index), index) + } +} + func TestStripScheme(t *testing.T) { tests := []struct { name string diff --git a/internal/task/archive_tasks.go b/internal/task/archive_tasks.go index c81a50e40..4f023546c 100644 --- a/internal/task/archive_tasks.go +++ b/internal/task/archive_tasks.go @@ -302,7 +302,7 @@ func (c *archiveCataloger) runSubPipeline(ctx context.Context, resolver file.Res scratch := &sbom.SBOM{Artifacts: sbom.Artifacts{Packages: pkg.NewCollection()}} scratchBuilder := sbomsync.NewBuilder(scratch) for _, t := range c.subPipeline { - err := runTaskSafely(ctx, t, resolver, scratchBuilder) + err := runSubTaskSafely(ctx, t, resolver, scratchBuilder) if err == nil { continue } @@ -317,6 +317,18 @@ func (c *archiveCataloger) runSubPipeline(ctx context.Context, resolver file.Res return scratch } +// runSubTaskSafely is runTaskSafely without the task name in a panic's error, since appendUnknowns +// already prefixes it. +func runSubTaskSafely(ctx context.Context, t Task, resolver file.Resolver, builder sbomsync.Builder) (err error) { + defer func() { + if r := recover(); r != nil { + log.WithFields("task", t.Name()).Debugf("archive sub-cataloger task panic stack:\n%s", debug.Stack()) + err = fmt.Errorf("panic: %v", r) + } + }() + return t.Execute(ctx, resolver, builder) +} + // archiveCandidate is one file the walk will try to open. type archiveCandidate struct { location file.Location diff --git a/internal/task/archive_tasks_test.go b/internal/task/archive_tasks_test.go index 71e376dbd..52772daad 100644 --- a/internal/task/archive_tasks_test.go +++ b/internal/task/archive_tasks_test.go @@ -577,7 +577,7 @@ func Test_archiveCataloger_subPipelineFailureIsRecordedAsAnUnknown(t *testing.T) reasons, ok := s.Artifacts.Unknowns[file.Coordinates{RealPath: "/outer.zip"}] require.True(t, ok) require.Len(t, reasons, 1) - assert.Contains(t, reasons[0], "panicking-cataloger: nope") + assert.Equal(t, "panicking-cataloger: panic: nope", reasons[0]) }) } diff --git a/internal/task/executor.go b/internal/task/executor.go index 6cee1242f..610f0a963 100644 --- a/internal/task/executor.go +++ b/internal/task/executor.go @@ -51,7 +51,8 @@ func runTaskSafely(ctx context.Context, t Task, resolver file.Resolver, s sbomsy // handle individual cataloger panics defer func() { if e := recover(); e != nil { - err = fmt.Errorf("%v at:\n%s", e, string(debug.Stack())) + log.WithFields("task", t.Name()).Debugf("task panic stack:\n%s", debug.Stack()) + err = fmt.Errorf("panic in task %q: %v", t.Name(), e) } }() diff --git a/internal/task/executor_test.go b/internal/task/executor_test.go index 527066cd0..af165b4be 100644 --- a/internal/task/executor_test.go +++ b/internal/task/executor_test.go @@ -8,8 +8,12 @@ import ( "github.com/wagoodman/go-progress" "github.com/anchore/syft/internal/sbomsync" + "github.com/anchore/syft/syft/artifact" "github.com/anchore/syft/syft/event/monitor" "github.com/anchore/syft/syft/file" + "github.com/anchore/syft/syft/pkg" + "github.com/anchore/syft/syft/pkg/cataloger/generic" + "github.com/anchore/syft/syft/sbom" ) func Test_TaskExecutor_PanicHandling(t *testing.T) { @@ -21,6 +25,20 @@ func Test_TaskExecutor_PanicHandling(t *testing.T) { Manual: progress.NewManual(-1), }) - require.Error(t, err) - require.Contains(t, err.Error(), "executor_test.go") + require.EqualError(t, err, `panic in task "panicking-cataloger": something bad happened`) +} + +func Test_RunTask_GenericParserPanicBecomesUnknown(t *testing.T) { + c := generic.NewCataloger("panicking-cataloger"). + WithParserByPath(func(context.Context, file.Resolver, *generic.Environment, file.LocationReadCloser) ([]pkg.Package, []artifact.Relationship, error) { + panic("boom") + }, "executor_test.go") + s := &sbom.SBOM{} + + err := RunTask(context.Background(), NewPackageTask(DefaultCatalogingFactoryConfig(), c), file.NewMockResolverForPaths("executor_test.go"), sbomsync.NewBuilder(s), &monitor.TaskProgress{ + Manual: progress.NewManual(-1), + }) + + require.NoError(t, err) + require.Len(t, s.Artifacts.Unknowns, 1) } diff --git a/internal/unknown/coordinate_error.go b/internal/unknown/coordinate_error.go index caa0a160e..2db6caf7e 100644 --- a/internal/unknown/coordinate_error.go +++ b/internal/unknown/coordinate_error.go @@ -183,6 +183,10 @@ func visitErrors(err error, fn func(error) error) error { // unwrap singly wrapped errors if e, ok := err.(interface{ Unwrap() error }); ok { wrapped := e.Unwrap() + if wrapped == nil { + // nothing is wrapped (e.g. a sync.PanicError with a non-error value), so this is a leaf + return fn(err) + } got := visitErrors(wrapped, fn) if got == nil { return nil diff --git a/internal/unknown/coordinate_error_test.go b/internal/unknown/coordinate_error_test.go index 27c53a5ff..e863e6359 100644 --- a/internal/unknown/coordinate_error_test.go +++ b/internal/unknown/coordinate_error_test.go @@ -8,6 +8,7 @@ import ( "github.com/stretchr/testify/require" + "github.com/anchore/go-sync" "github.com/anchore/syft/syft/file" ) @@ -234,3 +235,11 @@ func Test_Append(t *testing.T) { e3 = Appendf(e2, file.NewLocation("l3"), "%s", "e3") require.Equal(t, "l1: e1\nl3: e3", e3.Error()) } + +func Test_ExtractCoordinateErrors_keepsNilUnwrap(t *testing.T) { + // sync.PanicError unwraps to nil when the panic value is not an error + err := sync.PanicError{Value: "x"} + unknowns, remaining := ExtractCoordinateErrors(err) + require.Empty(t, unknowns) + require.Equal(t, err, remaining) +} diff --git a/syft/file/cataloger/executable/cataloger.go b/syft/file/cataloger/executable/cataloger.go index 0fc71f862..cb73dc8f4 100644 --- a/syft/file/cataloger/executable/cataloger.go +++ b/syft/file/cataloger/executable/cataloger.go @@ -7,6 +7,7 @@ import ( "debug/macho" "encoding/binary" "fmt" + "runtime/debug" "sort" "github.com/bmatcuk/doublestar/v4" @@ -89,7 +90,7 @@ func (i *Cataloger) CatalogCtx(ctx context.Context, resolver file.Resolver) (map return results, errs } -func processExecutableLocation(loc file.Location, resolver file.Resolver) (*file.Executable, error) { +func processExecutableLocation(loc file.Location, resolver file.Resolver) (exec *file.Executable, err error) { reader, err := resolver.FileContentsByLocation(loc) if err != nil { log.WithFields("error", err, "path", loc.RealPath).Debug("unable to get file contents") @@ -97,6 +98,15 @@ func processExecutableLocation(loc file.Location, resolver file.Resolver) (*file } defer internal.CloseAndLogError(reader, loc.RealPath) + defer func() { + if r := recover(); r != nil { + // a malformed binary must not take down the whole SBOM; the caller records this as an unknown + log.WithFields("path", loc.RealPath, "panic", r).Debug("recovered from panic while reading executable") + log.Tracef("executable panic stack:\n%s", debug.Stack()) + exec, err = nil, fmt.Errorf("recovered from panic while reading executable: %v", r) + } + }() + uReader, err := unionreader.GetUnionReader(reader) if err != nil { log.WithFields("error", err, "path", loc.RealPath).Debug("unable to get union reader") diff --git a/syft/file/cataloger/executable/cataloger_test.go b/syft/file/cataloger/executable/cataloger_test.go new file mode 100644 index 000000000..7b670170c --- /dev/null +++ b/syft/file/cataloger/executable/cataloger_test.go @@ -0,0 +1,40 @@ +package executable + +import ( + "context" + "io" + "testing" + + "github.com/stretchr/testify/require" + + "github.com/anchore/syft/internal/unknown" + "github.com/anchore/syft/syft/file" +) + +type panicReader struct{} + +func (panicReader) Read([]byte) (int, error) { panic("boom") } + +type panickingContentsResolver struct { + *file.MockResolver +} + +func (r panickingContentsResolver) FilesByMIMEType(...string) ([]file.Location, error) { + return r.FilesByPath("cataloger_test.go") +} + +func (r panickingContentsResolver) FileContentsByLocation(file.Location) (io.ReadCloser, error) { + return io.NopCloser(panicReader{}), nil +} + +func TestCatalogCtx_panicBecomesUnknown(t *testing.T) { + resolver := panickingContentsResolver{file.NewMockResolverForPaths("cataloger_test.go")} + + results, err := NewCataloger(DefaultConfig()).CatalogCtx(context.Background(), resolver) + + require.Empty(t, results) + unknowns, remaining := unknown.ExtractCoordinateErrors(err) + require.NoError(t, remaining) + require.Len(t, unknowns, 1) + require.ErrorContains(t, unknowns[0].Reason, "recovered from panic while reading executable: boom") +} diff --git a/syft/file/cataloger/executable/elf.go b/syft/file/cataloger/executable/elf.go index e7f6d588d..b58b3eaa2 100644 --- a/syft/file/cataloger/executable/elf.go +++ b/syft/file/cataloger/executable/elf.go @@ -156,32 +156,12 @@ func hasElfDynFlag1(f *elf.File, flag elf.DynFlag1) bool { } func hasElfDynTag(f *elf.File, tag elf.DynTag) bool { - // source https://github.com/golang/go/blob/9b4b3e5acca2dabe107fa2c3ed963097d78a4562/src/cmd/cgo/internal/testshared/shared_test.go#L280 - - ds := f.SectionByType(elf.SHT_DYNAMIC) - if ds == nil { - return false - } - d, err := ds.Data() + vals, err := f.DynValue(tag) if err != nil { + log.WithFields("error", err).Trace("unable to read dynamic section from elf file") return false } - - for len(d) > 0 { - var t elf.DynTag - switch f.Class { - case elf.ELFCLASS32: - t = elf.DynTag(f.ByteOrder.Uint32(d[0:4])) - d = d[8:] - case elf.ELFCLASS64: - t = elf.DynTag(f.ByteOrder.Uint64(d[0:8])) - d = d[16:] - } - if t == tag { - return true - } - } - return false + return len(vals) > 0 } func isELFPIE(f *elf.File) bool { diff --git a/syft/file/cataloger/executable/elf_test.go b/syft/file/cataloger/executable/elf_test.go index a380a5cd6..23b19eced 100644 --- a/syft/file/cataloger/executable/elf_test.go +++ b/syft/file/cataloger/executable/elf_test.go @@ -1,7 +1,9 @@ package executable import ( + "bytes" "debug/elf" + "encoding/binary" "os" "path/filepath" "testing" @@ -226,3 +228,89 @@ func Test_elfHasExports(t *testing.T) { }) } } + +func Test_hasElfDynTag(t *testing.T) { + dynEntry := func(class elf.Class, tag elf.DynTag) []byte { + if class == elf.ELFCLASS32 { + return binary.LittleEndian.AppendUint32(binary.LittleEndian.AppendUint32(nil, uint32(tag)), 0) + } + return binary.LittleEndian.AppendUint64(binary.LittleEndian.AppendUint64(nil, uint64(tag)), 0) + } + + tests := []struct { + name string + class elf.Class + dynamic []byte + want bool + }{ + { + name: "32-bit tag present", + class: elf.ELFCLASS32, + dynamic: dynEntry(elf.ELFCLASS32, elf.DT_BIND_NOW), + want: true, + }, + { + name: "64-bit tag present", + class: elf.ELFCLASS64, + dynamic: dynEntry(elf.ELFCLASS64, elf.DT_BIND_NOW), + want: true, + }, + { + name: "64-bit tag absent", + class: elf.ELFCLASS64, + dynamic: dynEntry(elf.ELFCLASS64, elf.DT_NULL), + want: false, + }, + { + name: "32-bit truncated dynamic section", + class: elf.ELFCLASS32, + dynamic: dynEntry(elf.ELFCLASS32, elf.DT_BIND_NOW)[:4], + want: false, + }, + { + name: "64-bit truncated dynamic section", + class: elf.ELFCLASS64, + dynamic: dynEntry(elf.ELFCLASS64, elf.DT_BIND_NOW)[:12], + want: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + f, err := elf.NewFile(bytes.NewReader(elfWithDynamicSection(t, tt.class, tt.dynamic))) + require.NoError(t, err) + assert.Equal(t, tt.want, hasElfDynTag(f, elf.DT_BIND_NOW)) + }) + } +} + +// elfWithDynamicSection builds a little-endian ELF holding only a null section and a SHT_DYNAMIC section. +func elfWithDynamicSection(t *testing.T, class elf.Class, dynamic []byte) []byte { + t.Helper() + ident := [elf.EI_NIDENT]byte{0x7f, 'E', 'L', 'F', byte(class), byte(elf.ELFDATA2LSB), byte(elf.EV_CURRENT)} + var buf bytes.Buffer + write := func(v any) { + require.NoError(t, binary.Write(&buf, binary.LittleEndian, v)) + } + + if class == elf.ELFCLASS32 { + ehsize, shentsize := binary.Size(elf.Header32{}), binary.Size(elf.Section32{}) + dataOff := ehsize + 2*shentsize + write(elf.Header32{ + Ident: ident, Type: uint16(elf.ET_DYN), Machine: uint16(elf.EM_386), Version: uint32(elf.EV_CURRENT), + Shoff: uint32(ehsize), Ehsize: uint16(ehsize), Shentsize: uint16(shentsize), Shnum: 2, + }) + write(elf.Section32{}) + write(elf.Section32{Type: uint32(elf.SHT_DYNAMIC), Off: uint32(dataOff), Size: uint32(len(dynamic)), Addralign: 1}) + } else { + ehsize, shentsize := binary.Size(elf.Header64{}), binary.Size(elf.Section64{}) + dataOff := ehsize + 2*shentsize + write(elf.Header64{ + Ident: ident, Type: uint16(elf.ET_DYN), Machine: uint16(elf.EM_X86_64), Version: uint32(elf.EV_CURRENT), + Shoff: uint64(ehsize), Ehsize: uint16(ehsize), Shentsize: uint16(shentsize), Shnum: 2, + }) + write(elf.Section64{}) + write(elf.Section64{Type: uint32(elf.SHT_DYNAMIC), Off: uint64(dataOff), Size: uint64(len(dynamic)), Addralign: 1}) + } + buf.Write(dynamic) + return buf.Bytes() +} diff --git a/syft/file/cataloger/filedigest/cataloger.go b/syft/file/cataloger/filedigest/cataloger.go index bc0ec534f..447405648 100644 --- a/syft/file/cataloger/filedigest/cataloger.go +++ b/syft/file/cataloger/filedigest/cataloger.go @@ -5,6 +5,7 @@ import ( "crypto" "errors" "fmt" + "runtime/debug" "github.com/dustin/go-humanize" @@ -87,7 +88,7 @@ func (i *Cataloger) Catalog(ctx context.Context, resolver file.Resolver, coordin return results, err } -func (i *Cataloger) catalogLocation(ctx context.Context, resolver file.Resolver, location file.Location) ([]file.Digest, error) { +func (i *Cataloger) catalogLocation(ctx context.Context, resolver file.Resolver, location file.Location) (digests []file.Digest, err error) { meta, err := resolver.FileMetadataByLocation(location) if err != nil { return nil, err @@ -104,7 +105,16 @@ func (i *Cataloger) catalogLocation(ctx context.Context, resolver file.Resolver, } defer internal.CloseAndLogError(contentReader, location.AccessPath) - digests, err := intFile.NewDigestsFromFile(ctx, contentReader, i.hashes) + defer func() { + if r := recover(); r != nil { + // the caller records this as an unknown for the location instead of failing the whole SBOM + log.WithFields("path", location.RealPath, "panic", r).Debug("recovered from panic while digesting file") + log.Tracef("file digest panic stack:\n%s", debug.Stack()) + digests, err = nil, fmt.Errorf("recovered from panic while digesting file: %v", r) + } + }() + + digests, err = intFile.NewDigestsFromFile(ctx, contentReader, i.hashes) if err != nil { return nil, internal.ErrPath{Context: "digests-cataloger", Path: location.RealPath, Err: err} } diff --git a/syft/file/cataloger/filedigest/cataloger_test.go b/syft/file/cataloger/filedigest/cataloger_test.go index 00e72a7d1..6188bd066 100644 --- a/syft/file/cataloger/filedigest/cataloger_test.go +++ b/syft/file/cataloger/filedigest/cataloger_test.go @@ -15,6 +15,7 @@ import ( stereoscopeFile "github.com/anchore/stereoscope/pkg/file" "github.com/anchore/stereoscope/pkg/imagetest" intFile "github.com/anchore/syft/internal/file" + "github.com/anchore/syft/internal/unknown" "github.com/anchore/syft/syft/file" "github.com/anchore/syft/syft/source" "github.com/anchore/syft/syft/source/directorysource" @@ -202,3 +203,27 @@ func TestFileDigestCataloger_GivenCoordinates(t *testing.T) { } } + +type panicReader struct{} + +func (panicReader) Read([]byte) (int, error) { panic("boom") } + +type panickingContentsResolver struct { + *file.MockResolver +} + +func (r panickingContentsResolver) FileContentsByLocation(file.Location) (io.ReadCloser, error) { + return io.NopCloser(panicReader{}), nil +} + +func TestDigestsCataloger_panicBecomesUnknown(t *testing.T) { + resolver := panickingContentsResolver{file.NewMockResolverForPaths("cataloger_test.go")} + + results, err := NewCataloger([]crypto.Hash{crypto.SHA256}).Catalog(context.Background(), resolver, file.NewLocation("cataloger_test.go").Coordinates) + + require.Empty(t, results) + unknowns, remaining := unknown.ExtractCoordinateErrors(err) + require.NoError(t, remaining) + require.Len(t, unknowns, 1) + require.ErrorContains(t, unknowns[0].Reason, "recovered from panic while digesting file: boom") +} diff --git a/syft/format/common/spdxhelpers/to_format_model.go b/syft/format/common/spdxhelpers/to_format_model.go index 3fd6019e0..a86193aef 100644 --- a/syft/format/common/spdxhelpers/to_format_model.go +++ b/syft/format/common/spdxhelpers/to_format_model.go @@ -2,6 +2,7 @@ package spdxhelpers import ( + "cmp" "crypto/sha1" "fmt" "path" @@ -180,7 +181,8 @@ func toRootRelationships(rootPackage *spdx.Package, packages []*spdx.Package) (o func toRootPackage(s source.Description) *spdx.Package { var prefix string - name := s.Name + // the SPDX package name is mandatory, and a source decoded from another SBOM may not carry one + name := cmp.Or(s.Name, imageRepoName(s.Metadata), s.ID, helpers.DocumentName(s)) version := s.Version var purl *packageurl.PackageURL @@ -211,7 +213,7 @@ func toRootPackage(s source.Description) *spdx.Package { checksums = append(checksums, *c) purl = &packageurl.PackageURL{ Type: "oci", - Name: s.Name, + Name: name, Version: m.ManifestDigest, Qualifiers: qualifiers, } @@ -241,7 +243,7 @@ func toRootPackage(s source.Description) *spdx.Package { checksums = append(checksums, *c) purl = &packageurl.PackageURL{ Type: "oci", - Name: s.Name, + Name: name, Version: m.ManifestDigest, Qualifiers: qualifiers, } @@ -276,10 +278,6 @@ func toRootPackage(s source.Description) *spdx.Package { default: prefix = prefixUnknown purpose = spdxPrimaryPurposeOther - - if name == "" { - name = s.ID - } } p := &spdx.Package{ @@ -309,6 +307,25 @@ func toRootPackage(s source.Description) *spdx.Package { return p } +// imageRepoName returns the repository name from an image or model user input, the same way the image source names +// itself, so a source decoded from another SBOM (which may only carry the user input) gets a name without the tag. +func imageRepoName(metadata any) string { + var userInput string + switch m := metadata.(type) { + case source.ImageMetadata: + userInput = m.UserInput + case source.OCIModelMetadata: + userInput = m.UserInput + default: + return "" + } + ref, _ := reference.Parse(userInput) + if named, ok := ref.(reference.Named); ok { + return named.Name() + } + return "" +} + func toSPDXID(identifiable artifact.Identifiable) spdx.ElementID { id := string(identifiable.ID()) if after, ok := strings.CutPrefix(id, "SPDXRef-"); ok { @@ -582,10 +599,13 @@ func toPackageChecksums(p pkg.Package) ([]spdx.Checksum, bool) { }) case pkg.OpamPackage: for _, checksum := range meta.Checksums { - parts := strings.Split(checksum, "=") + algorithm, value, ok := strings.Cut(checksum, "=") + if !ok { + continue + } checksums = append(checksums, spdx.Checksum{ - Algorithm: spdx.ChecksumAlgorithm(strings.ToUpper(parts[0])), - Value: parts[1], + Algorithm: spdx.ChecksumAlgorithm(strings.ToUpper(algorithm)), + Value: value, }) } } diff --git a/syft/format/common/spdxhelpers/to_format_model_test.go b/syft/format/common/spdxhelpers/to_format_model_test.go index 758995d62..c756995a0 100644 --- a/syft/format/common/spdxhelpers/to_format_model_test.go +++ b/syft/format/common/spdxhelpers/to_format_model_test.go @@ -516,6 +516,20 @@ func Test_toPackageChecksums(t *testing.T) { }, filesAnalyzed: false, }, + { + name: "Opam Package with a checksum missing its algorithm separator", + pkg: pkg.Package{ + Name: "test", + Version: "1.0.0", + Metadata: pkg.OpamPackage{ + Checksums: []string{"f5f1c0b4", "md5=abc"}, + }, + }, + expected: []spdx.Checksum{ + {Algorithm: "MD5", Value: "abc"}, + }, + filesAnalyzed: false, + }, { name: "Package with no metadata type", pkg: pkg.Package{ @@ -1174,3 +1188,49 @@ func Test_toFiles_nestedFilesAreNamedByTheirChainAndRoundTrip(t *testing.T) { require.NotNil(t, b, "names: %v", byName) assert.Equal(t, inB, toSyftCoordinates(b), "the entry path's escaping is undone on decode") } + +func Test_toRootPackage_name(t *testing.T) { + tests := []struct { + name string + src source.Description + wantName string + wantPurl string + }{ + { + name: "source name wins", + src: source.Description{Name: "named", ID: "id", Metadata: source.ImageMetadata{UserInput: "alpine:3.18"}}, + wantName: "named", + }, + { + // a CycloneDX container component decodes to an image source with only the user input set + name: "image user input without a source name", + src: source.Description{Metadata: source.ImageMetadata{ + UserInput: "alpine:3.18", + ManifestDigest: "sha256:0e4b2e0d6b2f3c2b1d3bb3a1c8f1c4d9e6a9b1e8c6f4d2b3a1f0e9d8c7b6a5f4", + }}, + wantName: "alpine", + wantPurl: "pkg:oci/alpine@sha256%3A0e4b2e0d6b2f3c2b1d3bb3a1c8f1c4d9e6a9b1e8c6f4d2b3a1f0e9d8c7b6a5f4?arch=&tag=3.18", + }, + { + name: "id without a source name", + src: source.Description{ID: "some-id"}, + wantName: "some-id", + }, + { + name: "nothing to name it by", + src: source.Description{Metadata: source.DirectoryMetadata{}}, + wantName: "unknown", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + p := toRootPackage(tt.src) + assert.Equal(t, tt.wantName, p.PackageName) + var purl string + if len(p.PackageExternalReferences) > 0 { + purl = p.PackageExternalReferences[0].Locator + } + assert.Equal(t, tt.wantPurl, purl) + }) + } +} diff --git a/syft/format/common/spdxhelpers/to_syft_model.go b/syft/format/common/spdxhelpers/to_syft_model.go index 4941a7898..325aacd67 100644 --- a/syft/format/common/spdxhelpers/to_syft_model.go +++ b/syft/format/common/spdxhelpers/to_syft_model.go @@ -35,6 +35,8 @@ func ToSyftModel(doc *spdx.Document) (*sbom.SBOM, error) { return nil, errors.New("cannot convert SPDX document to Syft model because document is nil") } + removeNilElements(doc) + spdxIDMap := make(map[string]any) s := &sbom.SBOM{ @@ -56,6 +58,28 @@ func ToSyftModel(doc *spdx.Document) (*sbom.SBOM, error) { return s, nil } +// removeNilElements drops null entries from the document's lists. A JSON null (or an element the SPDX version +// converter cannot map) decodes to a nil pointer, and the rest of the conversion dereferences these elements. +func removeNilElements(doc *spdx.Document) { + doc.Packages = withoutNil(doc.Packages) + for _, p := range doc.Packages { + p.Files = withoutNil(p.Files) + p.PackageExternalReferences = withoutNil(p.PackageExternalReferences) + } + doc.Files = withoutNil(doc.Files) + doc.Relationships = withoutNil(doc.Relationships) +} + +func withoutNil[T any](items []*T) []*T { + var out []*T + for _, item := range items { + if item != nil { + out = append(out, item) + } + } + return out +} + func isDirectory(name string) bool { if name == "." || name == ".." || strings.HasSuffix(name, "/") || !strings.Contains(path.Base(name), ".") { return true @@ -124,12 +148,24 @@ func extractSource(spdxIDMap map[string]any, doc *spdx.Document) source.Descript } p := rootPackages[0] + rootType := syftRootType(p.PackageSPDXIdentifier) - switch p.PrimaryPackagePurpose { + purpose := p.PrimaryPackagePurpose + if purpose == "" { + // SPDX 2.1 and 2.2 have no primary package purpose, so fall back to the source type syft puts in the root's ID + purpose = syftRootPurpose(rootType) + } + + switch purpose { case spdxPrimaryPurposeContainer: src = containerSource(p) case spdxPrimaryPurposeFile: src = fileSource(p) + case spdxPrimaryPurposeOther: + // syft writes a root package even for a source it knows nothing about, which is not a package in the SBOM + if rootType != prefixUnknown { + return src + } default: return src } @@ -142,6 +178,30 @@ func extractSource(spdxIDMap map[string]any, doc *spdx.Document) source.Descript return src } +// syftRootType returns the source type syft encodes in a root package ID ("DocumentRoot--"), or "" for +// any other ID. +func syftRootType(id spdx.ElementID) string { + rest, ok := strings.CutPrefix(string(id), "DocumentRoot-") + if !ok { + return "" + } + rootType, _, _ := strings.Cut(rest, "-") + return rootType +} + +// syftRootPurpose returns the primary package purpose syft writes for a root package of the given source type. +func syftRootPurpose(rootType string) string { + switch rootType { + case prefixImage, prefixOCIModel, prefixSnap: + return spdxPrimaryPurposeContainer + case prefixDirectory, prefixFile: + return spdxPrimaryPurposeFile + case prefixUnknown: + return spdxPrimaryPurposeOther + } + return "" +} + func containerSource(p *spdx.Package) source.Description { id := string(p.PackageSPDXIdentifier) @@ -202,7 +262,7 @@ func fileSource(p *spdx.Package) source.Description { } supplier := "" - if p.PackageSupplier.Supplier != helpers.NOASSERTION { + if p.PackageSupplier != nil && p.PackageSupplier.Supplier != helpers.NOASSERTION { supplier = p.PackageSupplier.Supplier } diff --git a/syft/format/cpes/decoder.go b/syft/format/cpes/decoder.go index b5d3ab62e..9063f6fae 100644 --- a/syft/format/cpes/decoder.go +++ b/syft/format/cpes/decoder.go @@ -7,6 +7,7 @@ import ( "io" "strings" + syftinternal "github.com/anchore/syft/internal" "github.com/anchore/syft/internal/log" "github.com/anchore/syft/syft/cpe" "github.com/anchore/syft/syft/format/internal" @@ -61,7 +62,7 @@ func toSyftModel(r io.Reader) (*sbom.SBOM, error) { var errs []error pkgs := pkg.NewCollection() - scanner := bufio.NewScanner(r) + scanner := syftinternal.NewLineScanner(r) for scanner.Scan() { line := strings.TrimSpace(scanner.Text()) @@ -86,6 +87,9 @@ func toSyftModel(r io.Reader) (*sbom.SBOM, error) { p.SetID() pkgs.Add(p) } + if err := scanner.Err(); err != nil { + errs = append(errs, err) + } return &sbom.SBOM{ Artifacts: sbom.Artifacts{ diff --git a/syft/format/internal/spdxutil/helpers/document_name.go b/syft/format/internal/spdxutil/helpers/document_name.go index b34c7a445..afba5a5a7 100644 --- a/syft/format/internal/spdxutil/helpers/document_name.go +++ b/syft/format/internal/spdxutil/helpers/document_name.go @@ -9,16 +9,21 @@ func DocumentName(src source.Description) string { return src.Name } + var name string switch metadata := src.Metadata.(type) { case source.ImageMetadata: - return metadata.UserInput + name = metadata.UserInput case source.OCIModelMetadata: - return metadata.UserInput + name = metadata.UserInput case source.DirectoryMetadata: - return metadata.Path + name = metadata.Path case source.FileMetadata: - return metadata.Path - default: + name = metadata.Path + } + + // the SPDX document name is mandatory, and a source read from another SBOM may not have one + if name == "" { return "unknown" } + return name } diff --git a/syft/format/internal/spdxutil/helpers/document_name_test.go b/syft/format/internal/spdxutil/helpers/document_name_test.go index 26dbf1ebf..eab8fc18f 100644 --- a/syft/format/internal/spdxutil/helpers/document_name_test.go +++ b/syft/format/internal/spdxutil/helpers/document_name_test.go @@ -1,8 +1,6 @@ package helpers import ( - "fmt" - "strings" "testing" "github.com/stretchr/testify/assert" @@ -73,11 +71,23 @@ func Test_DocumentName(t *testing.T) { }, expected: "some/name", }, + { + name: "no name or metadata", + srcMetadata: source.Description{}, + expected: "unknown", + }, + { + name: "metadata without a name", + srcMetadata: source.Description{ + Metadata: source.ImageMetadata{}, + }, + expected: "unknown", + }, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { actual := DocumentName(test.srcMetadata) - assert.True(t, strings.HasPrefix(actual, test.expected), fmt.Sprintf("actual name %q", actual)) + assert.Equal(t, test.expected, actual) // track each scheme tested (passed or not) tracker.Tested(t, test.srcMetadata.Metadata) diff --git a/syft/format/purls/decoder.go b/syft/format/purls/decoder.go index 9cce2dcf5..89e37f739 100644 --- a/syft/format/purls/decoder.go +++ b/syft/format/purls/decoder.go @@ -8,6 +8,7 @@ import ( "strings" "github.com/anchore/packageurl-go" + syftinternal "github.com/anchore/syft/internal" "github.com/anchore/syft/internal/log" "github.com/anchore/syft/syft/format/internal" "github.com/anchore/syft/syft/pkg" @@ -61,7 +62,7 @@ func toSyftModel(r io.Reader) (*sbom.SBOM, error) { var errs []error pkgs := pkg.NewCollection() - scanner := bufio.NewScanner(r) + scanner := syftinternal.NewLineScanner(r) for scanner.Scan() { line := strings.TrimSpace(scanner.Text()) @@ -84,6 +85,9 @@ func toSyftModel(r io.Reader) (*sbom.SBOM, error) { p.SetID() pkgs.Add(p) } + if err := scanner.Err(); err != nil { + errs = append(errs, err) + } return &sbom.SBOM{ Artifacts: sbom.Artifacts{ diff --git a/syft/format/spdxjson/decoder_test.go b/syft/format/spdxjson/decoder_test.go index 9fb0d05dd..aca7fba6e 100644 --- a/syft/format/spdxjson/decoder_test.go +++ b/syft/format/spdxjson/decoder_test.go @@ -5,6 +5,7 @@ import ( "io" "os" "path/filepath" + "strings" "testing" "github.com/stretchr/testify/assert" @@ -183,3 +184,37 @@ func TestDecoder_Identify(t *testing.T) { }) } } + +func TestDecoder_DecodeNullElements(t *testing.T) { + // a JSON null in these lists (or an SPDX 2.1 package the version converter cannot map) decodes to a nil pointer. + // a null entry directly in "packages" is not covered: for SPDX 2.2 and 2.3 it panics inside spdx/tools-golang + for _, doc := range []string{ + `{"spdxVersion":"SPDX-2.1","packages":[{}]}`, + `{"spdxVersion":"SPDX-2.3","files":[null]}`, + `{"spdxVersion":"SPDX-2.3","packages":[{"SPDXID":"SPDXRef-a","name":"a","files":[null]}]}`, + `{"spdxVersion":"SPDX-2.3","packages":[{"SPDXID":"SPDXRef-a","name":"a","externalRefs":[null]}]}`, + } { + t.Run(doc, func(t *testing.T) { + assert.NotPanics(t, func() { + _, _, _, _ = NewFormatDecoder().Decode(strings.NewReader(doc)) + }) + }) + } +} + +func TestDecoder_DecodeFileRootWithoutSupplier(t *testing.T) { + // the package supplier is optional in SPDX, including on the package the document describes + f, err := os.Open("testdata/spdx/file-root-without-supplier.spdx.json") + require.NoError(t, err) + defer f.Close() + + s, _, _, err := NewFormatDecoder().Decode(f) + require.NoError(t, err) + assert.Equal(t, "app.tar.gz", s.Source.Name) + + var names []string + for _, p := range s.Artifacts.Packages.Sorted() { + names = append(names, p.Name) + } + assert.Equal(t, []string{"requests"}, names) +} diff --git a/syft/format/spdxjson/encoder_test.go b/syft/format/spdxjson/encoder_test.go index be4680942..42912d5d5 100644 --- a/syft/format/spdxjson/encoder_test.go +++ b/syft/format/spdxjson/encoder_test.go @@ -236,23 +236,6 @@ func TestSupportedVersions(t *testing.T) { subject := testutil.DirectoryInput(t, t.TempDir()) dec := NewFormatDecoder() - relationshipOffsetPerVersion := map[string]int{ - // the package representing the source gets a relationship from the source package to all other packages found - // these relationships cannot be removed until the primaryPackagePurpose info is available in 2.3 - "2.1": 2, - "2.2": 2, - // the source-to-package relationships can be removed since the primaryPackagePurpose info is available in 2.3 - "2.3": 0, - "3.0": 0, - } - - pkgCountOffsetPerVersion := map[string]int{ - "2.1": 1, // the source is mapped as a package, but cannot distinguish it since the primaryPackagePurpose info is not available until 2.3 - "2.2": 1, // the source is mapped as a package, but cannot distinguish it since the primaryPackagePurpose info is not available until 2.3 - "2.3": 0, // the source package can be removed since the primaryPackagePurpose info is available - "3.0": 0, - } - for _, enc := range encs { t.Run(enc.Version(), func(t *testing.T) { require.Contains(t, versions, enc.Version()) @@ -274,13 +257,9 @@ func TestSupportedVersions(t *testing.T) { require.NotEmpty(t, s.Artifacts.Packages.PackageCount()) - offset := relationshipOffsetPerVersion[enc.Version()] + assert.Equal(t, len(subject.Relationships), len(s.Relationships), "mismatched relationship count") - assert.Equal(t, len(subject.Relationships)+offset, len(s.Relationships), "mismatched relationship count") - - offset = pkgCountOffsetPerVersion[enc.Version()] - - if !assert.Equal(t, subject.Artifacts.Packages.PackageCount()+offset, s.Artifacts.Packages.PackageCount(), "mismatched package count") { + if !assert.Equal(t, subject.Artifacts.Packages.PackageCount(), s.Artifacts.Packages.PackageCount(), "mismatched package count") { t.Logf("expected: %d", subject.Artifacts.Packages.PackageCount()) for _, p := range subject.Artifacts.Packages.Sorted() { t.Logf(" - %s", p.String()) diff --git a/syft/format/spdxjson/testdata/snapshot/TestSPDX22JSONRequredProperties.golden b/syft/format/spdxjson/testdata/snapshot/TestSPDX22JSONRequredProperties.golden index 8587d7bc0..5f2b09b5b 100644 --- a/syft/format/spdxjson/testdata/snapshot/TestSPDX22JSONRequredProperties.golden +++ b/syft/format/spdxjson/testdata/snapshot/TestSPDX22JSONRequredProperties.golden @@ -47,13 +47,13 @@ "copyrightText": "NOASSERTION" }, { - "SPDXID": "SPDXRef-DocumentRoot-Unknown-", + "SPDXID": "SPDXRef-DocumentRoot-Unknown-unknown", "copyrightText": "NOASSERTION", "downloadLocation": "NOASSERTION", "filesAnalyzed": false, "licenseConcluded": "NOASSERTION", "licenseDeclared": "NOASSERTION", - "name": "", + "name": "unknown", "supplier": "NOASSERTION" } ], @@ -82,18 +82,18 @@ "relationshipType": "CONTAINS" }, { - "spdxElementId": "SPDXRef-DocumentRoot-Unknown-", + "spdxElementId": "SPDXRef-DocumentRoot-Unknown-unknown", "relatedSpdxElement": "SPDXRef-Package-files-analyzed-false-7d37ba9d2f7c574b", "relationshipType": "CONTAINS" }, { - "spdxElementId": "SPDXRef-DocumentRoot-Unknown-", + "spdxElementId": "SPDXRef-DocumentRoot-Unknown-unknown", "relatedSpdxElement": "SPDXRef-Package-files-analyzed-true-035066c2086b8bb4", "relationshipType": "CONTAINS" }, { "spdxElementId": "SPDXRef-DOCUMENT", - "relatedSpdxElement": "SPDXRef-DocumentRoot-Unknown-", + "relatedSpdxElement": "SPDXRef-DocumentRoot-Unknown-unknown", "relationshipType": "DESCRIBES" } ] diff --git a/syft/format/spdxjson/testdata/spdx/file-root-without-supplier.spdx.json b/syft/format/spdxjson/testdata/spdx/file-root-without-supplier.spdx.json new file mode 100644 index 000000000..25630796b --- /dev/null +++ b/syft/format/spdxjson/testdata/spdx/file-root-without-supplier.spdx.json @@ -0,0 +1,29 @@ +{ + "spdxVersion": "SPDX-2.3", + "dataLicense": "CC0-1.0", + "SPDXID": "SPDXRef-DOCUMENT", + "name": "app.tar.gz", + "documentNamespace": "https://example.com/spdx/app-1.0", + "creationInfo": { "created": "2026-09-24T00:00:00Z", "creators": ["Tool: example-1.0"] }, + "packages": [ + { + "SPDXID": "SPDXRef-app", + "name": "app.tar.gz", + "downloadLocation": "NOASSERTION", + "filesAnalyzed": false, + "primaryPackagePurpose": "FILE" + }, + { + "SPDXID": "SPDXRef-requests", + "name": "requests", + "versionInfo": "2.31.0", + "downloadLocation": "NOASSERTION", + "filesAnalyzed": false, + "externalRefs": [{ "referenceCategory": "PACKAGE-MANAGER", "referenceType": "purl", "referenceLocator": "pkg:pypi/requests@2.31.0" }] + } + ], + "relationships": [ + { "spdxElementId": "SPDXRef-DOCUMENT", "relationshipType": "DESCRIBES", "relatedSpdxElement": "SPDXRef-app" }, + { "spdxElementId": "SPDXRef-app", "relationshipType": "CONTAINS", "relatedSpdxElement": "SPDXRef-requests" } + ] +} diff --git a/syft/format/spdxtagvalue/encoder_test.go b/syft/format/spdxtagvalue/encoder_test.go index e5d470ec8..991d1ad1e 100644 --- a/syft/format/spdxtagvalue/encoder_test.go +++ b/syft/format/spdxtagvalue/encoder_test.go @@ -3,6 +3,7 @@ package spdxtagvalue import ( "bytes" "flag" + "fmt" "testing" "github.com/stretchr/testify/assert" @@ -143,21 +144,6 @@ func TestSupportedVersions(t *testing.T) { subject := testutil.DirectoryInput(t, t.TempDir()) dec := NewFormatDecoder() - relationshipOffsetPerVersion := map[string]int{ - // the package representing the source gets a relationship from the source package to all other packages found - // these relationships cannot be removed until the primaryPackagePurpose info is available in 2.3 - "2.1": 2, - "2.2": 2, - // the source-to-package relationships can be removed since the primaryPackagePurpose info is available in 2.3 - "2.3": 0, - } - - pkgCountOffsetPerVersion := map[string]int{ - "2.1": 1, // the source is mapped as a package, but cannot distinguish it since the primaryPackagePurpose info is not available until 2.3 - "2.2": 1, // the source is mapped as a package, but cannot distinguish it since the primaryPackagePurpose info is not available until 2.3 - "2.3": 0, // the source package can be removed since the primaryPackagePurpose info is available - } - for _, enc := range encs { t.Run(enc.Version(), func(t *testing.T) { require.Contains(t, versions, enc.Version()) @@ -178,13 +164,9 @@ func TestSupportedVersions(t *testing.T) { require.NotEmpty(t, s.Artifacts.Packages.PackageCount()) - offset := relationshipOffsetPerVersion[enc.Version()] + assert.Equal(t, len(subject.Relationships), len(s.Relationships), "mismatched relationship count") - assert.Equal(t, len(subject.Relationships)+offset, len(s.Relationships), "mismatched relationship count") - - offset = pkgCountOffsetPerVersion[enc.Version()] - - if !assert.Equal(t, subject.Artifacts.Packages.PackageCount()+offset, s.Artifacts.Packages.PackageCount(), "mismatched package count") { + if !assert.Equal(t, subject.Artifacts.Packages.PackageCount(), s.Artifacts.Packages.PackageCount(), "mismatched package count") { t.Logf("expected: %d", subject.Artifacts.Packages.PackageCount()) for _, p := range subject.Artifacts.Packages.Sorted() { t.Logf(" - %s", p.String()) @@ -209,3 +191,39 @@ func defaultFormatEncoders() []sbom.FormatEncoder { } return encs } + +func TestEncodeDecodeSourceWithoutName(t *testing.T) { + // a source read from another SBOM (a purl list, a CycloneDX document without metadata.component) may have no name, + // but the SPDX package name is mandatory and the tag-value decoder needs it to start the root package section + p := pkg.Package{Name: "left-pad", Version: "1.3.0", Type: pkg.NpmPkg} + p.SetID() + + for _, src := range []source.Description{ + {}, + {Metadata: source.FileMetadata{}}, + {Metadata: source.DirectoryMetadata{}}, + {Metadata: source.ImageMetadata{}}, + {Metadata: source.OCIModelMetadata{}}, + {Metadata: source.SnapMetadata{}}, + } { + for _, enc := range defaultFormatEncoders() { + t.Run(fmt.Sprintf("%T/%s", src.Metadata, enc.Version()), func(t *testing.T) { + var buf bytes.Buffer + require.NoError(t, enc.Encode(&buf, sbom.SBOM{ + Source: src, + Artifacts: sbom.Artifacts{Packages: pkg.NewCollection(p)}, + })) + + s, _, _, err := NewFormatDecoder().Decode(bytes.NewReader(buf.Bytes())) + require.NoError(t, err, "encoded document:\n%s", buf.String()) + + var names []string + for _, decoded := range s.Artifacts.Packages.Sorted() { + names = append(names, decoded.Name) + } + // the root package describes the source, so it must not come back as a package of the SBOM + assert.Equal(t, []string{"left-pad"}, names) + }) + } + } +} diff --git a/syft/format/untyped_metadata_test.go b/syft/format/untyped_metadata_test.go new file mode 100644 index 000000000..a1a28b9d0 --- /dev/null +++ b/syft/format/untyped_metadata_test.go @@ -0,0 +1,94 @@ +package format + +import ( + "bytes" + "fmt" + "math" + "testing" + + "github.com/anchore/syft/syft/format/syftjson" + "github.com/anchore/syft/syft/pkg" + "github.com/anchore/syft/syft/sbom" + "github.com/anchore/syft/syft/source" +) + +// TestEncodersHandleUntypedMetadataValues pushes awkward values through the untyped metadata fields +// allowlisted in internal/packagemetadata (see untypedFieldAllowlist) and asserts that SetID and every +// encoder neither panic nor fail. A new allowlist entry should get a case here too. +func TestEncodersHandleUntypedMetadataValues(t *testing.T) { + cases := []struct { + name string + value any + // encoding/json rejects non-finite floats, so syft-json returns an error for them. Producers + // have to replace these before they reach the model (the GGUF cataloger does); the case stays + // here so the expectation flips visibly if syft-json ever learns to encode them. + syftJSONRejects bool + }{ + {name: "NaN", value: math.NaN(), syftJSONRejects: true}, + {name: "+Inf float32", value: float32(math.Inf(1)), syftJSONRejects: true}, + {name: "-Inf", value: math.Inf(-1), syftJSONRejects: true}, + {name: "NaN in array", value: []any{1.0, math.NaN()}, syftJSONRejects: true}, + {name: "invalid utf-8", value: "\xff\xfe"}, + {name: "NUL and control characters", value: "a\x00b\x1b"}, + {name: "max uint64", value: uint64(math.MaxUint64)}, + {name: "deep array", value: nested(2000, func(v any) any { return []any{v} })}, + {name: "deep map", value: nested(2000, func(v any) any { return map[string]any{"k": v} })}, + {name: "large array", value: make([]any, 100_000)}, + } + + for _, tt := range cases { + t.Run(tt.name, func(t *testing.T) { + p := pkg.Package{ + Name: "model", + Version: "1.0", + Type: pkg.ModelPkg, + Metadata: pkg.GGUFFileHeader{ + RemainingKeyValues: map[string]any{"general.value": tt.value}, + }, + } + if !noPanic(t, "SetID", p.SetID) { + return + } + + s := sbom.SBOM{ + Artifacts: sbom.Artifacts{Packages: pkg.NewCollection(p)}, + Source: source.Description{Metadata: source.DirectoryMetadata{Path: "/models"}}, + } + for _, enc := range Encoders() { + id := fmt.Sprintf("%s@%s", enc.ID(), enc.Version()) + var buf bytes.Buffer + noPanic(t, id, func() { + err := enc.Encode(&buf, s) + switch { + case tt.syftJSONRejects && enc.ID() == syftjson.ID: + if err == nil { + t.Errorf("%s: expected an encode error for a non-finite float", id) + } + case err != nil: + t.Errorf("%s: encode failed: %v", id, err) + } + }) + } + }) + } +} + +func nested(depth int, wrap func(any) any) any { + var v any = "leaf" + for range depth { + v = wrap(v) + } + return v +} + +func noPanic(t *testing.T, what string, fn func()) (ok bool) { + t.Helper() + defer func() { + if r := recover(); r != nil { + t.Errorf("%s: panic: %v", what, r) + ok = false + } + }() + fn() + return true +} diff --git a/syft/internal/fileresolver/chroot_context.go b/syft/internal/fileresolver/chroot_context.go index 88647be3a..deb8cd000 100644 --- a/syft/internal/fileresolver/chroot_context.go +++ b/syft/internal/fileresolver/chroot_context.go @@ -216,5 +216,8 @@ func (r ChrootContext) ToChrootPath(nativePath string) string { responsePath = strings.TrimPrefix(responsePath, prefix) } - return responsePath + // chroot paths are posix regardless of host (these end up as location paths in the SBOM). This uses + // filepath (not path) since responsePath is native here (FromPosix above, then trimmed of a native prefix), + // and only filepath.ToSlash knows the host separator to convert. + return filepath.ToSlash(responsePath) } diff --git a/syft/internal/fileresolver/chroot_context_test.go b/syft/internal/fileresolver/chroot_context_test.go index 06f6e283a..fbe2f20e1 100644 --- a/syft/internal/fileresolver/chroot_context_test.go +++ b/syft/internal/fileresolver/chroot_context_test.go @@ -3,6 +3,7 @@ package fileresolver import ( "os" "path/filepath" + "runtime" "testing" "github.com/stretchr/testify/assert" @@ -10,6 +11,9 @@ import ( ) func Test_ChrootContext_RequestResponse(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("fixture relies on symlinks, which git does not check out as links on windows by default") + } // / // somewhere/ // outside.txt @@ -563,6 +567,9 @@ func TestNewChrootContext_BaseValidation(t *testing.T) { } func TestToNativeGlob(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("chroot context and expectations are posix paths") + } tests := []struct { name string chrootContext ChrootContext diff --git a/syft/internal/fileresolver/directory_indexer_test.go b/syft/internal/fileresolver/directory_indexer_test.go index d24a86201..39685ccce 100644 --- a/syft/internal/fileresolver/directory_indexer_test.go +++ b/syft/internal/fileresolver/directory_indexer_test.go @@ -4,7 +4,6 @@ import ( "fmt" "io/fs" "os" - "path" "path/filepath" "runtime" "sort" @@ -18,6 +17,7 @@ import ( "github.com/wagoodman/go-progress" "github.com/anchore/stereoscope/pkg/file" + "github.com/anchore/syft/syft/internal/windows" ) type indexerMock struct { @@ -138,6 +138,9 @@ func TestDirectoryIndexer_handleFileAccessErr(t *testing.T) { } func TestDirectoryIndexer_IncludeRootPathInIndex(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("indexes the posix filesystem root") + } filterFn := func(_, path string, _ os.FileInfo, _ error) error { if path != "/" { return fs.SkipDir @@ -206,11 +209,7 @@ func TestDirectoryIndexer_index(t *testing.T) { info, err := os.Stat(test.path) assert.NoError(t, err) - // note: the index uses absolute paths, so assertions MUST keep this in mind - cwd, err := os.Getwd() - require.NoError(t, err) - - p := file.Path(path.Join(cwd, test.path)) + p := indexedPath(t, test.path) assert.Equal(t, true, tree.HasPath(p)) exists, ref, err := tree.File(p) assert.Equal(t, true, exists) @@ -226,6 +225,9 @@ func TestDirectoryIndexer_index(t *testing.T) { } func TestDirectoryIndexer_index_for_AncestorSymlinks(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("fixture relies on symlinks, which git does not check out as links on windows by default") + } // note: this test is testing the effects from NewFromDirectory, indexTree, and addPathToIndex _, filename, _, ok := runtime.Caller(0) require.True(t, ok) @@ -256,11 +258,7 @@ func TestDirectoryIndexer_index_for_AncestorSymlinks(t *testing.T) { info, err := os.Stat(test.path) assert.NoError(t, err) - // note: the index uses absolute paths, so assertions MUST keep this in mind - cwd, err := os.Getwd() - require.NoError(t, err) - - p := file.Path(path.Join(cwd, test.path)) + p := indexedPath(t, test.path) assert.Equal(t, true, tree.HasPath(p)) exists, ref, err := tree.File(p) assert.Equal(t, true, exists) @@ -287,6 +285,9 @@ func TestDirectoryIndexer_index_survive_badSymlink(t *testing.T) { } func TestDirectoryIndexer_index_survive_inaccessibleSymlinkTarget(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("relies on symlinks and posix permissions") + } // a symlink that resolves into a directory we don't have permission to traverse should be skipped // with a warning like any other inaccessible path, not abort the entire scan (see #3286). if os.Geteuid() == 0 { @@ -323,6 +324,9 @@ func TestDirectoryIndexer_index_survive_inaccessibleSymlinkTarget(t *testing.T) } func TestDirectoryIndexer_SkipsAlreadyVisitedLinkDestinations(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("fixture relies on symlinks, which git does not check out as links on windows by default") + } var observedPaths []string pathObserver := func(_, p string, _ os.FileInfo, _ error) error { fields := strings.Split(p, "testdata/symlinks-prune-indexing") @@ -430,6 +434,11 @@ func TestDirectoryIndexer_IndexesAllTypes(t *testing.T) { } func Test_allContainedPaths(t *testing.T) { + if runtime.GOOS == "windows" { + // ponytail: production passes native windows paths here, which these posix cases don't represent. + // Ancestor indexing for a root reached through a symlink is likely broken on windows; fix separately. + t.Skip("cases are posix paths") + } tests := []struct { name string @@ -485,6 +494,9 @@ func Test_allContainedPaths(t *testing.T) { } func Test_relativePath(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("exercises a posix-only test helper") + } tests := []struct { name string basePath string @@ -548,3 +560,17 @@ func relativePath(basePath, givenPath string) string { return relPath } + +// indexedPath returns how the given cwd-relative path is keyed in the index (absolute and always posix) +func indexedPath(t *testing.T, rel string) file.Path { + t.Helper() + cwd, err := os.Getwd() + require.NoError(t, err) + // filepath (not path): cwd is a native host path, so it must be joined with the host separator before + // being converted to the posix form the index is keyed on + p := filepath.Join(cwd, rel) + if windows.HostRunningOnWindows() { + p = windows.ToPosix(p) + } + return file.Path(p) +} diff --git a/syft/internal/fileresolver/directory_roundtrip_test.go b/syft/internal/fileresolver/directory_roundtrip_test.go new file mode 100644 index 000000000..22f39cc18 --- /dev/null +++ b/syft/internal/fileresolver/directory_roundtrip_test.go @@ -0,0 +1,48 @@ +package fileresolver + +import ( + "context" + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + stereoscopeFile "github.com/anchore/stereoscope/pkg/file" +) + +// a location handed out by the resolver should be resolvable again by its own path. This is what the file +// digest cataloger relies on, and it silently found nothing on windows (see #5325). +func TestDirectory_LocationPathsRoundTrip(t *testing.T) { + // resolve symlinks up front (e.g. /var -> /private/var on macOS) so the cwd prefix matches the indexed paths + root, err := filepath.EvalSymlinks(t.TempDir()) + require.NoError(t, err) + require.NoError(t, os.MkdirAll(filepath.Join(root, "sub", "dir"), 0o755)) + for _, p := range []string{"top.txt", filepath.Join("sub", "mid.txt"), filepath.Join("sub", "dir", "deep.txt")} { + require.NoError(t, os.WriteFile(filepath.Join(root, p), []byte(p), 0o600)) + } + + // mirror `syft dir:.` from within the scanned directory + t.Chdir(root) + + resolver, err := NewFromDirectory(".", "") + require.NoError(t, err) + + var regular int + for loc := range resolver.AllLocations(context.Background()) { + md, err := resolver.FileMetadataByLocation(loc) + require.NoError(t, err) + if md.Type != stereoscopeFile.TypeRegular { + continue + } + regular++ + + assert.NotContains(t, loc.RealPath, `\`, "location paths should be posix") + + found, err := resolver.FilesByPath(loc.RealPath) + require.NoError(t, err) + assert.Len(t, found, 1, "unable to resolve %q by its own path", loc.RealPath) + } + assert.Equal(t, 3, regular) +} diff --git a/syft/internal/fileresolver/directory_windows_test.go b/syft/internal/fileresolver/directory_windows_test.go deleted file mode 100644 index 115fb30a3..000000000 --- a/syft/internal/fileresolver/directory_windows_test.go +++ /dev/null @@ -1,126 +0,0 @@ -package fileresolver - -import "testing" - -func Test_windowsToPosix(t *testing.T) { - type args struct { - windowsPath string - } - tests := []struct { - name string - args args - wantPosixPath string - }{ - { - name: "basic case", - args: args{ - windowsPath: `C:\some\windows\place`, - }, - wantPosixPath: "/c/some/windows/place", - }, - { - name: "escaped case", - args: args{ - windowsPath: `C:\\some\\windows\\place`, - }, - wantPosixPath: "/c/some/windows/place", - }, - { - name: "forward slash", - args: args{ - windowsPath: `C:/foo/bar`, - }, - wantPosixPath: "/c/foo/bar", - }, - { - name: "mix slash", - args: args{ - windowsPath: `C:\foo/bar\`, - }, - wantPosixPath: "/c/foo/bar", - }, - { - name: "case sensitive case", - args: args{ - windowsPath: `C:\Foo/bAr\`, - }, - wantPosixPath: "/c/Foo/bAr", - }, - { - name: "special char case", - args: args{ - windowsPath: `C:\ふー\バー`, - }, - wantPosixPath: "/c/ふー/バー", - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - if gotPosixPath := windowsToPosix(tt.args.windowsPath); gotPosixPath != tt.wantPosixPath { - t.Errorf("windowsToPosix() = %v, want %v", gotPosixPath, tt.wantPosixPath) - } - }) - } -} - -func Test_posixToWindows(t *testing.T) { - type args struct { - posixPath string - } - tests := []struct { - name string - args args - wantWindowsPath string - }{ - { - name: "basic case", - args: args{ - posixPath: "/c/some/windows/place", - }, - wantWindowsPath: `C:\some\windows\place`, - }, - { - name: "escaped case", - args: args{ - posixPath: "/c/some/windows/place", - }, - wantWindowsPath: `C:\\some\\windows\\place`, - }, - { - name: "forward slash", - args: args{ - posixPath: "/c/foo/bar", - }, - wantWindowsPath: `C:/foo/bar`, - }, - { - name: "mix slash", - args: args{ - posixPath: "/c/foo/bar", - }, - wantWindowsPath: `C:\foo/bar\`, - }, - { - name: "case sensitive case", - args: args{ - posixPath: "/c/Foo/bAr", - }, - wantWindowsPath: `C:\Foo/bAr\`, - }, - { - name: "special char case", - args: args{ - posixPath: "/c/ふー/バー", - }, - wantWindowsPath: `C:\ふー\バー`, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - if gotWindowsPath := posixToWindows(tt.args.posixPath); gotWindowsPath != tt.wantWindowsPath { - t.Errorf("posixToWindows() = %v, want %v", gotWindowsPath, tt.wantWindowsPath) - } - }) - } -} diff --git a/syft/internal/fileresolver/file_indexer.go b/syft/internal/fileresolver/file_indexer.go index 086dc75a9..2c47b2b48 100644 --- a/syft/internal/fileresolver/file_indexer.go +++ b/syft/internal/fileresolver/file_indexer.go @@ -29,11 +29,15 @@ func newFileIndexer(path, base string, visitors ...PathIndexVisitor) *fileIndexe base: base, tree: filetree.New(), index: filetree.NewIndex(), + // Unlike the directory indexer there is no skipPathsByMountTypeAndName here: it exists + // to keep a directory walk out of /proc, /sys and similar, and to do so it reads the + // whole mount table on construction. A file source indexes exactly one file and its + // parent, so there is nothing to walk into, and the mount table read is paid for + // every file source with nothing in return. pathIndexVisitors: append( []PathIndexVisitor{ requireFileInfo, disallowByFileType, - skipPathsByMountTypeAndName(path), }, visitors..., ), diff --git a/syft/internal/fileresolver/file_indexer_bench_test.go b/syft/internal/fileresolver/file_indexer_bench_test.go new file mode 100644 index 000000000..11befc4cf --- /dev/null +++ b/syft/internal/fileresolver/file_indexer_bench_test.go @@ -0,0 +1,51 @@ +package fileresolver + +import ( + "os" + "path/filepath" + "testing" + + "github.com/moby/sys/mountinfo" +) + +// BenchmarkNewFromFile is the cost of building a resolver for a single-file source, which is +// paid once per file by callers that catalog files individually. The default case is +// NewFromFile as shipped. The with-mount-skipper case attaches skipPathsByMountTypeAndName +// the way the file indexer used to, so the difference is what reading the mount table for +// every file source cost. That cost scales with the mount table, so its size is reported as +// the mounts metric: expect a bigger gap on a container host than on a laptop. +// +// go test ./syft/internal/fileresolver/ -run '^$' -bench NewFromFile -benchmem -count 6 | tee bench.txt +// benchstat -col /variant bench.txt +func BenchmarkNewFromFile(b *testing.B) { + path := filepath.Join(b.TempDir(), "file.txt") + if err := os.WriteFile(path, []byte("hello\n"), 0o644); err != nil { + b.Fatal(err) + } + mounts, err := mountinfo.GetMounts(nil) + if err != nil { + b.Fatal(err) + } + + b.Run("variant=default", func(b *testing.B) { + for b.Loop() { + if _, err := NewFromFile(path); err != nil { + b.Fatal(err) + } + } + b.ReportMetric(float64(len(mounts)), "mounts") + }) + + b.Run("variant=with-mount-skipper", func(b *testing.B) { + for b.Loop() { + r, err := newFromFileWithoutIndex(path, skipPathsByMountTypeAndName(path)) + if err != nil { + b.Fatal(err) + } + if err := r.buildIndex(); err != nil { + b.Fatal(err) + } + } + b.ReportMetric(float64(len(mounts)), "mounts") + }) +} diff --git a/syft/internal/fileresolver/file_indexer_test.go b/syft/internal/fileresolver/file_indexer_test.go index d9ba1d8c9..a4e0364d5 100644 --- a/syft/internal/fileresolver/file_indexer_test.go +++ b/syft/internal/fileresolver/file_indexer_test.go @@ -3,13 +3,11 @@ package fileresolver import ( "io/fs" "os" - "path" + "path/filepath" "testing" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - - "github.com/anchore/stereoscope/pkg/file" ) // - Verify that both the parent and the path are indexed @@ -38,11 +36,7 @@ func Test_index(t *testing.T) { info, err := os.Stat(test.path) assert.NoError(t, err) - // note: the index uses absolute paths, so assertions MUST keep this in mind - cwd, err := os.Getwd() - require.NoError(t, err) - - p := file.Path(path.Join(cwd, test.path)) + p := indexedPath(t, test.path) assert.Equal(t, true, tree.HasPath(p)) exists, ref, err := tree.File(p) assert.Equal(t, true, exists) @@ -71,7 +65,9 @@ func Test_ignoresPathIfFiltered(t *testing.T) { testPath := "testdata/system_paths/target/home/place" cwd, cwdErr := os.Getwd() require.NoError(t, cwdErr) - ignorePath := path.Join(cwd, testPath) + // filepath (not path): filters see the native absolute path (before posix normalization), so the + // expected value must be joined with the host separator to match on windows + ignorePath := filepath.Join(cwd, testPath) filterFn := func(_, path string, _ os.FileInfo, _ error) error { if path == ignorePath { return ErrSkipPath @@ -91,7 +87,8 @@ func Test_ignoresPathIfParentFiltered(t *testing.T) { cwd, cwdErr := os.Getwd() require.NoError(t, cwdErr) - ignorePath := path.Join(cwd, parentPath) + // filepath (not path): see Test_ignoresPathIfFiltered + ignorePath := filepath.Join(cwd, parentPath) filterFn := func(_, path string, _ os.FileInfo, _ error) error { if path == ignorePath { return fs.SkipDir diff --git a/syft/internal/fileresolver/filetree_resolver.go b/syft/internal/fileresolver/filetree_resolver.go index 68b3b8395..20c60b57a 100644 --- a/syft/internal/fileresolver/filetree_resolver.go +++ b/syft/internal/fileresolver/filetree_resolver.go @@ -6,6 +6,7 @@ import ( "fmt" "io" "os" + "path/filepath" stereoscopeFile "github.com/anchore/stereoscope/pkg/file" "github.com/anchore/stereoscope/pkg/filetree" @@ -34,8 +35,17 @@ func nativeOSFileOpener(ref stereoscopeFile.Reference) (io.ReadCloser, error) { return stereoscopeFile.NewLazyReadCloser(filePath), nil } +// requestPath converts a user (chroot) path into the path the file tree is keyed on. The tree is always posix +// (windows paths are volume-encoded, e.g. /c/some/path), while ToNativePath yields a native host path. func (r *FiletreeResolver) requestPath(userPath string) (string, error) { - return r.Chroot.ToNativePath(userPath) + nativePath, err := r.Chroot.ToNativePath(userPath) + if err != nil { + return "", err + } + if windows.HostRunningOnWindows() { + return windows.ToPosix(nativePath), nil + } + return nativePath, nil } // responsePath takes a path from the underlying fs domain and converts it to a path that is relative to the root of the file resolver. @@ -85,10 +95,6 @@ func (r FiletreeResolver) FilesByPath(userPaths ...string) ([]file.Location, err continue } - if windows.HostRunningOnWindows() { - userStrPath = windows.ToPosix(userStrPath) - } - if ref.HasReference() { references = append(references, file.NewVirtualLocationFromDirectory( @@ -103,8 +109,19 @@ func (r FiletreeResolver) FilesByPath(userPaths ...string) ([]file.Location, err return references, nil } +// requestGlob is the glob equivalent of requestPath. func (r FiletreeResolver) requestGlob(pattern string) (string, error) { - return r.Chroot.ToNativeGlob(pattern) + glob, err := r.Chroot.ToNativeGlob(pattern) + if err != nil { + return "", err + } + // only a glob that was anchored to the root comes back native. Globs that start with a wildcard + // (e.g. **/foo) are returned untouched and are already posix, and ToPosix would root them at "/". + // This uses filepath (not path) since we are asking whether this is a native absolute path. + if windows.HostRunningOnWindows() && filepath.IsAbs(glob) { + return windows.ToPosix(glob), nil + } + return glob, nil } // FilesByGlob returns all file.References that match the given path glob pattern from any layer in the image. diff --git a/syft/internal/parsing/basic_parsing.go b/syft/internal/parsing/basic_parsing.go index 27a1175c8..7cc794f86 100644 --- a/syft/internal/parsing/basic_parsing.go +++ b/syft/internal/parsing/basic_parsing.go @@ -22,6 +22,15 @@ func SkipWhitespace(data []byte, i *int) { } } +// MaxDepth bounds nesting in the recursive hand-written parsers. Real files nest a handful of levels; without a +// bound, deep input overflows the goroutine stack, which is fatal rather than a recoverable panic. +const MaxDepth = 1000 + +// errorContext is how many bytes of each echoed line PrintError keeps around the error column, so an error in a +// huge single-line input stays small (and does not copy the file into the SBOM as an unknown). +const errorContext = 80 + +// PrintError describes where in data offset i is, echoing up to two lines of context clipped around the column. func PrintError(data []byte, i int) string { line := 1 char := 1 @@ -57,15 +66,23 @@ func PrintError(data []byte, i int) string { sep := ": " - lines := "" - if len(prev) > 1 { - lines += fmt.Sprintf("%s%s%s\n", l1, sep, prev[len(prev)-2]) - } - if len(prev) > 0 { - lines += fmt.Sprintf("%s%s%s\n", l2, sep, prev[len(prev)-1]) + start := max(0, char-1-errorContext/2) + clip := func(s string) string { + if start >= len(s) { + return "" + } + return s[start:min(len(s), start+errorContext)] } - pointer := strings.Repeat(" ", len(l2)+len(sep)+char-1) + "^" + lines := "" + if len(prev) > 1 { + lines += fmt.Sprintf("%s%s%s\n", l1, sep, clip(prev[len(prev)-2])) + } + if len(prev) > 0 { + lines += fmt.Sprintf("%s%s%s\n", l2, sep, clip(prev[len(prev)-1])) + } + + pointer := strings.Repeat(" ", len(l2)+len(sep)+char-1-start) + "^" return fmt.Sprintf("line: %v, char: %v\n%s%s", line, char, lines, pointer) } diff --git a/syft/internal/windows/path.go b/syft/internal/windows/path.go index 409917103..fc16cbe5f 100644 --- a/syft/internal/windows/path.go +++ b/syft/internal/windows/path.go @@ -35,7 +35,15 @@ func ToPosix(windowsPath string) (posixPath string) { func FromPosix(posixPath string) (windowsPath string) { // decode the volume (e.g. /c/ --> C:\\) - There should always be a volume name. // The volume may be a UNC path (e.g. /\\localhost\C$\ --> \\localhost\C$\) + if !strings.HasPrefix(posixPath, "/") { + // not a volume-encoded posix path (e.g. relative or already native), so there is nothing to decode + return posixPath + } pathFields := strings.Split(posixPath, "/") + if len(pathFields) < 2 { + // no separator means no volume to decode + return filepath.Clean(posixPath) + } rootPath := strings.ToUpper(pathFields[1]) volumeName := AppendRootTerminator(rootPath) diff --git a/syft/internal/windows/path_windows_test.go b/syft/internal/windows/path_windows_test.go new file mode 100644 index 000000000..ad278fb9e --- /dev/null +++ b/syft/internal/windows/path_windows_test.go @@ -0,0 +1,35 @@ +package windows + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestToPosix(t *testing.T) { + tests := map[string]string{ + `C:\some\windows\place`: "/c/some/windows/place", + `C:\\some\\windows\\place`: "/c/some/windows/place", + `C:/foo/bar`: "/c/foo/bar", + `C:\foo/bar\`: "/c/foo/bar", + `C:\Foo/bAr\`: "/c/Foo/bAr", + `C:\ふー\バー`: "/c/ふー/バー", + } + for in, want := range tests { + assert.Equal(t, want, ToPosix(in), in) + } +} + +func TestFromPosix(t *testing.T) { + tests := map[string]string{ + "/c/some/windows/place": `C:\some\windows\place`, + "/c/Foo/bAr": `C:\Foo\bAr`, + "/c/ふー/バー": `C:\ふー\バー`, + // non volume-encoded input is left alone + "relative/path": "relative/path", + `D:\already\native`: `D:\already\native`, + } + for in, want := range tests { + assert.Equal(t, want, FromPosix(in), in) + } +} diff --git a/syft/linux/identify_release.go b/syft/linux/identify_release.go index 5025cdde5..de45c2db7 100644 --- a/syft/linux/identify_release.go +++ b/syft/linux/identify_release.go @@ -156,8 +156,8 @@ var busyboxVersionMatcher = regexp.MustCompile(`BusyBox v[\d.]+`) func parseBusyBox(contents string) (*Release, error) { matches := busyboxVersionMatcher.FindAllString(contents, -1) for _, match := range matches { - parts := strings.Split(match, " ") - version := strings.ReplaceAll(parts[1], "v", "") + _, version, _ := strings.Cut(match, " ") + version = strings.ReplaceAll(version, "v", "") return simpleRelease(match, "busybox", version, ""), nil } diff --git a/syft/pkg/cataloger/ai/cataloger.go b/syft/pkg/cataloger/ai/cataloger.go index 5b7a75882..9aa7ef4b3 100644 --- a/syft/pkg/cataloger/ai/cataloger.go +++ b/syft/pkg/cataloger/ai/cataloger.go @@ -11,8 +11,12 @@ import ( const ( catalogerName = "gguf-cataloger" - ggufLayerMediaType = "application/vnd.docker.ai*" safeTensorsCatalogerName = "safetensors-cataloger" + + // dockerAIGGUFMediaType mirrors ggufLayerMediaType in + // syft/source/ocimodelsource/registry_client.go. It is matched exactly: the + // other application/vnd.docker.ai.* layers are not GGUF. + dockerAIGGUFMediaType = "application/vnd.docker.ai.gguf.v3" ) // NewGGUFCataloger returns a new cataloger instance for GGUF model files. @@ -21,7 +25,7 @@ const ( func NewGGUFCataloger() pkg.Cataloger { return generic.NewCataloger(catalogerName). WithParserByGlobs(parseGGUFModel, "**/*.gguf"). - WithParserByMediaType(parseGGUFModel, ggufLayerMediaType). + WithParserByMediaType(parseGGUFModel, dockerAIGGUFMediaType). WithProcessors(ggufMergeProcessor) } diff --git a/syft/pkg/cataloger/ai/cataloger_test.go b/syft/pkg/cataloger/ai/cataloger_test.go index d8c28d6d0..bc125314a 100644 --- a/syft/pkg/cataloger/ai/cataloger_test.go +++ b/syft/pkg/cataloger/ai/cataloger_test.go @@ -1,11 +1,17 @@ package ai import ( + "context" "os" "path/filepath" "testing" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/anchore/syft/internal/tmpdir" "github.com/anchore/syft/syft/artifact" + "github.com/anchore/syft/syft/internal/fileresolver" "github.com/anchore/syft/syft/pkg" "github.com/anchore/syft/syft/pkg/cataloger/internal/pkgtest" ) @@ -138,3 +144,42 @@ func TestGGUFCataloger(t *testing.T) { }) } } + +func TestGGUFCataloger_ociLayers(t *testing.T) { + // a GGUF OCI artifact is merged into one named package, while the other + // application/vnd.docker.ai.* layers (config, model.file, license and + // safetensors weights) never reach the GGUF parser. + named := newTestGGUFBuilder(). + withStringKV("general.architecture", "llama"). + withStringKV("general.name", "llama3-8b"). + build() + part := newTestGGUFBuilder().withStringKV("general.architecture", "llama").build() + + dir := t.TempDir() + layers := map[string]fileresolver.LayerInfo{} + for digest, l := range map[string]struct { + mediaType string + content []byte + }{ + "sha256:named": {dockerAIGGUFMediaType, named}, + "sha256:part": {dockerAIGGUFMediaType, part}, + "sha256:config": {dockerAIModelConfigMediaTypes[0], []byte("not a gguf file")}, + "sha256:file": {dockerAIModelFileMediaType, []byte("not a gguf file")}, + "sha256:license": {dockerAILicenseMediaType, []byte("not a gguf file")}, + "sha256:weights": {dockerAISafeTensorsMediaType, []byte("not a gguf file")}, + } { + p := filepath.Join(dir, digest[len("sha256:"):]) + require.NoError(t, os.WriteFile(p, l.content, 0o600)) + layers[digest] = fileresolver.LayerInfo{TempPath: p, MediaType: l.mediaType} + } + resolver := fileresolver.NewContainerImageModel(dir, layers, "example/model:latest") + ctx := tmpdir.WithValue(context.Background(), tmpdir.FromPath(t.TempDir())) + + pkgs, _, err := NewGGUFCataloger().Catalog(ctx, resolver) + require.NoError(t, err) + require.Len(t, pkgs, 1) + assert.Equal(t, "llama3-8b", pkgs[0].Name) + header, ok := pkgs[0].Metadata.(pkg.GGUFFileHeader) + require.True(t, ok) + assert.Len(t, header.Parts, 1) +} diff --git a/syft/pkg/cataloger/ai/gguf_processor.go b/syft/pkg/cataloger/ai/gguf_processor.go index fe31e3f8b..34b93f006 100644 --- a/syft/pkg/cataloger/ai/gguf_processor.go +++ b/syft/pkg/cataloger/ai/gguf_processor.go @@ -8,17 +8,17 @@ import ( // ggufMergeProcessor consolidates multiple GGUF packages into a single package // representing the AI model. When scanning OCI images with multiple layers, // each layer may produce a separate package. This processor finds the package -// with a name and merges metadata from nameless packages into its GGUFFileParts field. +// with a name and merges metadata from nameless packages into its Parts field. // Only packages with a non-empty name are returned in the final result. func ggufMergeProcessor(pkgs []pkg.Package, rels []artifact.Relationship, err error) ([]pkg.Package, []artifact.Relationship, error) { - if err != nil { - return pkgs, rels, err - } + // don't early-return on err: it aggregates per-file failures, and skipping the + // merge would drop every nameless layer and the named package's Parts along + // with it (see safeTensorsMergeProcessor). if len(pkgs) == 0 { return pkgs, rels, err } - // Separate packages with names from those without + // separate packages with names from those without var namedPkgs []pkg.Package var namelessHeaders []pkg.GGUFFileHeader @@ -27,14 +27,14 @@ func ggufMergeProcessor(pkgs []pkg.Package, rels []artifact.Relationship, err er namedPkgs = append(namedPkgs, p) } else { if header, ok := p.Metadata.(pkg.GGUFFileHeader); ok { - // We do not want a kv hash for nameless headers + // we do not want a kv hash for nameless headers header.MetadataKeyValuesHash = "" namelessHeaders = append(namelessHeaders, header) } } } - // If there are no named packages, return nothing + // if there are no named packages, return nothing if len(namedPkgs) == 0 { return nil, rels, err } diff --git a/syft/pkg/cataloger/ai/huggingface.go b/syft/pkg/cataloger/ai/huggingface.go index 6c333835b..9b9d644b1 100644 --- a/syft/pkg/cataloger/ai/huggingface.go +++ b/syft/pkg/cataloger/ai/huggingface.go @@ -3,7 +3,7 @@ package ai import ( "bytes" - "gopkg.in/yaml.v3" + "go.yaml.in/yaml/v3" "github.com/anchore/syft/internal/log" "github.com/anchore/syft/syft/pkg" @@ -15,6 +15,12 @@ type hfConfig struct { NameOrPath string `json:"_name_or_path"` } +// looksLikeHF reports whether the config carries any field we use. Other JSON +// (generation_config.json, unrelated app configs) is ignored. +func (c hfConfig) looksLikeHF() bool { + return len(c.Architectures) > 0 || c.NameOrPath != "" +} + func applyHFConfig(md *pkg.SafeTensorsModelInfo, cfg *hfConfig) { if md.Architecture == "" && len(cfg.Architectures) > 0 { md.Architecture = cfg.Architectures[0] @@ -23,8 +29,8 @@ func applyHFConfig(md *pkg.SafeTensorsModelInfo, cfg *hfConfig) { // readmeFrontmatter holds the subset of YAML frontmatter fields we extract. type readmeFrontmatter struct { - License string `yaml:"license"` - BaseModel []string `yaml:"base_model"` + Licenses []string + BaseModel []string } type licenseFrontmatter struct { @@ -32,21 +38,33 @@ type licenseFrontmatter struct { } // extractFrontmatterBlock returns the YAML bytes between the first and second -// "---" delimiters of a file +// "---" delimiter lines of a file func extractFrontmatterBlock(buf []byte) []byte { trimmed := bytes.TrimLeft(buf, "\xef\xbb\xbf \t\r\n") - if !bytes.HasPrefix(trimmed, []byte("---")) { + first, rest, _ := bytes.Cut(trimmed, []byte("\n")) + if !isFrontmatterDelimiter(first) { return nil } - rest := trimmed[3:] - if i := bytes.IndexByte(rest, '\n'); i >= 0 { - rest = rest[i+1:] + for off := 0; off < len(rest); { + line, _, found := bytes.Cut(rest[off:], []byte("\n")) + if isFrontmatterDelimiter(line) { + if off == 0 || off > maxFrontmatterSize { + return nil + } + return rest[:off] + } + if !found { + break + } + off += len(line) + 1 } - block, _, found := bytes.Cut(rest, []byte("\n---")) - if !found { - return nil - } - return block + return nil +} + +// isFrontmatterDelimiter reports whether a line is exactly "---", ignoring +// trailing whitespace and \r. +func isFrontmatterDelimiter(line []byte) bool { + return bytes.Equal(bytes.TrimRight(line, " \t\r"), []byte("---")) } // parseFrontmatter decodes a Hugging Face model card YAML frontmatter block @@ -57,8 +75,9 @@ func parseFrontmatter(buf []byte) *readmeFrontmatter { return nil } + // both fields may be a scalar or a list, so decode them as nodes var raw struct { - License string `yaml:"license"` + License yaml.Node `yaml:"license"` BaseModel yaml.Node `yaml:"base_model"` } if err := yaml.Unmarshal(block, &raw); err != nil { @@ -66,16 +85,37 @@ func parseFrontmatter(buf []byte) *readmeFrontmatter { return nil } - fm := readmeFrontmatter{License: raw.License} - switch raw.BaseModel.Kind { + return &readmeFrontmatter{ + Licenses: yamlStrings(raw.License), + BaseModel: yamlStrings(raw.BaseModel), + } +} + +// yamlStrings returns the non-empty, non-null values of a scalar or a sequence +// of scalars, keeping at most maxFrontmatterValues. Anything else yields nil. +func yamlStrings(n yaml.Node) []string { + switch n.Kind { case yaml.ScalarNode: - if raw.BaseModel.Value != "" { - fm.BaseModel = []string{raw.BaseModel.Value} + if isYAMLString(&n) { + return []string{n.Value} } case yaml.SequenceNode: - _ = raw.BaseModel.Decode(&fm.BaseModel) + var out []string + for _, c := range n.Content { + if len(out) == maxFrontmatterValues { + break + } + if c.Kind == yaml.ScalarNode && isYAMLString(c) { + out = append(out, c.Value) + } + } + return out } - return &fm + return nil +} + +func isYAMLString(n *yaml.Node) bool { + return n.Value != "" && n.ShortTag() != "!!null" } // parseLicenseFrontmatter returns the producer-declared SPDX identifier diff --git a/syft/pkg/cataloger/ai/identity_dir.go b/syft/pkg/cataloger/ai/identity_dir.go index eaa0d1efa..dd2460092 100644 --- a/syft/pkg/cataloger/ai/identity_dir.go +++ b/syft/pkg/cataloger/ai/identity_dir.go @@ -3,7 +3,6 @@ package ai import ( "context" "encoding/json" - "io" "path" "github.com/anchore/syft/internal" @@ -13,42 +12,42 @@ import ( ) // resolveSafeTensorsDirIdentity handles the directory-scan case for safe tensors -// find config.json beside the model files (walking up parent directories to the -// scanned source root if no sibling exists) and a sibling README.md. It returns +// find config.json beside the model files (or one directory up, for the shard +// subdirectory layout) and a sibling README.md. It returns // the group's name candidates, resolved licenses, and supporting evidence. func resolveSafeTensorsDirIdentity(ctx context.Context, resolver file.Resolver, dir string, md *pkg.SafeTensorsModelInfo) safeTensorsIdentity { id := safeTensorsIdentity{fallbackName: safeTensorsDirName(dir)} if loc, cfg := findDirHFConfig(resolver, dir); cfg != nil { applyHFConfig(md, cfg) - id.nameOrPath = cfg.NameOrPath + id.nameOrPath = sanitizeModelName(cfg.NameOrPath) id.supporting = append(id.supporting, *loc) } if loc, fm := readDirReadmeFrontmatter(resolver, path.Join(dir, "README.md")); fm != nil { - if fm.License != "" { - id.licenses = pkg.NewLicensesFromValuesWithContext(ctx, fm.License) + if len(fm.Licenses) > 0 { + id.licenses = pkg.NewLicensesFromValuesWithContext(ctx, fm.Licenses...) } - if id.nameOrPath == "" && len(fm.BaseModel) > 0 { - id.nameOrPath = fm.BaseModel[0] + if id.nameOrPath == "" { + id.nameOrPath = firstModelName(fm.BaseModel) } id.supporting = append(id.supporting, *loc) } return id } -// findDirHFConfig looks for a config.json beside the model files +// findDirHFConfig looks for an HF config.json beside the model files, then in +// the parent directory (covers weights kept in a subdirectory of the model). It +// does not walk further up, so an unrelated config.json higher in the tree can't +// name every model below it. func findDirHFConfig(resolver file.Resolver, dir string) (*file.Location, *hfConfig) { - for { - if loc, cfg := readDirHFConfig(resolver, path.Join(dir, "config.json")); cfg != nil { - return loc, cfg - } - parent := path.Dir(dir) - if parent == dir { - return nil, nil // reached the source root - } - dir = parent + if loc, cfg := readDirHFConfig(resolver, path.Join(dir, "config.json")); cfg != nil { + return loc, cfg } + if parent := path.Dir(dir); parent != dir { + return readDirHFConfig(resolver, path.Join(parent, "config.json")) + } + return nil, nil } func readDirHFConfig(resolver file.Resolver, p string) (*file.Location, *hfConfig) { @@ -62,11 +61,19 @@ func readDirHFConfig(resolver file.Resolver, p string) (*file.Location, *hfConfi } defer internal.CloseAndLogError(rc, p) + buf, err := readBounded(rc, maxHFConfigSize) + if err != nil { + log.Debugf("failed to read %s: %v", p, err) + return nil, nil + } var cfg hfConfig - if err := json.NewDecoder(rc).Decode(&cfg); err != nil { + if err := json.Unmarshal(buf, &cfg); err != nil { log.Debugf("failed to decode %s: %v", p, err) return nil, nil } + if !cfg.looksLikeHF() { + return nil, nil + } return &locations[0], &cfg } @@ -81,8 +88,9 @@ func readDirReadmeFrontmatter(resolver file.Resolver, p string) (*file.Location, } defer internal.CloseAndLogError(rc, p) - buf, err := io.ReadAll(io.LimitReader(rc, 1024*1024)) + buf, err := readPrefix(rc, maxReadmePrefixSize) if err != nil { + log.Debugf("failed to read %s: %v", p, err) return nil, nil } fm := parseFrontmatter(buf) diff --git a/syft/pkg/cataloger/ai/identity_oci.go b/syft/pkg/cataloger/ai/identity_oci.go index 6209a3f8b..97d58aa6b 100644 --- a/syft/pkg/cataloger/ai/identity_oci.go +++ b/syft/pkg/cataloger/ai/identity_oci.go @@ -4,8 +4,8 @@ import ( "bytes" "context" "encoding/json" - "io" "path" + "sort" gcrname "github.com/google/go-containerregistry/pkg/name" @@ -28,9 +28,18 @@ func resolveSafeTensorsOCIIdentity(ctx context.Context, resolver file.Resolver, log.Debugf("failed to list docker AI model-file layers: %v", err) } - // Collect config / readme candidates separately so the layer-iteration order - // returned by the resolver doesn't decide the precedence. - var configName, readmeName, readmeLicense string + // the resolver returns layers in map order, and the first config / README + // seen wins below, so sort for a stable result across runs. For OCI + // artifacts every RealPath is "/", so in practice this is digest order. + sort.Slice(modelFileLocs, func(i, j int) bool { + if modelFileLocs[i].RealPath != modelFileLocs[j].RealPath { + return modelFileLocs[i].RealPath < modelFileLocs[j].RealPath + } + return modelFileLocs[i].FileSystemID < modelFileLocs[j].FileSystemID + }) + + var configName, readmeName string + var readmeLicenses []string var supporting []file.Location for _, loc := range modelFileLocs { cfg, fm := classifyOCIModelFileLayer(resolver, loc) @@ -38,15 +47,15 @@ func resolveSafeTensorsOCIIdentity(ctx context.Context, resolver file.Resolver, case cfg != nil: applyHFConfig(md, cfg) if configName == "" { - configName = cfg.NameOrPath + configName = sanitizeModelName(cfg.NameOrPath) } supporting = append(supporting, loc) case fm != nil: - if readmeLicense == "" { - readmeLicense = fm.License + if len(readmeLicenses) == 0 { + readmeLicenses = fm.Licenses } - if readmeName == "" && len(fm.BaseModel) > 0 { - readmeName = fm.BaseModel[0] + if readmeName == "" { + readmeName = firstModelName(fm.BaseModel) } supporting = append(supporting, loc) } @@ -64,7 +73,7 @@ func resolveSafeTensorsOCIIdentity(ctx context.Context, resolver file.Resolver, supporting: supporting, } - // License precedence: a dedicated vnd.docker.ai.license layer is a + // License precedence: a dedicated vnd.docker.ai.license layer // outranks the free-text license field in a model card's README frontmatter. licLocs, err := ociResolver.FilesByMediaType(dockerAILicenseMediaType) if err != nil { @@ -74,8 +83,8 @@ func resolveSafeTensorsOCIIdentity(ctx context.Context, resolver file.Resolver, case len(licLocs) > 0: id.licenses = identifyLicenseLayers(ctx, resolver, licLocs) id.supporting = append(id.supporting, licLocs...) - case readmeLicense != "": - id.licenses = pkg.NewLicensesFromValuesWithContext(ctx, readmeLicense) + case len(readmeLicenses) > 0: + id.licenses = pkg.NewLicensesFromValuesWithContext(ctx, readmeLicenses...) } return id @@ -108,10 +117,11 @@ func ociImageRefBasename(resolver file.Resolver) string { return path.Base(parsed.Context().RepositoryStr()) } -// classifyOCIModelFileLayer reads up to 4 MiB of a model.file layer and decodes -// it as either an HF config.json or a README model card's YAML frontmatter, -// based on its leading bytes. It returns whichever it recognized; both are nil -// when the layer is neither (or fails to decode). The caller owns precedence and +// classifyOCIModelFileLayer reads a model.file layer (up to maxModelFileLayerSize) +// and decodes it as either an HF config.json or a README model card's YAML +// frontmatter, based on its leading bytes. It returns whichever it recognized; +// both are nil when the layer is neither, fails to decode, or is JSON that +// doesn't look like an HF config (e.g. generation_config.json). The caller owns precedence and // metadata enrichment. func classifyOCIModelFileLayer(resolver file.Resolver, loc file.Location) (*hfConfig, *readmeFrontmatter) { rc, err := resolver.FileContentsByLocation(loc) @@ -120,17 +130,18 @@ func classifyOCIModelFileLayer(resolver file.Resolver, loc file.Location) (*hfCo } defer internal.CloseAndLogError(rc, loc.RealPath) - buf, err := io.ReadAll(io.LimitReader(rc, 4*1024*1024)) + buf, err := readBounded(rc, maxModelFileLayerSize) if err != nil { + log.Debugf("failed to read docker AI model-file layer: %v", err) return nil, nil } trimmed := bytes.TrimLeft(buf, "\xef\xbb\xbf \t\r\n") switch { case bytes.HasPrefix(trimmed, []byte("---")): - return nil, parseFrontmatter(buf) + return nil, parseFrontmatter(trimmed) case bytes.HasPrefix(trimmed, []byte("{")): var cfg hfConfig - if err := json.Unmarshal(buf, &cfg); err != nil { + if err := json.Unmarshal(trimmed, &cfg); err != nil || !cfg.looksLikeHF() { return nil, nil } return &cfg, nil @@ -165,8 +176,9 @@ func readLicenseSPDXIDFromFrontmatter(resolver file.Resolver, loc file.Location) } defer internal.CloseAndLogError(rc, loc.RealPath) - buf, err := io.ReadAll(io.LimitReader(rc, 64*1024)) + buf, err := readPrefix(rc, maxLicensePrefixSize) if err != nil { + log.Debugf("failed to read docker AI license layer: %v", err) return "" } return parseLicenseFrontmatter(buf) diff --git a/syft/pkg/cataloger/ai/limits.go b/syft/pkg/cataloger/ai/limits.go new file mode 100644 index 000000000..9de7bfd58 --- /dev/null +++ b/syft/pkg/cataloger/ai/limits.go @@ -0,0 +1,46 @@ +package ai + +import ( + "fmt" + "io" + + "github.com/anchore/syft/internal" +) + +// size limits for model companion files and headers. JSON reads that hit a +// limit fail (see readBounded) instead of parsing a truncated document. +// Frontmatter-only reads (README, license) take a prefix instead, since the +// frontmatter sits at the top and is itself capped by maxFrontmatterSize. +const ( + maxHFConfigSize = 4 * 1024 * 1024 + maxReadmePrefixSize = 1 * 1024 * 1024 // dir scans; OCI README layers use maxModelFileLayerSize + maxModelFileLayerSize = 4 * 1024 * 1024 + maxLicensePrefixSize = 64 * 1024 + maxModelConfigBlobSize = 1 * 1024 * 1024 + maxFrontmatterSize = 256 * 1024 + maxFrontmatterValues = 32 + maxModelNameLength = 256 + + // maxSafeTensorsHeaderSize caps the header JSON (excluding its 8-byte length + // prefix). It is shared with the OCI source so dir and OCI scans accept the + // same headers. + maxSafeTensorsHeaderSize = internal.MaxSafeTensorsHeaderSize +) + +// readPrefix reads at most limit bytes. It is for frontmatter-only reads, where +// the rest of the file is never needed. +func readPrefix(r io.Reader, limit int64) ([]byte, error) { + return io.ReadAll(io.LimitReader(r, limit)) +} + +// readBounded reads at most limit bytes and reports overflow instead of silently truncating. +func readBounded(r io.Reader, limit int64) ([]byte, error) { + b, err := io.ReadAll(io.LimitReader(r, limit+1)) + if err != nil { + return nil, err + } + if int64(len(b)) > limit { + return nil, fmt.Errorf("content exceeds %d bytes", limit) + } + return b, nil +} diff --git a/syft/pkg/cataloger/ai/merge.go b/syft/pkg/cataloger/ai/merge.go index a8d2b4e74..24e37e685 100644 --- a/syft/pkg/cataloger/ai/merge.go +++ b/syft/pkg/cataloger/ai/merge.go @@ -75,7 +75,7 @@ func mergeShardsInto(merged *pkg.SafeTensorsModelInfo, shards []pkg.SafeTensorsM seenKV := map[string]bool{} for _, s := range shards { shardTensorTotal += s.TensorCount - merged.Parameters += s.Parameters + merged.Parameters = saturatingAdd(merged.Parameters, s.Parameters) firstNonEmpty(&merged.Quantization, s.Quantization) for _, kv := range s.UserMetadata { if seenKV[kv.Key] { diff --git a/syft/pkg/cataloger/ai/merge_test.go b/syft/pkg/cataloger/ai/merge_test.go index 7276d8896..08f22939f 100644 --- a/syft/pkg/cataloger/ai/merge_test.go +++ b/syft/pkg/cataloger/ai/merge_test.go @@ -1,6 +1,7 @@ package ai import ( + "math" "testing" "github.com/stretchr/testify/assert" @@ -164,3 +165,10 @@ func TestRollupHash(t *testing.T) { assert.NotEqual(t, "a", ab) assert.NotEqual(t, "b", ab) } + +func TestMergeShardsInto_parametersSaturate(t *testing.T) { + var merged pkg.SafeTensorsModelInfo + shards := []pkg.SafeTensorsModelInfo{{Parameters: math.MaxUint64 - 1}, {Parameters: 2}} + mergeShardsInto(&merged, shards) + assert.Equal(t, uint64(math.MaxUint64), merged.Parameters) +} diff --git a/syft/pkg/cataloger/ai/naming.go b/syft/pkg/cataloger/ai/naming.go index ed2571dcb..65de1d367 100644 --- a/syft/pkg/cataloger/ai/naming.go +++ b/syft/pkg/cataloger/ai/naming.go @@ -1,15 +1,44 @@ package ai -import "path" +import ( + "path" + "strings" + "unicode/utf8" +) // pickSafeTensorsName implements the documented naming precedence chain: func pickSafeTensorsName(nameOrPath, fallbackName string) string { - if nameOrPath != "" { - return path.Base(nameOrPath) + if name := sanitizeModelName(nameOrPath); name != "" { + return name } return fallbackName } +// firstModelName returns the first value that sanitizes to a usable name. +func firstModelName(values []string) string { + for _, v := range values { + if name := sanitizeModelName(v); name != "" { + return name + } + } + return "" +} + +// sanitizeModelName reduces a producer-supplied name or path (_name_or_path, +// base_model) to its last path element, capped at maxModelNameLength runes. It +// returns "" when nothing usable is left. +func sanitizeModelName(nameOrPath string) string { + base := path.Base(strings.ReplaceAll(nameOrPath, `\`, "/")) + switch base { + case "/", ".", "..", "": + return "" + } + if utf8.RuneCountInString(base) > maxModelNameLength { + base = string([]rune(base)[:maxModelNameLength]) + } + return base +} + // safeTensorsDirName returns the directory-scan naming fallback: the base name // of the group's parent directory (the group key is already that directory). func safeTensorsDirName(directory string) string { diff --git a/syft/pkg/cataloger/ai/package.go b/syft/pkg/cataloger/ai/package.go index 3abb42343..978ad2a95 100644 --- a/syft/pkg/cataloger/ai/package.go +++ b/syft/pkg/cataloger/ai/package.go @@ -1,17 +1,19 @@ package ai import ( + "context" + "github.com/anchore/syft/syft/file" "github.com/anchore/syft/syft/pkg" ) -func newGGUFPackage(metadata *pkg.GGUFFileHeader, modelName, version, license string, locations ...file.Location) pkg.Package { +func newGGUFPackage(ctx context.Context, metadata *pkg.GGUFFileHeader, modelName, version, license string, locations ...file.Location) pkg.Package { p := pkg.Package{ Name: modelName, Version: version, Locations: file.NewLocationSet(locations...), Type: pkg.ModelPkg, - Licenses: pkg.NewLicenseSet(pkg.NewLicensesFromValues(license)...), + Licenses: pkg.NewLicenseSet(pkg.NewLicensesFromValuesWithContext(ctx, license)...), Metadata: *metadata, // NOTE: PURL is intentionally not set as the package-url spec // has not yet finalized support for ML model packages diff --git a/syft/pkg/cataloger/ai/package_test.go b/syft/pkg/cataloger/ai/package_test.go index ea58304d6..3c8efbb7f 100644 --- a/syft/pkg/cataloger/ai/package_test.go +++ b/syft/pkg/cataloger/ai/package_test.go @@ -1,6 +1,7 @@ package ai import ( + "context" "testing" "github.com/stretchr/testify/require" @@ -103,6 +104,7 @@ func TestNewGGUFPackage(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { actual := newGGUFPackage( + context.Background(), tt.metadata, tt.input.modelName, tt.input.version, diff --git a/syft/pkg/cataloger/ai/parse_gguf.go b/syft/pkg/cataloger/ai/parse_gguf.go index 3150110a3..efcfc232d 100644 --- a/syft/pkg/cataloger/ai/parse_gguf.go +++ b/syft/pkg/cataloger/ai/parse_gguf.go @@ -1,9 +1,14 @@ package ai import ( + "bufio" "encoding/binary" + "errors" "fmt" "io" + "math" + "slices" + "strings" gguf_parser "github.com/gpustack/gguf-parser-go" ) @@ -12,12 +17,41 @@ import ( const ( ggufMagicNumber = 0x46554747 // "GGUF" in little-endian maxHeaderSize = 50 * 1024 * 1024 // 50MB for large tokenizer vocabularies + + // maxGGUFArrayDepth allows arrays of arrays but nothing deeper. + maxGGUFArrayDepth = 2 + // maxGGUFDecodedItems bounds the items, across the whole header, of arrays that gguf-parser-go + // always decodes in full. Real files hold a few per-layer arrays. + maxGGUFDecodedItems = 65536 + // maxGGUFSplitTensors bounds split.tensors.count, which the library preallocates tensor infos for. + // Shards declare the total across all splits, so this can't be the header tensor count. + maxGGUFSplitTensors = 1 << 17 + // maxGGUFKeyLength is the key length limit from the GGUF spec. + maxGGUFKeyLength = 65535 + // maxGGUFKVCount bounds the metadata KV count. Real files hold hundreds; vocabularies live in arrays. + maxGGUFKVCount = 65536 + // maxGGUFReadStringBytes bounds, across the header, the key and string bytes the library reads in + // full and syft then hashes and encodes (chat templates are the largest, at tens of KB). + maxGGUFReadStringBytes = 8 << 20 + // maxGGUFSkippedStrings bounds, across the header, the strings the library skips one at a time + // inside arrays (vocabularies run to a few hundred thousand tokens and merges). + maxGGUFSkippedStrings = 1 << 22 ) -// copyHeader copies the GGUF header from the reader to the writer. +// ggufDecodedArraySuffixes are the keys gguf-parser-go decodes in full even with SkipLargeMetadata +// (hardcoded in its ReadArray as of v0.26.3; re-check on upgrade). +var ggufDecodedArraySuffixes = []string{ + ".feed_forward_length", ".attention.head_count", ".attention.head_count_kv", ".attention.sliding_window_pattern", +} + +// ggufScalarSizes is the encoded size of each value type, indexed by type (0 for strings and arrays). +var ggufScalarSizes = [...]int{1, 1, 2, 2, 4, 4, 4, 1, 0, 0, 8, 8, 8} + +// copyHeader copies at most maxHeaderSize bytes of the GGUF header from the reader to the writer. // It validates the magic number first, then copies the rest of the data. -// The reader should be wrapped with io.LimitedReader to prevent OOM issues. func copyHeader(w io.Writer, r io.Reader) error { + r = io.LimitReader(r, maxHeaderSize) + // Read initial chunk to validate magic number // GGUF format: magic(4) + version(4) + tensor_count(8) + metadata_kv_count(8) + metadata_kvs + tensors_info initialBuf := make([]byte, 24) // Enough for magic, version, tensor count, and kv count @@ -37,7 +71,6 @@ func copyHeader(w io.Writer, r io.Reader) error { } // Copy the rest of the header from reader to writer - // The LimitedReader will return EOF once maxHeaderSize is reached if _, err := io.Copy(w, r); err != nil { return fmt.Errorf("failed to copy GGUF header: %w", err) } @@ -45,7 +78,269 @@ func copyHeader(w io.Writer, r io.Reader) error { return nil } -// Helper to convert gguf_parser metadata to simpler types +// validateGGUFHeader walks the metadata KV section and rejects values that gguf-parser-go would panic +// on, recurse on without bound, or overallocate for. It reads without storing anything. +// ponytail: tensor infos are not checked here (zero block size types, dimension overflow); those +// panics are recoverable and are left to the per-file panic recovery in the generic cataloger. +func validateGGUFHeader(r io.Reader) error { + v := &ggufHeaderValidator{ + r: bufio.NewReader(r), + decodedBudget: maxGGUFDecodedItems, + readStringBudget: maxGGUFReadStringBytes, + skippedStringBudget: maxGGUFSkippedStrings, + } + var prefix struct { + Magic, Version uint32 + } + if err := binary.Read(v.r, binary.LittleEndian, &prefix); err != nil { + return fmt.Errorf("reading header prefix: %w", err) + } + v.v1 = prefix.Version <= 1 + // the tensor count is validated by the library + if _, err := v.readLen(); err != nil { + return fmt.Errorf("reading tensor count: %w", err) + } + kvCount, err := v.readLen() + if err != nil { + return fmt.Errorf("reading metadata count: %w", err) + } + if kvCount > maxGGUFKVCount { + return fmt.Errorf("metadata count %d exceeds %d", kvCount, maxGGUFKVCount) + } + + for i := uint64(0); i < kvCount; i++ { + key, err := v.readKey() + if err != nil { + return fmt.Errorf("metadata key %d: %w", i, err) + } + var vt uint32 + if err := binary.Read(v.r, binary.LittleEndian, &vt); err != nil { + return fmt.Errorf("metadata key %q: %w", key, err) + } + if err := v.checkKey(key, gguf_parser.GGUFMetadataValueType(vt)); err != nil { + return fmt.Errorf("metadata key %q: %w", key, err) + } + } + return nil +} + +type ggufHeaderValidator struct { + r *bufio.Reader + v1 bool // v1 uses 32-bit lengths + decodedBudget uint64 + readStringBudget uint64 + skippedStringBudget uint64 +} + +func (v *ggufHeaderValidator) checkKey(key string, vt gguf_parser.GGUFMetadataValueType) error { + decoded := slices.ContainsFunc(ggufDecodedArraySuffixes, func(s string) bool { return strings.HasSuffix(key, s) }) + n, err := v.readValue(vt, decoded, 1) + if err != nil { + return err + } + + // these are read with typed accessors that panic on a type mismatch + switch key { + case "general.architecture", "general.name", "general.license", "general.type", "general.author", + "general.url", "general.description", "controlvector.model_hint": + if vt != gguf_parser.GGUFMetadataValueTypeString { + return fmt.Errorf("expected string, got %v", vt) + } + case "general.quantization_version", "general.file_type": + if !vt.IsNumeric() { + return fmt.Errorf("expected number, got %v", vt) + } + case "general.alignment": + if vt != gguf_parser.GGUFMetadataValueTypeUint32 || n == 0 { + return fmt.Errorf("expected non-zero uint32, got %v %v", vt, n) + } + case "split.tensors.count": + if !vt.IsNumeric() { + return fmt.Errorf("expected number, got %v", vt) + } + if math.IsNaN(n) || n < 0 || n > maxGGUFSplitTensors { + return fmt.Errorf("unreasonable split tensor count %.0f", n) + } + } + return nil +} + +// readValue consumes one value, returning scalar numbers as float64 for the key checks. +func (v *ggufHeaderValidator) readValue(vt gguf_parser.GGUFMetadataValueType, decoded bool, depth int) (float64, error) { + if uint32(vt) >= uint32(len(ggufScalarSizes)) { + return 0, fmt.Errorf("invalid value type %d", vt) + } + switch vt { + case gguf_parser.GGUFMetadataValueTypeString: + // only strings inside skipped arrays are skipped by the library, the rest are read in full + return 0, v.skipString(depth > 1 && !decoded) + case gguf_parser.GGUFMetadataValueTypeArray: + return 0, v.readArray(decoded, depth) + } + size := ggufScalarSizes[vt] + b, err := v.r.Peek(size) + if err != nil { + return 0, err + } + var buf [8]byte + copy(buf[:], b) + if _, err := v.r.Discard(size); err != nil { + return 0, err + } + u := binary.LittleEndian.Uint64(buf[:]) + switch vt { + case gguf_parser.GGUFMetadataValueTypeInt8: + return float64(int8(u)), nil + case gguf_parser.GGUFMetadataValueTypeInt16: + return float64(int16(u)), nil + case gguf_parser.GGUFMetadataValueTypeInt32: + return float64(int32(u)), nil + case gguf_parser.GGUFMetadataValueTypeInt64: + return float64(int64(u)), nil + case gguf_parser.GGUFMetadataValueTypeFloat32: + return float64(math.Float32frombits(uint32(u))), nil + case gguf_parser.GGUFMetadataValueTypeFloat64: + return math.Float64frombits(u), nil + } + return float64(u), nil +} + +func (v *ggufHeaderValidator) readArray(decoded bool, depth int) error { + var itemType uint32 + if err := binary.Read(v.r, binary.LittleEndian, &itemType); err != nil { + return err + } + n, err := v.readLen() + if err != nil { + return err + } + it := gguf_parser.GGUFMetadataValueType(itemType) + switch { + case itemType >= uint32(len(ggufScalarSizes)): + return fmt.Errorf("invalid array item type %d", itemType) + case decoded && n > v.decodedBudget: + return fmt.Errorf("decoded arrays exceed %d items", maxGGUFDecodedItems) + case it == gguf_parser.GGUFMetadataValueTypeArray && (!decoded || depth >= maxGGUFArrayDepth): + // the library can only skip over arrays of scalars + return errors.New("unsupported nested array") + case n > maxHeaderSize: + return fmt.Errorf("array length %d exceeds header size", n) + case !decoded && ggufScalarSizes[itemType] > 0: + // skipped arrays of fixed size items can be passed over in one go + _, err := v.r.Discard(int(n) * ggufScalarSizes[itemType]) + return err + } + if decoded { + v.decodedBudget -= n + } + for i := uint64(0); i < n; i++ { + if _, err := v.readValue(it, decoded, depth+1); err != nil { + return err + } + } + return nil +} + +func (v *ggufHeaderValidator) readLen() (uint64, error) { + if v.v1 { + var n uint32 + err := binary.Read(v.r, binary.LittleEndian, &n) + return uint64(n), err + } + var n uint64 + err := binary.Read(v.r, binary.LittleEndian, &n) + return n, err +} + +// readKey reads a key the way gguf-parser-go will see it, whitespace trimmed. +func (v *ggufHeaderValidator) readKey() (string, error) { + n, err := v.readLen() + if err != nil { + return "", err + } + if n == 0 || n > maxGGUFKeyLength { + return "", fmt.Errorf("invalid key length %d", n) + } + if err := v.spendReadString(n); err != nil { + return "", err + } + buf := make([]byte, n) + if _, err := io.ReadFull(v.r, buf); err != nil { + return "", err + } + return strings.TrimSpace(string(buf)), nil +} + +func (v *ggufHeaderValidator) skipString(allowEmpty bool) error { + n, err := v.readLen() + if err != nil { + return err + } + // gguf-parser-go reads an empty string as a full pool buffer, so everything after it would be + // parsed out of step with this walk + if n == 0 && !allowEmpty { + return errors.New("empty string") + } + if n > maxHeaderSize { + return fmt.Errorf("string length %d exceeds header size", n) + } + if allowEmpty { + if v.skippedStringBudget == 0 { + return fmt.Errorf("skipped strings exceed %d", maxGGUFSkippedStrings) + } + v.skippedStringBudget-- + } else if err := v.spendReadString(n); err != nil { + return err + } + _, err = v.r.Discard(int(n)) + return err +} + +func (v *ggufHeaderValidator) spendReadString(n uint64) error { + if n > v.readStringBudget { + return fmt.Errorf("strings exceed %d bytes", maxGGUFReadStringBytes) + } + v.readStringBudget -= n + return nil +} + +// sanitizeGGUFValue replaces non-finite floats, which JSON can not encode, leaving everything else as is. +func sanitizeGGUFValue(v any) any { + switch t := v.(type) { + case float32: + if f := float64(t); math.IsNaN(f) || math.IsInf(f, 0) { + return nonFiniteString(f) + } + case float64: + if math.IsNaN(t) || math.IsInf(t, 0) { + return nonFiniteString(t) + } + case gguf_parser.GGUFMetadataKVArrayValue: + // keep the library shape as is; items are only present for the few keys the library + // decodes in full, which the pre-scan keeps small and shallow + if len(t.Array) > 0 { + items := make([]any, len(t.Array)) + for i, item := range t.Array { + items[i] = sanitizeGGUFValue(item) + } + t.Array = items + } + return t + } + return v +} + +func nonFiniteString(f float64) string { + switch { + case math.IsInf(f, 1): + return "+Inf" + case math.IsInf(f, -1): + return "-Inf" + } + return "NaN" +} + +// helper to convert gguf_parser metadata to simpler types func convertGGUFMetadataKVs(kvs gguf_parser.GGUFMetadataKVs) map[string]any { result := make(map[string]any) @@ -56,7 +351,7 @@ func convertGGUFMetadataKVs(kvs gguf_parser.GGUFMetadataKVs) map[string]any { "general.version", "general.parameter_count", "general.quantization": continue } - result[kv.Key] = kv.Value + result[kv.Key] = sanitizeGGUFValue(kv.Value) } return result diff --git a/syft/pkg/cataloger/ai/parse_gguf_model.go b/syft/pkg/cataloger/ai/parse_gguf_model.go index f59fddfd5..6b9a89259 100644 --- a/syft/pkg/cataloger/ai/parse_gguf_model.go +++ b/syft/pkg/cataloger/ai/parse_gguf_model.go @@ -13,9 +13,7 @@ import ( gguf_parser "github.com/gpustack/gguf-parser-go" "github.com/anchore/syft/internal" - "github.com/anchore/syft/internal/log" "github.com/anchore/syft/internal/tmpdir" - "github.com/anchore/syft/internal/unknown" "github.com/anchore/syft/syft/artifact" "github.com/anchore/syft/syft/file" "github.com/anchore/syft/syft/pkg" @@ -38,14 +36,20 @@ func parseGGUFModel(ctx context.Context, _ file.Resolver, _ *generic.Environment defer cleanup() tempPath := tempFile.Name() - // Copy and validate the GGUF file header using LimitedReader to prevent OOM - // We use LimitedReader to cap reads at maxHeaderSize (50MB) - limitedReader := &io.LimitedReader{R: reader, N: maxHeaderSize} - if err := copyHeader(tempFile, limitedReader); err != nil { + // copy and validate the GGUF file header (bounded to maxHeaderSize) + if err := copyHeader(tempFile, reader); err != nil { tempFile.Close() return nil, nil, fmt.Errorf("failed to copy GGUF header: %w", err) } + if _, err := tempFile.Seek(0, io.SeekStart); err != nil { + tempFile.Close() + return nil, nil, fmt.Errorf("failed to rewind GGUF header: %w", err) + } + err = validateGGUFHeader(tempFile) tempFile.Close() + if err != nil { + return nil, nil, fmt.Errorf("invalid GGUF header: %w", err) + } // Parse using gguf-parser-go with options to skip unnecessary data ggufFile, err := gguf_parser.ParseGGUFFile(tempPath, @@ -61,6 +65,11 @@ func parseGGUFModel(ctx context.Context, _ file.Resolver, _ *generic.Environment // Extract version separately (will be set on Package.Version) modelVersion := extractVersion(ggufFile.Header.MetadataKV) + metadataHash, err := computeKVMetadataHash(ggufFile.Header.MetadataKV) + if err != nil { + return nil, nil, err + } + // Convert to syft metadata structure syftMetadata := &pkg.GGUFFileHeader{ Architecture: metadata.Architecture, @@ -69,7 +78,7 @@ func parseGGUFModel(ctx context.Context, _ file.Resolver, _ *generic.Environment GGUFVersion: uint32(ggufFile.Header.Version), TensorCount: ggufFile.Header.TensorCount, RemainingKeyValues: convertGGUFMetadataKVs(ggufFile.Header.MetadataKV), - MetadataKeyValuesHash: computeKVMetadataHash(ggufFile.Header.MetadataKV), + MetadataKeyValuesHash: metadataHash, } // If model name is not in metadata, use filename @@ -79,6 +88,7 @@ func parseGGUFModel(ctx context.Context, _ file.Resolver, _ *generic.Environment // Create package from metadata p := newGGUFPackage( + ctx, syftMetadata, metadata.Name, modelVersion, @@ -86,14 +96,17 @@ func parseGGUFModel(ctx context.Context, _ file.Resolver, _ *generic.Environment reader.WithAnnotation(pkg.EvidenceAnnotationKey, pkg.PrimaryEvidenceAnnotation), ) - return []pkg.Package{p}, nil, unknown.IfEmptyf([]pkg.Package{p}, "unable to parse GGUF file") + return []pkg.Package{p}, nil, nil } -// computeKVMetadataHash computes a stable hash of the KV metadata for use as a global identifier -func computeKVMetadataHash(metadata gguf_parser.GGUFMetadataKVs) string { +// computeKVMetadataHash computes a stable hash of the sanitized KV metadata for use as a global identifier +func computeKVMetadataHash(metadata gguf_parser.GGUFMetadataKVs) (string, error) { // Sort the KV pairs by key for stable hashing sortedKVs := make([]gguf_parser.GGUFMetadataKV, len(metadata)) - copy(sortedKVs, metadata) + for i, kv := range metadata { + kv.Value = sanitizeGGUFValue(kv.Value) + sortedKVs[i] = kv + } sort.Slice(sortedKVs, func(i, j int) bool { return sortedKVs[i].Key < sortedKVs[j].Key }) @@ -101,13 +114,12 @@ func computeKVMetadataHash(metadata gguf_parser.GGUFMetadataKVs) string { // Marshal sorted KVs to JSON for stable hashing jsonBytes, err := json.Marshal(sortedKVs) if err != nil { - log.Debugf("failed to marshal metadata for hashing: %v", err) - return "" + return "", fmt.Errorf("failed to marshal GGUF metadata for hashing: %w", err) } // Compute xxhash hash := xxhash.Sum64(jsonBytes) - return fmt.Sprintf("%016x", hash) // 16 hex chars (64 bits) + return fmt.Sprintf("%016x", hash), nil // 16 hex chars (64 bits) } // extractVersion attempts to extract version from metadata KV pairs diff --git a/syft/pkg/cataloger/ai/parse_gguf_test.go b/syft/pkg/cataloger/ai/parse_gguf_test.go new file mode 100644 index 000000000..c5d8d5253 --- /dev/null +++ b/syft/pkg/cataloger/ai/parse_gguf_test.go @@ -0,0 +1,251 @@ +package ai + +import ( + "bytes" + "context" + "encoding/binary" + "encoding/json" + "io" + "math" + "strings" + "testing" + + gguf_parser "github.com/gpustack/gguf-parser-go" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/anchore/syft/internal/tmpdir" + "github.com/anchore/syft/syft/file" + "github.com/anchore/syft/syft/format/syftjson" + "github.com/anchore/syft/syft/pkg" + "github.com/anchore/syft/syft/sbom" +) + +func le(v any) []byte { + buf := new(bytes.Buffer) + binary.Write(buf, binary.LittleEndian, v) + return buf.Bytes() +} + +func ggufString(s string) []byte { + return append(le(uint64(len(s))), s...) +} + +func parseGGUFBytes(t *testing.T, data []byte) ([]pkg.Package, error) { + t.Helper() + ctx, td := tmpdir.Root(context.Background(), "gguf-test") + t.Cleanup(func() { _ = td.Cleanup() }) + reader := file.NewLocationReadCloser(file.NewLocation("/model.gguf"), io.NopCloser(bytes.NewReader(data))) + var pkgs []pkg.Package + var err error + require.NotPanics(t, func() { + pkgs, _, err = parseGGUFModel(ctx, nil, nil, reader) + }) + return pkgs, err +} + +func TestParseGGUFModel_malformedHeader(t *testing.T) { + // nested arrays of arrays, each one level deeper, ending in an empty uint8 array + nested := func(depth int) []byte { + v := ggufArray(ggufTypeUint8, 0, nil) + for i := 1; i < depth; i++ { + v = ggufArray(ggufTypeArray, 1, v) + } + return v + } + + // a header that declares more KVs than the cap, with none following + tooManyKVs := append(append([]byte("GGUF"), le(uint32(3))...), le([]uint64{0, maxGGUFKVCount + 1})...) + + tests := []struct { + name string + builder *testGGUFBuilder + data []byte // used instead of builder when set + wantErr bool + }{ + {name: "too many KVs", data: tooManyKVs, wantErr: true}, + {name: "split tensor count is a string", builder: newTestGGUFBuilder().withStringKV("split.tensors.count", "1"), wantErr: true}, + {name: "split tensor count over the cap", builder: newTestGGUFBuilder().withUint32KV("split.tensors.count", maxGGUFSplitTensors+1), wantErr: true}, + {name: "read strings too long in total", builder: newTestGGUFBuilder().withStringKV("x", strings.Repeat("a", maxGGUFReadStringBytes)), wantErr: true}, + { + name: "skipped strings too many in total", + builder: newTestGGUFBuilder().withRawKV("tokenizer.ggml.tokens", ggufTypeArray, ggufArray(ggufTypeString, maxGGUFSkippedStrings+1, make([]byte, 8*(maxGGUFSkippedStrings+1)))), + wantErr: true, + }, + {name: "v1 header", builder: newTestGGUFBuilder().withVersion(1).withStringKV("general.name", "m").withUint32KV("llama.context_length", 8)}, + {name: "architecture not a string", builder: newTestGGUFBuilder().withUint32KV("general.architecture", 1), wantErr: true}, + {name: "name not a string", builder: newTestGGUFBuilder().withUint32KV("general.name", 1), wantErr: true}, + {name: "license not a string", builder: newTestGGUFBuilder().withUint32KV("general.license", 1), wantErr: true}, + {name: "type not a string", builder: newTestGGUFBuilder().withUint32KV("general.type", 1), wantErr: true}, + {name: "author not a string", builder: newTestGGUFBuilder().withUint32KV("general.author", 1), wantErr: true}, + {name: "url not a string", builder: newTestGGUFBuilder().withUint32KV("general.url", 1), wantErr: true}, + {name: "description not a string", builder: newTestGGUFBuilder().withUint32KV("general.description", 1), wantErr: true}, + {name: "model hint not a string", builder: newTestGGUFBuilder().withUint32KV("controlvector.model_hint", 1), wantErr: true}, + {name: "quantization version is a string", builder: newTestGGUFBuilder().withStringKV("general.quantization_version", "2"), wantErr: true}, + {name: "alignment is a string", builder: newTestGGUFBuilder().withStringKV("general.alignment", "32"), wantErr: true}, + {name: "alignment is uint64", builder: newTestGGUFBuilder().withUint64KV("general.alignment", 32), wantErr: true}, + {name: "alignment is zero", builder: newTestGGUFBuilder().withUint32KV("general.alignment", 0), wantErr: true}, + { + name: "array of arrays on a skipped key", + builder: newTestGGUFBuilder().withRawKV("tokenizer.ggml.merges", ggufTypeArray, ggufArray(ggufTypeArray, 1, ggufArray(ggufTypeUint8, 0, nil))), + wantErr: true, + }, + { + name: "invalid array item type", + builder: newTestGGUFBuilder().withRawKV("tokenizer.ggml.scores", ggufTypeArray, ggufArray(99, 0, nil)), + wantErr: true, + }, + {name: "split tensor count out of range", builder: newTestGGUFBuilder().withUint64KV("split.tensors.count", 1<<62), wantErr: true}, + {name: "split tensor count huge", builder: newTestGGUFBuilder().withUint64KV("split.tensors.count", 1<<40), wantErr: true}, + {name: "split tensor count negative", builder: newTestGGUFBuilder().withRawKV("split.tensors.count", ggufTypeInt64, le(int64(-1))), wantErr: true}, + { + name: "nesting too deep on a decoded key", + builder: newTestGGUFBuilder().withRawKV("llama.attention.head_count", ggufTypeArray, nested(3)), + wantErr: true, + }, + { + name: "decoded array too long", + builder: newTestGGUFBuilder().withRawKV("llama.feed_forward_length", ggufTypeArray, ggufArray(ggufTypeUint8, maxGGUFDecodedItems+1, make([]byte, maxGGUFDecodedItems+1))), + wantErr: true, + }, + {name: "empty string value", builder: newTestGGUFBuilder().withStringKV("general.author", ""), wantErr: true}, + {name: "padded key", builder: newTestGGUFBuilder().withUint32KV(" general.alignment\t", 0), wantErr: true}, + {name: "long padded key", builder: newTestGGUFBuilder().withUint32KV(strings.Repeat(" ", 300)+"general.alignment", 0), wantErr: true}, + {name: "file type is a string", builder: newTestGGUFBuilder().withUint32KV("general.file_type", 1).withStringKV("general.file_type", "x"), wantErr: true}, + { + name: "decoded arrays too long in total", + builder: newTestGGUFBuilder(). + withRawKV("a.feed_forward_length", ggufTypeArray, ggufArray(ggufTypeUint8, maxGGUFDecodedItems, make([]byte, maxGGUFDecodedItems))). + withRawKV("b.feed_forward_length", ggufTypeArray, ggufArray(ggufTypeUint8, 1, make([]byte, 1))), + wantErr: true, + }, + {name: "invalid value type", builder: newTestGGUFBuilder().withRawKV("x", 0xFFFFFFFF, nil), wantErr: true}, + {name: "empty key", builder: newTestGGUFBuilder().withUint32KV("", 1), wantErr: true}, + { + name: "empty string in a decoded array", + builder: newTestGGUFBuilder().withRawKV("llama.attention.head_count", ggufTypeArray, ggufArray(ggufTypeString, 1, le(uint64(0)))), + wantErr: true, + }, + { + name: "empty string in a skipped array", + builder: newTestGGUFBuilder().withRawKV("tokenizer.ggml.tokens", ggufTypeArray, ggufArray(ggufTypeString, 1, le(uint64(0)))), + }, + { + name: "array of arrays on a decoded key", + builder: newTestGGUFBuilder().withRawKV("llama.attention.head_count", ggufTypeArray, nested(2)), + }, + {name: "alignment is a uint32", builder: newTestGGUFBuilder().withUint32KV("general.alignment", 32)}, + { + name: "split keys as written by llama.cpp", + builder: newTestGGUFBuilder(). + withRawKV("split.no", ggufTypeUint16, le(uint16(0))). + withRawKV("split.count", ggufTypeUint16, le(uint16(3))). + withRawKV("split.tensors.count", ggufTypeInt32, le(int32(3000))), + }, + { + name: "split shard declares the total tensor count", + builder: newTestGGUFBuilder().withRawKV("split.tensors.count", ggufTypeInt32, le(int32(3000))), + }, + { + name: "skipped scalar array", + builder: newTestGGUFBuilder().withRawKV("tokenizer.ggml.scores", ggufTypeArray, ggufArray(ggufTypeFloat32, 3, make([]byte, 12))), + }, + // ponytail: zero block size tensor types and dimension overflow panic inside tensor info parsing; + // those are left to the generic cataloger's panic handling rather than checked here. + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + data := tt.data + if data == nil { + data = tt.builder.withStringKV("general.architecture", "llama").build() + } + pkgs, err := parseGGUFBytes(t, data) + if tt.wantErr { + // the pre-scan has to be what rejects these, before the library sees them + require.ErrorContains(t, err, "invalid GGUF header") + assert.Empty(t, pkgs) + return + } + require.NoError(t, err) + require.Len(t, pkgs, 1) + }) + } +} + +func TestParseGGUFModel_nonFiniteFloat(t *testing.T) { + data := newTestGGUFBuilder(). + withStringKV("general.architecture", "llama"). + withRawKV("llama.rope.freq_base", ggufTypeFloat32, le(float32(math.Inf(1)))). + withRawKV("llama.rope.scale", ggufTypeFloat64, le(math.Inf(-1))). + withRawKV("llama.attention.head_count", ggufTypeArray, ggufArray(ggufTypeFloat32, 2, le([]float32{8, float32(math.NaN())}))). + withRawKV("tokenizer.ggml.scores", ggufTypeArray, ggufArray(ggufTypeFloat32, 1, le(float32(0)))). + build() + + pkgs, err := parseGGUFBytes(t, data) + require.NoError(t, err) + require.Len(t, pkgs, 1) + + meta := pkgs[0].Metadata.(pkg.GGUFFileHeader) + assert.NotEmpty(t, meta.MetadataKeyValuesHash) + assert.Equal(t, "+Inf", meta.RemainingKeyValues["llama.rope.freq_base"]) + assert.Equal(t, "-Inf", meta.RemainingKeyValues["llama.rope.scale"]) + headCount := meta.RemainingKeyValues["llama.attention.head_count"].(gguf_parser.GGUFMetadataKVArrayValue) + assert.Equal(t, []any{float32(8), "NaN"}, headCount.Array) + + var buf bytes.Buffer + s := sbom.SBOM{Artifacts: sbom.Artifacts{Packages: pkg.NewCollection(pkgs...)}} + require.NoError(t, syftjson.NewFormatEncoder().Encode(&buf, s)) + assert.NotEmpty(t, buf.Bytes()) +} + +// TestParseGGUFModel_outputUnchanged pins the header output and hash for ordinary values to what syft +// produced as of f7f3c6efc, before malformed header handling. The expected values were captured by +// running this same fixture on that commit. float32 0.1 catches any widening to float64. +func TestParseGGUFModel_outputUnchanged(t *testing.T) { + data := newTestGGUFBuilder(). + withStringKV("general.architecture", "llama"). + withRawKV("tokenizer.ggml.tokens", ggufTypeArray, ggufArray(ggufTypeString, 2, append(ggufString("a"), ggufString("b")...))). + withRawKV("llama.attention.head_count", ggufTypeArray, ggufArray(ggufTypeFloat32, 2, le([]float32{0.1, 0.2}))). + withRawKV("llama.feed_forward_length", ggufTypeArray, ggufArray(ggufTypeArray, 2, append( + ggufArray(ggufTypeUint32, 2, le([]uint32{1, 2})), + ggufArray(ggufTypeUint32, 1, le([]uint32{3}))...))). + withRawKV("llama.rope.freq_base", ggufTypeFloat32, le(float32(0.1))). + build() + + pkgs, err := parseGGUFBytes(t, data) + require.NoError(t, err) + require.Len(t, pkgs, 1) + + meta := pkgs[0].Metadata.(pkg.GGUFFileHeader) + assert.Equal(t, "4f28d55f0985a985", meta.MetadataKeyValuesHash) + header, err := json.Marshal(meta.RemainingKeyValues) + require.NoError(t, err) + assert.JSONEq(t, `{ + "llama.attention.head_count": {"type": 6, "len": 2, "array": [0.1, 0.2], "startOffset": 170, "size": 8}, + "llama.feed_forward_length": {"type": 9, "len": 2, "array": [ + {"type": 4, "len": 2, "array": [1, 2], "startOffset": 239, "size": 8}, + {"type": 4, "len": 1, "array": [3], "startOffset": 259, "size": 4} + ], "startOffset": 227, "size": 36}, + "llama.rope.freq_base": 0.1, + "tokenizer.ggml.tokens": {"type": 8, "len": 2, "startOffset": 102, "size": 18} + }`, string(header)) +} + +// TestCopyHeader_appliesItsOwnBound pins that copyHeader caps the copy at maxHeaderSize itself, +// whatever reader it is handed. +// +// Bytes retained is asserted directly rather than through testutils.MeasureAlloc: the destination is a +// bytes.Buffer, whose doubling growth costs a multiple of what it holds and lands on a different power +// of two depending on the platform. buf.Len() is the same property without that noise. +func TestCopyHeader_appliesItsOwnBound(t *testing.T) { + const payload = 2*maxHeaderSize + 1 + + file := make([]byte, 24, 24+payload) + binary.LittleEndian.PutUint32(file[0:4], ggufMagicNumber) + file = append(file, bytes.Repeat([]byte{'x'}, payload)...) + + var buf bytes.Buffer + require.NoError(t, copyHeader(&buf, bytes.NewReader(file))) + assert.Equal(t, maxHeaderSize, buf.Len(), "copyHeader must cap the copy at maxHeaderSize") +} diff --git a/syft/pkg/cataloger/ai/parse_safetensors.go b/syft/pkg/cataloger/ai/parse_safetensors.go index bc1ff1402..2ee3f206b 100644 --- a/syft/pkg/cataloger/ai/parse_safetensors.go +++ b/syft/pkg/cataloger/ai/parse_safetensors.go @@ -5,6 +5,8 @@ import ( "encoding/json" "fmt" "io" + "math" + "math/bits" "sort" "strings" @@ -15,9 +17,7 @@ import ( // SafeTensors file format: [8 bytes u64 LE header size] [N bytes JSON header] [tensor data]. // Reference: https://github.com/huggingface/safetensors#format -const ( - maxSafeTensorsHeaderSize = 100 * 1024 * 1024 // 100MB ceiling on header JSON to prevent OOM -) +// The header JSON is capped at maxSafeTensorsHeaderSize (see limits.go). // safeTensorsHeader is the decoded JSON header. Tensor entries live alongside a // reserved "__metadata__" key holding a string-to-string producer map. We decode @@ -84,47 +84,67 @@ func readSafeTensorsHeader(r io.Reader) (*safeTensorsHeader, error) { return h, nil } -// parameterCount sums the element counts across all tensors in the header. -func (h *safeTensorsHeader) parameterCount() uint64 { - var total uint64 - for _, t := range h.tensors { - count := uint64(1) - for _, dim := range t.Shape { - if dim <= 0 { - count = 0 - break - } - count *= uint64(dim) - } - total += count +// modelInfo returns the metadata derivable from the header bytes alone. Naming, +// licenses and ShardCount are left to the merge processor. +func (h *safeTensorsHeader) modelInfo() pkg.SafeTensorsModelInfo { + params, dtype := h.parameterStats() + return pkg.SafeTensorsModelInfo{ + Format: "safetensors", + TensorCount: uint64(len(h.tensors)), + Parameters: params, + Quantization: normalizeDType(dtype), + UserMetadata: userMetadataKeyValues(h.metadata), + MetadataHash: h.metadataHash(), } - return total } -// dominantDType returns the dtype that accounts for the largest fraction of parameters. -// For mixed-precision models the "dominant" dtype is still a useful summary. -func (h *safeTensorsHeader) dominantDType() string { +// parameterStats sums the element counts across all tensors and returns the +// dtype that accounts for the largest share of them. For mixed-precision models +// the "dominant" dtype is still a useful summary. Totals saturate rather than wrap. +func (h *safeTensorsHeader) parameterStats() (total uint64, dominantDType string) { sizeByDType := make(map[string]uint64) for _, t := range h.tensors { - count := uint64(1) - for _, dim := range t.Shape { - if dim <= 0 { - count = 0 - break - } - count *= uint64(dim) + count, ok := tensorElements(t.Shape) + if !ok { + continue } - sizeByDType[t.DType] += count + total = saturatingAdd(total, count) + sizeByDType[t.DType] = saturatingAdd(sizeByDType[t.DType], count) } - var best string var bestSize uint64 for dtype, size := range sizeByDType { - if size > bestSize || (size == bestSize && dtype < best) { - best = dtype + if size > bestSize || (size == bestSize && dtype < dominantDType) { + dominantDType = dtype bestSize = size } } - return best + return total, dominantDType +} + +// tensorElements returns the element count for a shape, or ok=false on a +// non-positive dim or when the product overflows uint64. +func tensorElements(shape []int64) (uint64, bool) { + n := uint64(1) + for _, d := range shape { + if d <= 0 { + return 0, false + } + hi, lo := bits.Mul64(n, uint64(d)) + if hi != 0 { + return 0, false + } + n = lo + } + return n, true +} + +// saturatingAdd returns a+b, clamped to math.MaxUint64 instead of wrapping. +func saturatingAdd(a, b uint64) uint64 { + sum, carry := bits.Add64(a, b, 0) + if carry != 0 { + return math.MaxUint64 + } + return sum } // metadataHash returns a stable xxhash64 over the logical tensor content diff --git a/syft/pkg/cataloger/ai/parse_safetensors_model.go b/syft/pkg/cataloger/ai/parse_safetensors_model.go index a9a8f3c66..da3234240 100644 --- a/syft/pkg/cataloger/ai/parse_safetensors_model.go +++ b/syft/pkg/cataloger/ai/parse_safetensors_model.go @@ -6,7 +6,6 @@ import ( "io" "github.com/anchore/syft/internal" - "github.com/anchore/syft/internal/unknown" "github.com/anchore/syft/syft/artifact" "github.com/anchore/syft/syft/file" "github.com/anchore/syft/syft/pkg" @@ -26,22 +25,13 @@ func parseSafeTensorsFile(_ context.Context, _ file.Resolver, _ *generic.Environ return nil, nil, fmt.Errorf("failed to read safetensors header: %w", err) } - // ShardCount is intentionally not set here: the merge processor is the single - // owner of ShardCount and derives it from the number of shards in the group. - md := pkg.SafeTensorsModelInfo{ - Format: "safetensors", - TensorCount: uint64(len(header.tensors)), - Parameters: header.parameterCount(), - Quantization: normalizeDType(header.dominantDType()), - UserMetadata: userMetadataKeyValues(header.metadata), - MetadataHash: header.metadataHash(), - } + md := header.modelInfo() p := newSafeTensorsPackage( &md, reader.WithAnnotation(pkg.EvidenceAnnotationKey, pkg.PrimaryEvidenceAnnotation), ) - return []pkg.Package{p}, nil, unknown.IfEmptyf([]pkg.Package{p}, "unable to parse safetensors file") + return []pkg.Package{p}, nil, nil } // integrity check diff --git a/syft/pkg/cataloger/ai/parse_safetensors_oci.go b/syft/pkg/cataloger/ai/parse_safetensors_oci.go index 7fd9f3b3e..8aa04e5b0 100644 --- a/syft/pkg/cataloger/ai/parse_safetensors_oci.go +++ b/syft/pkg/cataloger/ai/parse_safetensors_oci.go @@ -8,7 +8,6 @@ import ( "strings" "github.com/anchore/syft/internal" - "github.com/anchore/syft/internal/unknown" "github.com/anchore/syft/syft/artifact" "github.com/anchore/syft/syft/file" "github.com/anchore/syft/syft/pkg" @@ -50,7 +49,7 @@ type dockerAIModelConfig struct { func parseSafeTensorsOCIConfig(_ context.Context, _ file.Resolver, _ *generic.Environment, reader file.LocationReadCloser) ([]pkg.Package, []artifact.Relationship, error) { defer internal.CloseAndLogError(reader, reader.Path()) - body, err := io.ReadAll(io.LimitReader(reader, 1024*1024)) + body, err := readBounded(reader, maxModelConfigBlobSize) if err != nil { return nil, nil, fmt.Errorf("failed to read docker AI model config: %w", err) } @@ -81,7 +80,7 @@ func parseSafeTensorsOCIConfig(_ context.Context, _ file.Resolver, _ *generic.En &md, reader.WithAnnotation(pkg.EvidenceAnnotationKey, pkg.PrimaryEvidenceAnnotation), ) - return []pkg.Package{p}, nil, unknown.IfEmptyf([]pkg.Package{p}, "unable to parse docker AI safetensors config") + return []pkg.Package{p}, nil, nil } // parseSafeTensorsOCILayer decodes the JSON header of a SafeTensors weight @@ -94,14 +93,7 @@ func parseSafeTensorsOCILayer(_ context.Context, _ file.Resolver, _ *generic.Env return nil, nil, fmt.Errorf("failed to read safetensors layer header: %w", err) } - md := pkg.SafeTensorsModelInfo{ - Format: "safetensors", - TensorCount: uint64(len(header.tensors)), - Parameters: header.parameterCount(), - Quantization: normalizeDType(header.dominantDType()), - UserMetadata: userMetadataKeyValues(header.metadata), - MetadataHash: header.metadataHash(), - } + md := header.modelInfo() p := newSafeTensorsPackage( &md, diff --git a/syft/pkg/cataloger/ai/parse_safetensors_test.go b/syft/pkg/cataloger/ai/parse_safetensors_test.go index c210bfbe0..c5d4457cb 100644 --- a/syft/pkg/cataloger/ai/parse_safetensors_test.go +++ b/syft/pkg/cataloger/ai/parse_safetensors_test.go @@ -6,8 +6,11 @@ import ( "encoding/binary" "encoding/json" "io" + "math" "os" "path/filepath" + "slices" + "strings" "testing" "github.com/stretchr/testify/assert" @@ -48,7 +51,7 @@ func buildSafeTensorsFile(t *testing.T, metadata map[string]string, tensors map[ // // The naming precedence (owned by the merge processor's pickSafeTensorsName) is: // 1. config.json _name_or_path (path.Base applied), found beside the model or -// by walking up parent directories to the scan root +// in its parent directory // 2. otherwise the model's immediate parent directory base name // → drop (no package) when neither yields a usable name // @@ -231,9 +234,9 @@ func TestSafeTensorsCataloger(t *testing.T) { expectedPackages: nil, }, { - // rung 1 via parent-walk: findDirHFConfig walks up from the model - // directory, so a config.json in an ancestor names a nested model. - name: "config.json in an ancestor directory names a nested model", + // rung 1 via the parent: findDirHFConfig also checks the model + // directory's parent, so a config.json there names a nested model. + name: "config.json in the parent directory names a nested model", setup: func(t *testing.T) string { dir := t.TempDir() model(t, filepath.Join(dir, "dir", "someothertensor")) @@ -248,6 +251,87 @@ func TestSafeTensorsCataloger(t *testing.T) { }, }, }, + { + // the walk stops at the parent: a config.json two levels up is not used. + name: "config.json two levels up is not used", + setup: func(t *testing.T) string { + dir := t.TempDir() + model(t, filepath.Join(dir, "a", "b", "someothertensor")) + writeFile(t, filepath.Join(dir, "a", "config.json"), `{"_name_or_path":"org/TooFar"}`) + return dir + }, + expectedPackages: []pkg.Package{ + { + Name: "someothertensor", + Type: pkg.ModelPkg, + Metadata: wantMetadata(""), + }, + }, + }, + { + // JSON without any HF field is not treated as a model config, so the + // lookup keeps going to the parent. + name: "non-HF config.json is skipped", + setup: func(t *testing.T) string { + dir := t.TempDir() + modelDir := filepath.Join(dir, "app", "sometensor") + model(t, modelDir) + writeFile(t, filepath.Join(modelDir, "config.json"), `{"port":8080}`) + writeFile(t, filepath.Join(dir, "app", "config.json"), `{"_name_or_path":"org/Parent"}`) + return dir + }, + expectedPackages: []pkg.Package{ + { + Name: "Parent", + Type: pkg.ModelPkg, + Metadata: wantMetadata(""), + }, + }, + }, + { + // an oversized config.json is skipped entirely, while a large README + // still yields its frontmatter (only a prefix is read). + name: "oversized config.json is ignored and a large README still names the model", + setup: func(t *testing.T) string { + dir := t.TempDir() + modelDir := filepath.Join(dir, "sometensor") + model(t, modelDir) + writeFile(t, filepath.Join(modelDir, "config.json"), + `{"architectures":["X"],"_name_or_path":"`+strings.Repeat("A", maxHFConfigSize)+`"}`) + writeFile(t, filepath.Join(modelDir, "README.md"), + "---\nlicense: MIT\nbase_model: org/Readme\n---\n"+strings.Repeat("A", maxReadmePrefixSize)) + return dir + }, + expectedPackages: []pkg.Package{ + { + Name: "Readme", + Type: pkg.ModelPkg, + Licenses: pkg.NewLicenseSet( + pkg.NewLicenseFromFields("MIT", "", nil), + ), + Metadata: wantMetadata(""), + }, + }, + }, + { + // a _name_or_path that sanitizes to nothing doesn't block README base_model. + name: "degenerate _name_or_path falls through to README base_model", + setup: func(t *testing.T) string { + dir := t.TempDir() + modelDir := filepath.Join(dir, "sometensor") + model(t, modelDir) + writeFile(t, filepath.Join(modelDir, "config.json"), `{"_name_or_path":"."}`) + writeFile(t, filepath.Join(modelDir, "README.md"), "---\nbase_model: org/Readme\n---\n") + return dir + }, + expectedPackages: []pkg.Package{ + { + Name: "Readme", + Type: pkg.ModelPkg, + Metadata: wantMetadata(""), + }, + }, + }, { // grouping: independent models in one scan are grouped by their own // parent directory and each named from it. @@ -601,6 +685,60 @@ spdx-id: Apache-2.0 assertHasLicense(t, out[0], "Apache-2.0") }) + t.Run("OCI: config precedence is stable across layer order", func(t *testing.T) { + dir := t.TempDir() + a := filepath.Join(dir, "a.json") + b := filepath.Join(dir, "b.json") + gen := filepath.Join(dir, "generation_config.json") + require.NoError(t, os.WriteFile(a, []byte(`{"_name_or_path":"org/first"}`), 0o644)) + require.NoError(t, os.WriteFile(b, []byte(`{"_name_or_path":"org/second"}`), 0o644)) + require.NoError(t, os.WriteFile(gen, []byte(`{"max_new_tokens":10}`), 0o644)) + + configMd := pkg.SafeTensorsModelInfo{Format: "safetensors", TensorCount: 1} + for _, order := range [][]string{{a, b, gen}, {gen, b, a}} { + var locs []file.Location + for _, p := range order { + locs = append(locs, file.NewLocation(p)) + } + resolver := file.NewMockResolverForMediaTypes(map[string][]file.Location{ + dockerAIModelFileMediaType: locs, + }) + out, _, err := safeTensorsMergeProcessor(context.Background(), resolver, []pkg.Package{ociPkg(configMd)}, nil, nil) + require.NoError(t, err) + require.Len(t, out, 1) + assert.Equal(t, "first", out[0].Name) + // the non-HF generation config is not recorded as supporting evidence + assert.Len(t, out[0].Locations.ToSlice(), 3) + } + }) + + t.Run("dir scan: non-HF config.json in the parent is not supporting evidence", func(t *testing.T) { + dir := t.TempDir() + cfgPath := filepath.Join(dir, "config.json") + require.NoError(t, os.WriteFile(cfgPath, []byte(`{"port":8080}`), 0o644)) + loc, cfg := findDirHFConfig(file.NewMockResolverForPaths(cfgPath), filepath.Join(dir, "model")) + assert.Nil(t, loc) + assert.Nil(t, cfg) + }) + + t.Run("OCI: a large license layer still yields its frontmatter spdx-id", func(t *testing.T) { + dir := t.TempDir() + licensePath := filepath.Join(dir, "LICENSE") + require.NoError(t, os.WriteFile(licensePath, + []byte("---\nspdx-id: Apache-2.0\n---\n"+strings.Repeat("A", maxLicensePrefixSize)), 0o644)) + assert.Equal(t, "Apache-2.0", readLicenseSPDXIDFromFrontmatter(file.NewMockResolverForPaths(licensePath), file.NewLocation(licensePath))) + }) + + t.Run("OCI: oversized model.file layer is rejected", func(t *testing.T) { + dir := t.TempDir() + cfgPath := filepath.Join(dir, "config.json") + require.NoError(t, os.WriteFile(cfgPath, + []byte(`{"_name_or_path":"org/x","pad":"`+strings.Repeat("A", maxModelFileLayerSize)+`"}`), 0o644)) + cfg, fm := classifyOCIModelFileLayer(file.NewMockResolverForPaths(cfgPath), file.NewLocation(cfgPath)) + assert.Nil(t, cfg) + assert.Nil(t, fm) + }) + t.Run("names packages and still propagates an upstream error", func(t *testing.T) { // A non-nil upstream error (e.g. another file in the run failed to parse) // must NOT short-circuit naming. The package is still nameless at this @@ -622,7 +760,7 @@ spdx-id: Apache-2.0 // (or neither, asserting the drop path). // // Precedence (highest → lowest): -// 1. config.json _name_or_path (path.Base applied; both dir-scan and OCI) +// 1. config.json _name_or_path (sanitized to its last path element; both dir-scan and OCI) // 2. fallback name — OCI image-ref last segment, or dir-scan parent directory // base name (the merge processor computes the right one per group) // → drop (empty name) when nothing matches @@ -663,6 +801,31 @@ func TestSafeTensorsNamingPrecedence(t *testing.T) { want: "tiny-llama", }, + // sanitizing + { + name: "windows-style path uses the last element", + nameOrPath: `C:\models\win-model`, + fallbackName: "fallback-name", + want: "win-model", + }, + { + name: "degenerate path falls back", + nameOrPath: "..", + fallbackName: "fallback-name", + want: "fallback-name", + }, + { + name: "root path falls back", + nameOrPath: "/", + fallbackName: "fallback-name", + want: "fallback-name", + }, + { + name: "long names are truncated", + nameOrPath: "org/" + strings.Repeat("x", maxModelNameLength+10), + want: strings.Repeat("x", maxModelNameLength), + }, + // drops { name: "drop: both rungs empty", @@ -984,8 +1147,9 @@ func TestSafeTensorsHeader_parameterCountAndDType(t *testing.T) { "small": {DType: "F32", Shape: []int64{16, 16}}, "scalar": {DType: "F32", Shape: []int64{}}, // empty shape contributes 1 }} - assert.Equal(t, uint64(1000*16+16*16+1), h.parameterCount()) - assert.Equal(t, "BF16", h.dominantDType()) + params, dtype := h.parameterStats() + assert.Equal(t, uint64(1000*16+16*16+1), params) + assert.Equal(t, "BF16", dtype) } func TestSafeTensorsHeader_parameterCount_nonPositiveDims(t *testing.T) { @@ -997,7 +1161,29 @@ func TestSafeTensorsHeader_parameterCount_nonPositiveDims(t *testing.T) { "zero": {DType: "F32", Shape: []int64{4, 0}}, "negative": {DType: "F32", Shape: []int64{-1, 8}}, }} - assert.Equal(t, uint64(16), h.parameterCount()) + params, _ := h.parameterStats() + assert.Equal(t, uint64(16), params) +} + +func TestSafeTensorsHeader_parameterStats_overflow(t *testing.T) { + // a shape whose product overflows uint64 is skipped instead of wrapping (the + // first tensor would wrap to 0) and can't flip the dominant dtype; the total + // saturates instead of wrapping. + h := &safeTensorsHeader{tensors: map[string]safeTensorsEntry{ + "good": {DType: "BF16", Shape: []int64{4, 4}}, + "wraps": {DType: "F32", Shape: []int64{4294967296, 4294967296}}, + "overflow": {DType: "F32", Shape: []int64{math.MaxInt64, 3}}, + }} + params, dtype := h.parameterStats() + assert.Equal(t, uint64(16), params) + assert.Equal(t, "BF16", dtype) + + h = &safeTensorsHeader{tensors: map[string]safeTensorsEntry{ + "a": {DType: "F32", Shape: []int64{math.MaxInt64, 2}}, + "b": {DType: "F32", Shape: []int64{math.MaxInt64, 2}}, + }} + params, _ = h.parameterStats() + assert.Equal(t, uint64(math.MaxUint64), params) } func TestSafeTensorsHeader_dominantDType_tieBreak(t *testing.T) { @@ -1007,7 +1193,8 @@ func TestSafeTensorsHeader_dominantDType_tieBreak(t *testing.T) { "a": {DType: "F32", Shape: []int64{4, 4}}, "b": {DType: "BF16", Shape: []int64{4, 4}}, }} - assert.Equal(t, "BF16", h.dominantDType()) + _, dtype := h.parameterStats() + assert.Equal(t, "BF16", dtype) } func TestNormalizeDType(t *testing.T) { @@ -1036,25 +1223,78 @@ func TestParseFrontmatter(t *testing.T) { name string input string wantNil bool - wantLicense string + wantLicenses []string wantBaseModel []string }{ { name: "list base_model", input: "---\nlicense: mit\nbase_model:\n - org/Model\n---\nbody", - wantLicense: "mit", + wantLicenses: []string{"mit"}, wantBaseModel: []string{"org/Model"}, }, { name: "scalar base_model", input: "---\nlicense: apache-2.0\nbase_model: org/Model\n---\n", - wantLicense: "apache-2.0", + wantLicenses: []string{"apache-2.0"}, wantBaseModel: []string{"org/Model"}, }, { - name: "leading BOM", - input: "\xef\xbb\xbf---\nlicense: mit\n---\n", - wantLicense: "mit", + name: "list license keeps base_model", + input: "---\nlicense: [mit, apache-2.0]\nbase_model: org/Model\n---\n", + wantLicenses: []string{"mit", "apache-2.0"}, + wantBaseModel: []string{"org/Model"}, + }, + { + name: "mapping-valued fields are ignored, not fatal", + input: "---\nlicense: {name: mit}\nbase_model: org/Model\n---\n", + wantBaseModel: []string{"org/Model"}, + }, + { + name: "leading BOM", + input: "\xef\xbb\xbf---\nlicense: mit\n---\n", + wantLicenses: []string{"mit"}, + }, + { + name: "null license is ignored", + input: "---\nlicense: null\nbase_model: ~\n---\n", + wantLicenses: nil, + wantBaseModel: nil, + }, + { + name: "empty block", + input: "---\n---\n", + wantNil: true, + }, + { + name: "oversized block is rejected", + input: "---\nlicense: mit\n" + strings.Repeat("# pad\n", maxFrontmatterSize/6+1) + "---\n", + wantNil: true, + }, + { + name: "closing delimiter at EOF with trailing whitespace", + input: "---\nlicense: mit\n--- ", + wantLicenses: []string{"mit"}, + }, + { + name: "list values are capped", + input: "---\nlicense: [" + strings.Repeat("mit,", maxFrontmatterValues+5) + "mit]\n---\n", + wantLicenses: slices.Repeat([]string{"mit"}, maxFrontmatterValues), + }, + { + name: "CRLF delimiters", + input: "---\r\nlicense: mit\r\n---\r\nbody", + wantLicenses: []string{"mit"}, + }, + { + name: "opening delimiter must be exactly ---", + input: "----\nlicense: mit\n---\n", + wantNil: true, + }, + { + name: "a ---x line inside the block does not close it", + input: "---\nbase_model: org/Model\n---x: 1\nlicense: mit\n---\n", + wantLicenses: []string{"mit"}, + wantBaseModel: []string{"org/Model"}, }, { name: "no frontmatter", @@ -1066,6 +1306,12 @@ func TestParseFrontmatter(t *testing.T) { input: "---\nlicense: mit\n", wantNil: true, }, + { + // https://github.com/anchore/syft/issues/5327 + name: "malformed frontmatter that panics the yaml decoder", + input: "---\n<<:\r? 0:\n---\nbody", + wantNil: true, + }, } for _, tt := range tests { @@ -1076,7 +1322,7 @@ func TestParseFrontmatter(t *testing.T) { return } require.NotNil(t, fm) - assert.Equal(t, tt.wantLicense, fm.License) + assert.Equal(t, tt.wantLicenses, fm.Licenses) if tt.wantBaseModel != nil { assert.Equal(t, tt.wantBaseModel, fm.BaseModel) } @@ -1157,6 +1403,12 @@ limitations: input: "---\nspdx-id: MIT\n(never closes)\n", want: "", }, + { + // https://github.com/anchore/syft/issues/5327 + name: "malformed frontmatter that panics the yaml decoder", + input: "---\n<<:\r? 0:\n---\nbody\n", + want: "", + }, } for _, tt := range tests { @@ -1233,28 +1485,3 @@ func TestReadSafeTensorsHeader_capBoundsHeap(t *testing.T) { assert.Less(t, bounded, uint64(8*1024*1024), "the declared-size check has to reject before the body is read, not after") } - -// TestCopyHeader_boundIsTheCallersLimitedReader pins where the GGUF bound actually lives. copyHeader -// io.Copy's whatever it is handed, so maxHeaderSize only holds because every caller wraps the reader -// first. A caller that stops doing that reads the whole file, and nothing in copyHeader would object. -// -// Bytes retained is asserted directly rather than through testutils.MeasureAlloc: the destination is a -// bytes.Buffer, whose doubling growth costs a multiple of what it holds and lands on a different power -// of two depending on the platform. buf.Len() is the same property without that noise. -func TestCopyHeader_boundIsTheCallersLimitedReader(t *testing.T) { - const payload = 2*maxHeaderSize + 1 - - file := make([]byte, 24, 24+payload) - binary.LittleEndian.PutUint32(file[0:4], ggufMagicNumber) - file = append(file, bytes.Repeat([]byte{'x'}, payload)...) - - var unbounded bytes.Buffer - require.NoError(t, copyHeader(&unbounded, bytes.NewReader(file))) - require.Equal(t, len(file), unbounded.Len(), - "an unwrapped reader is copied whole; copyHeader applies no limit of its own") - - var bounded bytes.Buffer - require.NoError(t, copyHeader(&bounded, &io.LimitedReader{R: bytes.NewReader(file), N: maxHeaderSize})) - assert.Equal(t, maxHeaderSize, bounded.Len(), - "a wrapped reader must cap the copy at maxHeaderSize") -} diff --git a/syft/pkg/cataloger/ai/processor.go b/syft/pkg/cataloger/ai/processor.go index fd0e29292..5cb31c3e7 100644 --- a/syft/pkg/cataloger/ai/processor.go +++ b/syft/pkg/cataloger/ai/processor.go @@ -104,10 +104,10 @@ func mergeDirModels(ctx context.Context, resolver file.Resolver, pkgs []pkg.Pack // groupByParentDir buckets filesystem-scanned models by the directory their // primary-evidence file lives in. - +// // This encodes a deliberate assumption: a directory holds one logical // model, so every .safetensors file in a directory is treated as a shard of the -// same modeland merged into one package. +// same model and merged into one package. // The trade-off is that if a directory happens to contain several unrelated models, // they are merged into one package rather than reported separately. // We accept that because the conventional on-disk layout gives each model diff --git a/syft/pkg/cataloger/ai/processor_test.go b/syft/pkg/cataloger/ai/processor_test.go index 6d216ae95..e99f9a9ce 100644 --- a/syft/pkg/cataloger/ai/processor_test.go +++ b/syft/pkg/cataloger/ai/processor_test.go @@ -1,6 +1,7 @@ package ai import ( + "errors" "testing" "github.com/stretchr/testify/assert" @@ -61,3 +62,20 @@ func Test_ggufMergeProcessor(t *testing.T) { }) } } + +func Test_ggufMergeProcessor_mergesDespiteError(t *testing.T) { + // one layer failing to parse must not drop the rest of the model + parseErr := errors.New("one layer failed to parse") + pkgs := []pkg.Package{ + {Name: "model", Metadata: pkg.GGUFFileHeader{MetadataKeyValuesHash: "abc"}}, + {Name: "", Metadata: pkg.GGUFFileHeader{MetadataKeyValuesHash: "part1"}}, + } + + got, _, err := ggufMergeProcessor(pkgs, nil, parseErr) + require.ErrorIs(t, err, parseErr) + require.Len(t, got, 1) + assert.Equal(t, "model", got[0].Name) + header, ok := got[0].Metadata.(pkg.GGUFFileHeader) + require.True(t, ok) + assert.Len(t, header.Parts, 1) +} diff --git a/syft/pkg/cataloger/ai/test_helpers_test.go b/syft/pkg/cataloger/ai/test_helpers_test.go index 195e90d92..d11ce8bb1 100644 --- a/syft/pkg/cataloger/ai/test_helpers_test.go +++ b/syft/pkg/cataloger/ai/test_helpers_test.go @@ -72,8 +72,32 @@ func (b *testGGUFBuilder) withUint32KV(key string, value uint32) *testGGUFBuilde return b } +// withRawKV adds a KV whose value bytes are written verbatim, for arrays and malformed values. +func (b *testGGUFBuilder) withRawKV(key string, valueType uint32, raw []byte) *testGGUFBuilder { + b.kvPairs = append(b.kvPairs, testKVPair{key: key, valueType: valueType, value: raw}) + return b +} + +// ggufArray encodes an array value: item type, length, then the already encoded items. +func ggufArray(itemType uint32, n uint64, items []byte) []byte { + buf := new(bytes.Buffer) + binary.Write(buf, binary.LittleEndian, itemType) + binary.Write(buf, binary.LittleEndian, n) + buf.Write(items) + return buf.Bytes() +} + +// writeLen writes a count or length, which is 32-bit in GGUF v1 and 64-bit after. +func (b *testGGUFBuilder) writeLen(n uint64) { + if b.version <= 1 { + binary.Write(b.buf, binary.LittleEndian, uint32(n)) + return + } + binary.Write(b.buf, binary.LittleEndian, n) +} + func (b *testGGUFBuilder) writeString(s string) { - binary.Write(b.buf, binary.LittleEndian, uint64(len(s))) + b.writeLen(uint64(len(s))) b.buf.WriteString(s) } @@ -85,10 +109,10 @@ func (b *testGGUFBuilder) build() []byte { binary.Write(b.buf, binary.LittleEndian, b.version) // Write tensor count - binary.Write(b.buf, binary.LittleEndian, b.tensorCount) + b.writeLen(b.tensorCount) // Write KV count - binary.Write(b.buf, binary.LittleEndian, uint64(len(b.kvPairs))) + b.writeLen(uint64(len(b.kvPairs))) // Write KV pairs for _, kv := range b.kvPairs { @@ -97,6 +121,10 @@ func (b *testGGUFBuilder) build() []byte { // Write value type binary.Write(b.buf, binary.LittleEndian, kv.valueType) // Write value based on type + if raw, ok := kv.value.([]byte); ok { + b.buf.Write(raw) + continue + } switch kv.valueType { case ggufTypeString: b.writeString(kv.value.(string)) diff --git a/syft/pkg/cataloger/ai/testdata/safetensors/extract_header.go b/syft/pkg/cataloger/ai/testdata/safetensors/extract_header.go index 4f7d72a60..8445adcb7 100644 --- a/syft/pkg/cataloger/ai/testdata/safetensors/extract_header.go +++ b/syft/pkg/cataloger/ai/testdata/safetensors/extract_header.go @@ -28,9 +28,9 @@ import ( const ( safetensorsLayerMediaType = "application/vnd.docker.ai.safetensors" - // 8 MB matches maxHeaderBytes in the OCI model source. Real model headers - // are well under 1 MB; the extra slack covers outliers. - fetchBytes = 8 * 1024 * 1024 + // matches maxSafeTensorsHeaderBytes in the OCI model source (8 MB header + // cap plus the 8-byte length prefix). Real model headers are well under 1 MB. + fetchBytes = 8*1024*1024 + 8 ) func main() { diff --git a/syft/pkg/cataloger/alpine/parse_apk_db.go b/syft/pkg/cataloger/alpine/parse_apk_db.go index e661797a1..bf721c68d 100644 --- a/syft/pkg/cataloger/alpine/parse_apk_db.go +++ b/syft/pkg/cataloger/alpine/parse_apk_db.go @@ -270,6 +270,10 @@ func (f apkField) apply(p *parsedData, ctx *apkFileParsingContext) { ctx.indexOfLatestDirectory = len(ctx.files) - 1 case "M": i := ctx.indexOfLatestDirectory + if i < 0 { + log.Debugf("APK field %q appears before any directory entry", f.name) + return + } latest := ctx.files[i] var ok bool @@ -296,6 +300,10 @@ func (f apkField) apply(p *parsedData, ctx *apkFileParsingContext) { ctx.indexOfLatestRegularFile = len(ctx.files) - 1 case "a": i := ctx.indexOfLatestRegularFile + if i < 0 { + log.Debugf("APK field %q appears before any file entry", f.name) + return + } latest := ctx.files[i] var ok bool @@ -309,6 +317,10 @@ func (f apkField) apply(p *parsedData, ctx *apkFileParsingContext) { ctx.files[i] = latest case "Z": i := ctx.indexOfLatestRegularFile + if i < 0 { + log.Debugf("APK field %q appears before any file entry", f.name) + return + } latest := ctx.files[i] latest.Digest = processChecksum(f.value) diff --git a/syft/pkg/cataloger/alpine/parse_apk_db_test.go b/syft/pkg/cataloger/alpine/parse_apk_db_test.go index ee146d723..6ff453af7 100644 --- a/syft/pkg/cataloger/alpine/parse_apk_db_test.go +++ b/syft/pkg/cataloger/alpine/parse_apk_db_test.go @@ -869,3 +869,29 @@ https://foo.them.org/alpine/v3.14/community`, }) } } + +func TestParseApkDB_attributeBeforeItsEntry(t *testing.T) { + // ACL (M:, a:) and checksum (Z:) lines describe the most recent directory or file entry, and used to index + // files[-1] when no entry had been seen yet + tests := []struct { + name string + line string + }{ + {name: "directory ACL before any directory", line: "M:0:0:755"}, + {name: "file ACL before any file", line: "a:0:0:755"}, + {name: "checksum before any file", line: "Z:Q1Kz6TXnPVJeDQUfO0IJVxD3HnYmU="}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + contents := strings.Join([]string{"P:foo", "V:1.0-r0", tt.line, ""}, "\n") + reader := file.NewLocationReadCloser(file.NewLocation("installed"), io.NopCloser(strings.NewReader(contents))) + + pkgs, _, err := parseApkDB(context.Background(), nil, new(generic.Environment), reader) + + require.NoError(t, err) + require.Len(t, pkgs, 1) + assert.Equal(t, "foo", pkgs[0].Name) + assert.Empty(t, pkgs[0].Metadata.(pkg.ApkDBEntry).Files, "an orphaned attribute line should not invent a file") + }) + } +} diff --git a/syft/pkg/cataloger/arch/parse_alpm_db.go b/syft/pkg/cataloger/arch/parse_alpm_db.go index fe9762087..db7505be6 100644 --- a/syft/pkg/cataloger/arch/parse_alpm_db.go +++ b/syft/pkg/cataloger/arch/parse_alpm_db.go @@ -148,12 +148,7 @@ func parseAlpmDBEntry(reader io.Reader) (*parsedData, error) { } func newScanner(reader io.Reader) *bufio.Scanner { - // This is taken from the apk parser - // https://github.com/anchore/syft/blob/v0.47.0/syft/pkg/cataloger/apkdb/parse_apk_db.go#L37 - const maxScannerCapacity = 1024 * 1024 - bufScan := make([]byte, maxScannerCapacity) - scanner := bufio.NewScanner(reader) - scanner.Buffer(bufScan, maxScannerCapacity) + scanner := internal.NewLineScanner(reader) onDoubleLF := func(data []byte, atEOF bool) (advance int, token []byte, err error) { for i := range data { if i > 0 && data[i-1] == '\n' && data[i] == '\n' { diff --git a/syft/pkg/cataloger/arch/parse_alpm_db_test.go b/syft/pkg/cataloger/arch/parse_alpm_db_test.go index 6b26209e8..c147fa330 100644 --- a/syft/pkg/cataloger/arch/parse_alpm_db_test.go +++ b/syft/pkg/cataloger/arch/parse_alpm_db_test.go @@ -492,12 +492,16 @@ func Test_parseMtree_malformedInput(t *testing.T) { // bytes still costs multiples of its own size in heap. Asserting the error would keep passing if the // line cap were removed, because the byte cap would still reject a large enough input eventually. func Test_parseMtree_lineCapBoundsHeap(t *testing.T) { - // newline-only lines are the worst case: minimum bytes per entry, so the most entries per byte - const payloadSize = 16 * intFile.MB - spec := bytes.Repeat([]byte{'\n'}, payloadSize) + // the property is linear in the cap, so it is measured at a small cap and scaled to the shipped + // one. Running it at 300k lines means an uncapped parse costing ~16GB, which under -race is most + // of a CI minute for a test with no concurrency in it. + const ( + lineCap = 4096 + payloadSize = 64 * lineCap + ) - // well inside the byte cap, so only the line cap can stop this - require.Less(t, int64(payloadSize), int64(maxMtreeSize)) + // newline-only lines are the worst case: minimum bytes per entry, so the most entries per byte + spec := bytes.Repeat([]byte{'\n'}, payloadSize) unbounded := testutils.MeasureAlloc(t, func() { // the same listing with the line cap lifted, to prove the fixture is really a bomb @@ -507,19 +511,29 @@ func Test_parseMtree_lineCapBoundsHeap(t *testing.T) { require.Greater(t, unbounded, uint64(payloadSize), "fixture did not actually cost more than its own size; it is no longer a bomb") - bounded := testutils.MeasureAlloc(t, func() { - _, err := parseMtreeWithLimits(gzipOf(t, spec), maxMtreeSize, maxMtreeLines) - require.ErrorIs(t, err, errTooManyMtreeLines) - }) + boundedAt := func(limit int) uint64 { + gz := gzipOf(t, spec) + return testutils.MeasureAlloc(t, func() { + _, err := parseMtreeWithLimits(gz, maxMtreeSize, limit) + require.ErrorIs(t, err, errTooManyMtreeLines) + }) + } + bounded := boundedAt(lineCap) - t.Logf("unbounded allocated %d bytes, bounded allocated %d bytes", unbounded, bounded) + // the marginal cost between two caps, so fixed setup (gzip reader, parser) does not get scaled + perLine := (boundedAt(2*lineCap) - bounded) / lineCap + atShippedCap := perLine * maxMtreeLines + t.Logf("unbounded allocated %d bytes, bounded allocated %d bytes (%d per line, ~%d at the shipped cap)", + unbounded, bounded, perLine, atShippedCap) - // measured at ~238MB against ~16.5GB uncapped, so the cap is doing its job, but note what it - // actually buys: 300k entries still cost a few hundred MB, and that is per concurrent cataloger. - // The budget is set to catch the cap being removed or raised by an order of magnitude, not to pin - // the exact figure. - assert.Less(t, bounded, uint64(512*intFile.MB), + assert.Less(t, bounded, unbounded/10, "the line cap has to stop the parse before it builds an entry per line") + + // measured at ~238MB for the shipped 300k lines, so the cap is doing its job, but note what it + // actually buys: that is still a few hundred MB, per concurrent cataloger. The budget is set to + // catch the cap being raised by an order of magnitude, not to pin the exact figure. + assert.Less(t, atShippedCap, uint64(512*intFile.MB), + "the shipped line cap allows more heap than budgeted") } // Test_parseMtree_sizeCapBoundsHeap is the byte-cap counterpart to the line-cap budget above. @@ -532,9 +546,12 @@ func Test_parseMtree_lineCapBoundsHeap(t *testing.T) { // be ordinary for the byte cap to be the thing under test, and there have to be few enough of them // that the line cap does not fire first. func Test_parseMtree_sizeCapBoundsHeap(t *testing.T) { + // same reasoning as the line cap: measured at a small byte cap, since the shipped 64MB cap means + // gzipping and parsing ~200MB of fixture, which is slow under -race and proves nothing more. const ( + sizeCap = intFile.MB lineLen = 1024 - numLines = 96 * 1024 // ~96MB across ~98k lines: past the 64MB cap, well under the 300k line cap + numLines = 2 * sizeCap / lineLen // twice the cap, and far under the line cap ) var raw bytes.Buffer @@ -542,7 +559,7 @@ func Test_parseMtree_sizeCapBoundsHeap(t *testing.T) { for range numLines { raw.Write(line) } - require.Greater(t, int64(raw.Len()), int64(maxMtreeSize), "fixture must exceed the byte cap") + require.Greater(t, int64(raw.Len()), int64(sizeCap), "fixture must exceed the byte cap") require.Less(t, numLines, maxMtreeLines, "and must not trip the line cap first") unbounded := testutils.MeasureAlloc(t, func() { @@ -554,11 +571,13 @@ func Test_parseMtree_sizeCapBoundsHeap(t *testing.T) { "fixture did not actually cost more than its own size; it is no longer a bomb") bounded := testutils.MeasureAlloc(t, func() { - _, err := parseMtreeWithLimits(gzipOf(t, raw.Bytes()), maxMtreeSize, maxMtreeLines) + _, err := parseMtreeWithLimits(gzipOf(t, raw.Bytes()), sizeCap, maxMtreeLines) require.ErrorIs(t, err, errMtreeTooLarge) }) t.Logf("unbounded allocated %d bytes, bounded allocated %d bytes", unbounded, bounded) - assert.Less(t, bounded, uint64(512*intFile.MB), + + // the old budget was 512MB against the 64MB cap, so 8x the cap + assert.Less(t, bounded, uint64(8*sizeCap), "the size cap has to stop the read at the cap, not after the member is drained") } diff --git a/syft/pkg/cataloger/binary/classifier_cataloger_test.go b/syft/pkg/cataloger/binary/classifier_cataloger_test.go index 4b4e2ed7d..54cd04f3b 100644 --- a/syft/pkg/cataloger/binary/classifier_cataloger_test.go +++ b/syft/pkg/cataloger/binary/classifier_cataloger_test.go @@ -407,6 +407,30 @@ func Test_Cataloger_PositiveCases(t *testing.T) { Metadata: metadata("traefik-binary"), }, }, + { + // regression test: big-endian architectures (e.g. s390x) previously matched an unrelated + // single-NUL-terminated numeric string elsewhere in the binary instead of the real version + logicalFixture: "traefik/3.6.8/linux-s390x", + expected: pkg.Package{ + Name: "traefik", + Version: "3.6.8", + Type: "binary", + PURL: "pkg:generic/traefik@3.6.8", + Locations: locations("traefik"), + Metadata: metadata("traefik-binary"), + }, + }, + { + logicalFixture: "traefik/2.11.1/linux-arm", + expected: pkg.Package{ + Name: "traefik", + Version: "2.11.1", + Type: "binary", + PURL: "pkg:generic/traefik@2.11.1", + Locations: locations("traefik"), + Metadata: metadata("traefik-binary"), + }, + }, { logicalFixture: "memcached/1.6.18/linux-amd64", expected: pkg.Package{ diff --git a/syft/pkg/cataloger/binary/classifiers.go b/syft/pkg/cataloger/binary/classifiers.go index 17834ab0c..26929ae2a 100644 --- a/syft/pkg/cataloger/binary/classifiers.go +++ b/syft/pkg/cataloger/binary/classifiers.go @@ -284,11 +284,18 @@ func DefaultClassifiers() []binutils.Classifier { { Class: "traefik-binary", FileGlob: "**/traefik", - EvidenceMatcher: m.FileContentsVersionMatcher( + EvidenceMatcher: binutils.MatchAny( + // on some architectures (e.g. s390x) the real version string isn't NUL-prefixed + // and is instead terminated by two NUL bytes; checking this pattern first avoids + // matching an unrelated single-NUL-terminated numeric string elsewhere in the binary + m.FileContentsVersionMatcher( + `(?m)(?P[0-9]+\.[0-9]+\.[0-9]+(-alpha[0-9]|-beta[0-9]|-rc[0-9])?)\x00\x00`), // [NUL]v1.7.34[NUL] // [NUL]2.9.6[NUL] // 3.0.4[NUL] - `(?m)(\x00v?|\x{FFFD}.?)(?P[0-9]+\.[0-9]+\.[0-9]+(-alpha[0-9]|-beta[0-9]|-rc[0-9])?)\x00`), + m.FileContentsVersionMatcher( + `(?m)(\x00v?|\x{FFFD}.?)(?P[0-9]+\.[0-9]+\.[0-9]+(-alpha[0-9]|-beta[0-9]|-rc[0-9])?)\x00`), + ), Package: "traefik", PURL: mustPURL("pkg:generic/traefik@version"), CPEs: singleCPE("cpe:2.3:a:traefik:traefik:*:*:*:*:*:*:*:*", cpe.NVDDictionaryLookupSource), diff --git a/syft/pkg/cataloger/binary/internal/manager/internal/config/application.go b/syft/pkg/cataloger/binary/internal/manager/internal/config/application.go index d758184dd..fe0bd9a4b 100644 --- a/syft/pkg/cataloger/binary/internal/manager/internal/config/application.go +++ b/syft/pkg/cataloger/binary/internal/manager/internal/config/application.go @@ -102,10 +102,10 @@ func (c Application) Validate() error { } func (c Application) GetBinaryFromImage(name, version string) *BinaryFromImage { - if strings.Contains(name, "@") && version == "" { - parts := strings.Split(name, "@") - name = parts[0] - version = parts[1] + if version == "" { + if n, v, ok := strings.Cut(name, "@"); ok { + name, version = n, v + } } for _, entry := range c.FromImages { if entry.Name() == name && entry.Version == version { diff --git a/syft/pkg/cataloger/binary/testdata/classifiers/snippets/traefik/2.11.1/linux-arm/traefik b/syft/pkg/cataloger/binary/testdata/classifiers/snippets/traefik/2.11.1/linux-arm/traefik new file mode 100644 index 000000000..f0be458f0 Binary files /dev/null and b/syft/pkg/cataloger/binary/testdata/classifiers/snippets/traefik/2.11.1/linux-arm/traefik differ diff --git a/syft/pkg/cataloger/binary/testdata/classifiers/snippets/traefik/3.6.8/linux-s390x/traefik b/syft/pkg/cataloger/binary/testdata/classifiers/snippets/traefik/3.6.8/linux-s390x/traefik new file mode 100644 index 000000000..0141469f6 Binary files /dev/null and b/syft/pkg/cataloger/binary/testdata/classifiers/snippets/traefik/3.6.8/linux-s390x/traefik differ diff --git a/syft/pkg/cataloger/debian/parse_copyright.go b/syft/pkg/cataloger/debian/parse_copyright.go index e6936ce2d..0b5e86647 100644 --- a/syft/pkg/cataloger/debian/parse_copyright.go +++ b/syft/pkg/cataloger/debian/parse_copyright.go @@ -1,7 +1,6 @@ package debian import ( - "bufio" "io" "regexp" "sort" @@ -10,6 +9,7 @@ import ( "github.com/scylladb/go-set/strset" "github.com/anchore/syft/internal" + "github.com/anchore/syft/internal/log" ) // For more information see: https://www.debian.org/doc/packaging-manuals/copyright-format/1.0/#license-syntax @@ -34,7 +34,7 @@ const ( func parseLicensesFromCopyright(reader io.Reader) []string { findings := strset.New() - scanner := bufio.NewScanner(reader) + scanner := internal.NewLineScanner(reader) // per the DEP-5 spec, machine-readable copyright files MUST have a // Format field whose value is a URI for the specification. Only files @@ -69,6 +69,9 @@ func parseLicensesFromCopyright(reader io.Reader) []string { findings.Add(found) } } + if err := scanner.Err(); err != nil { + log.WithFields("error", err).Debug("failed to fully read debian copyright file") + } if !formatVerified { return nil diff --git a/syft/pkg/cataloger/debian/parse_deb_archive_test.go b/syft/pkg/cataloger/debian/parse_deb_archive_test.go index 7afa0a455..8b916a476 100644 --- a/syft/pkg/cataloger/debian/parse_deb_archive_test.go +++ b/syft/pkg/cataloger/debian/parse_deb_archive_test.go @@ -373,9 +373,11 @@ func Test_processDataTar_capsCopyrightFiles(t *testing.T) { // than after the fact. The existing bounds test uses a payload small enough that buffering the whole // thing would pass anyway; this one is large enough that the difference shows up as bytes. func Test_decompressionStream_failsEarlyInsteadOfBuffering(t *testing.T) { - const expanded = 512 * intFile.MB + // 4x the 8MB budget below is plenty to tell bounded from unbounded. Going bigger only makes the + // fixture slower to build and inflate, which is most of this test's time under -race. + const expanded = 32 * intFile.MB - // a few KB of input expanding to 512MB, so heap is what separates a streaming bound from a + // well under 1% of input expanding to 32MB, so heap is what separates a streaming bound from a // bound checked after the read completes var gz bytes.Buffer gw := gzip.NewWriter(&gz) @@ -386,7 +388,9 @@ func Test_decompressionStream_failsEarlyInsteadOfBuffering(t *testing.T) { remaining -= int64(n) } require.NoError(t, gw.Close()) - require.Less(t, gz.Len(), 1024*1024, "compressed payload should be tiny relative to what it expands to") + // a ratio rather than a fixed size: deflate output for zeros is toolchain dependent (about 510KB on + // go1.26, about 1MB on go1.27), and only the ratio matters to the test + require.Less(t, gz.Len(), expanded/100, "compressed payload should be tiny relative to what it expands to") // both arms use io.ReadAll on purpose. The stream itself never holds much whatever the cap says, // so copying to io.Discard measures nothing; what the cap actually protects is a consumer that diff --git a/syft/pkg/cataloger/erlang/erlang_parser.go b/syft/pkg/cataloger/erlang/erlang_parser.go index 819c3bb41..07079d9a8 100644 --- a/syft/pkg/cataloger/erlang/erlang_parser.go +++ b/syft/pkg/cataloger/erlang/erlang_parser.go @@ -15,6 +15,10 @@ type erlangNode struct { var errSkipComments = errors.New("") +var errUnexpectedEnd = errors.New("unexpected end of input") + +var errTooDeep = fmt.Errorf("nesting deeper than %d levels", parsing.MaxDepth) + func (e erlangNode) Slice() []erlangNode { out, ok := e.value.([]erlangNode) if ok { @@ -79,7 +83,7 @@ func parseErlang(reader io.Reader) (erlangNode, error) { } func parseErlangBlock(data []byte, i *int) (erlangNode, error) { - block, err := parseErlangNode(data, i) + block, err := parseErlangNode(data, i, 0) if err != nil { return node(nil), err } @@ -89,13 +93,19 @@ func parseErlangBlock(data []byte, i *int) (erlangNode, error) { return block, nil } -func parseErlangNode(data []byte, i *int) (erlangNode, error) { +func parseErlangNode(data []byte, i *int, depth int) (erlangNode, error) { parsing.SkipWhitespace(data, i) + if *i >= len(data) { + return node(nil), errUnexpectedEnd + } c := data[*i] switch c { case '[', '{': offset := *i + 1 parsing.SkipWhitespace(data, &offset) + if offset >= len(data) { + return node(nil), errUnexpectedEnd + } c2 := data[offset] // Add support for empty lists @@ -104,7 +114,7 @@ func parseErlangNode(data []byte, i *int) (erlangNode, error) { return node(nil), nil } - return parseErlangList(data, i) + return parseErlangList(data, i, depth+1) case '"': fallthrough case '\'': @@ -145,6 +155,9 @@ func parseErlangAngleString(data []byte, i *int) (erlangNode, error) { } func parseErlangString(data []byte, i *int) (erlangNode, error) { + if *i >= len(data) { + return node(nil), errUnexpectedEnd + } delim := data[*i] *i++ var buf bytes.Buffer @@ -167,13 +180,16 @@ func parseErlangString(data []byte, i *int) (erlangNode, error) { return node(nil), fmt.Errorf("unterminated string at %d", *i) } -func parseErlangList(data []byte, i *int) (erlangNode, error) { +func parseErlangList(data []byte, i *int, depth int) (erlangNode, error) { + if depth > parsing.MaxDepth { + return node(nil), errTooDeep + } *i++ out := erlangNode{ value: []erlangNode{}, } for *i < len(data) { - item, err := parseErlangNode(data, i) + item, err := parseErlangNode(data, i, depth) if err != nil { if err == errSkipComments { parsing.SkipWhitespace(data, i) @@ -183,6 +199,9 @@ func parseErlangList(data []byte, i *int) (erlangNode, error) { } out.value = append(out.value.([]erlangNode), item) parsing.SkipWhitespace(data, i) + if *i >= len(data) { + return node(nil), errUnexpectedEnd + } c := data[*i] switch c { case ',': @@ -210,7 +229,7 @@ func parseErlangComment(data []byte, i *int) { // Rest of a line is a comment. Deals with CR, LF and CR/LF if c == '\n' { break - } else if c == '\r' && data[*i] == '\n' { + } else if c == '\r' && *i < len(data) && data[*i] == '\n' { *i++ break } diff --git a/syft/pkg/cataloger/erlang/erlang_parser_test.go b/syft/pkg/cataloger/erlang/erlang_parser_test.go index 9b6950793..9fa30cefc 100644 --- a/syft/pkg/cataloger/erlang/erlang_parser_test.go +++ b/syft/pkg/cataloger/erlang/erlang_parser_test.go @@ -2,6 +2,7 @@ package erlang import ( "bytes" + "strings" "testing" "github.com/stretchr/testify/assert" @@ -128,3 +129,35 @@ func Test_parseErlangString_escapedByte(t *testing.T) { require.NoError(t, err) assert.Equal(t, `a"b`, got.String()) } + +func TestParseErlang_deepNesting(t *testing.T) { + // 4MB of '[' overflowed the goroutine stack (fatal, not recoverable) before nesting was bounded + _, err := parseErlang(strings.NewReader(strings.Repeat("[a,", 4<<20))) + require.ErrorIs(t, err, errTooDeep) + // the error echoes a clipped window of the single-line input, not the whole thing + assert.Less(t, len(err.Error()), 1024) +} + +func Test_parseErlang_truncated(t *testing.T) { + // each of these read past the end of the input before end-of-input was checked + tests := []struct { + name string + content string + wantErr require.ErrorAssertionFunc + }{ + {name: "whitespace only", content: " ", wantErr: require.Error}, + {name: "unterminated list", content: "[a", wantErr: require.Error}, + {name: "unterminated empty list", content: "[ ", wantErr: require.Error}, + {name: "lone binary string opener", content: "<", wantErr: require.Error}, + {name: "comment ending in CR", content: "%\r", wantErr: require.NoError}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + var err error + require.NotPanics(t, func() { + _, err = parseErlang(strings.NewReader(tt.content)) + }) + tt.wantErr(t, err) + }) + } +} diff --git a/syft/pkg/cataloger/generic/cataloger.go b/syft/pkg/cataloger/generic/cataloger.go index d357d0e07..ec4a5384a 100644 --- a/syft/pkg/cataloger/generic/cataloger.go +++ b/syft/pkg/cataloger/generic/cataloger.go @@ -2,6 +2,8 @@ package generic import ( "context" + "fmt" + "runtime/debug" "github.com/anchore/go-logger" "github.com/anchore/go-sync" @@ -221,12 +223,24 @@ func (c *Cataloger) Catalog(ctx context.Context, resolver file.Resolver) ([]pkg. func (c *Cataloger) process(ctx context.Context, resolver file.Resolver, pkgs []pkg.Package, rels []artifact.Relationship, err error) ([]pkg.Package, []artifact.Relationship, error) { for _, p := range c.processors { - pkgs, rels, err = p.process(ctx, resolver, pkgs, rels, err) + pkgs, rels, err = c.processSafely(ctx, p, resolver, pkgs, rels, err) } return pkgs, rels, err } -func invokeParser(ctx context.Context, resolver file.Resolver, location file.Location, logger logger.Logger, parser Parser, env *Environment) ([]pkg.Package, []artifact.Relationship, error) { +func (c *Cataloger) processSafely(ctx context.Context, p processExecutor, resolver file.Resolver, pkgs []pkg.Package, rels []artifact.Relationship, err error) (outPkgs []pkg.Package, outRels []artifact.Relationship, outErr error) { + defer func() { + if r := recover(); r != nil { + // keep the unprocessed results; there is no single location to attach an unknown to + log.WithFields("cataloger", c.upstreamCataloger, "panic", r).Warn("recovered from panic in package processor") + log.Tracef("processor panic stack:\n%s", debug.Stack()) + outPkgs, outRels, outErr = pkgs, rels, err + } + }() + return p.process(ctx, resolver, pkgs, rels, err) +} + +func invokeParser(ctx context.Context, resolver file.Resolver, location file.Location, logger logger.Logger, parser Parser, env *Environment) (pkgs []pkg.Package, rels []artifact.Relationship, err error) { contentReader, err := resolver.FileContentsByLocation(location) if err != nil { logger.WithFields("location", location.RealPath, "error", err).Debug("unable to fetch contents") @@ -234,6 +248,15 @@ func invokeParser(ctx context.Context, resolver file.Resolver, location file.Loc } defer internal.CloseAndLogError(contentReader, location.AccessPath) + defer func() { + if r := recover(); r != nil { + // a malformed file must not take down the whole SBOM; the caller records this as an unknown + logger.WithFields("location", location.RealPath, "panic", r).Debug("recovered from panic while parsing file") + logger.Tracef("parser panic stack:\n%s", debug.Stack()) + pkgs, rels, err = nil, nil, fmt.Errorf("recovered from panic while parsing file: %v", r) + } + }() + discoveredPackages, discoveredRelationships, err := parser(ctx, resolver, env, file.NewLocationReadCloser(location, contentReader)) if err != nil { // these errors are propagated up, and are likely to be coordinate errors diff --git a/syft/pkg/cataloger/generic/cataloger_test.go b/syft/pkg/cataloger/generic/cataloger_test.go index 2209a3fc9..93f7813d0 100644 --- a/syft/pkg/cataloger/generic/cataloger_test.go +++ b/syft/pkg/cataloger/generic/cataloger_test.go @@ -10,7 +10,6 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - "github.com/anchore/go-sync" "github.com/anchore/syft/internal/unknown" "github.com/anchore/syft/syft/artifact" "github.com/anchore/syft/syft/file" @@ -190,12 +189,122 @@ func TestClosesFileOnParserPanic(t *testing.T) { _, _, err := c.Catalog(ctx, resolver) require.Error(t, err) - var panicErr sync.PanicError - require.ErrorAs(t, err, &panicErr) - assert.Equal(t, "panic!", panicErr.Value) + var coordErr *unknown.CoordinateError + require.ErrorAs(t, err, &coordErr) + assert.ErrorContains(t, err, "recovered from panic while parsing file: panic!") require.True(t, spy.closed) } +func TestParserPanicBecomesUnknown(t *testing.T) { + tests := []struct { + name string + doPanic func() + wantErr string + }{ + { + name: "string panic", + doPanic: func() { panic("boom") }, + wantErr: "boom", + }, + { + name: "runtime panic", + doPanic: func() { + var m map[string]int + m["x"] = 1 + }, + wantErr: "assignment to entry in nil map", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + parser := func(_ context.Context, _ file.Resolver, _ *Environment, reader file.LocationReadCloser) ([]pkg.Package, []artifact.Relationship, error) { + if reader.Path() == "testdata/another-path.txt" { + tt.doPanic() + } + return []pkg.Package{{Name: "good", Locations: file.NewLocationSet(reader.Location)}}, nil, nil + } + + resolver := file.NewMockResolverForPaths("testdata/a-path.txt", "testdata/another-path.txt") + c := NewCataloger("unit-test-cataloger"). + WithParserByPath(parser, "testdata/a-path.txt", "testdata/another-path.txt") + + pkgs, _, err := c.Catalog(context.Background(), resolver) + + require.Len(t, pkgs, 1) + assert.Equal(t, "good", pkgs[0].Name) + + unknowns, remaining := unknown.ExtractCoordinateErrors(err) + require.NoError(t, remaining) + require.Len(t, unknowns, 1) + assert.Equal(t, "testdata/another-path.txt", unknowns[0].Coordinates.RealPath) + assert.ErrorContains(t, unknowns[0].Reason, tt.wantErr) + }) + } +} + +func TestProcessorPanicKeepsResults(t *testing.T) { + parser := func(_ context.Context, _ file.Resolver, _ *Environment, reader file.LocationReadCloser) ([]pkg.Package, []artifact.Relationship, error) { + return []pkg.Package{{Name: "good", Locations: file.NewLocationSet(reader.Location)}}, nil, nil + } + + rename := func(pkgs []pkg.Package, rels []artifact.Relationship, err error) ([]pkg.Package, []artifact.Relationship, error) { + for i := range pkgs { + pkgs[i].Name = "processed" + } + return pkgs, rels, err + } + + tests := []struct { + name string + cataloger *Cataloger + want string + }{ + { + name: "processor", + cataloger: NewCataloger("unit-test-cataloger"). + WithParserByPath(parser, "testdata/a-path.txt"). + WithProcessors(func([]pkg.Package, []artifact.Relationship, error) ([]pkg.Package, []artifact.Relationship, error) { + panic("boom") + }), + want: "good", + }, + { + // output from earlier processors survives a later processor panicking + name: "chained processors", + cataloger: NewCataloger("unit-test-cataloger"). + WithParserByPath(parser, "testdata/a-path.txt"). + WithProcessors(rename, func([]pkg.Package, []artifact.Relationship, error) ([]pkg.Package, []artifact.Relationship, error) { + panic("boom") + }), + want: "processed", + }, + { + name: "resolving processor", + cataloger: NewCataloger("unit-test-cataloger"). + WithParserByPath(parser, "testdata/a-path.txt"). + WithResolvingProcessors(func(context.Context, file.Resolver, []pkg.Package, []artifact.Relationship, error) ([]pkg.Package, []artifact.Relationship, error) { + var m map[string]int + m["x"] = 1 + return nil, nil, nil + }), + want: "good", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + resolver := file.NewMockResolverForPaths("testdata/a-path.txt") + + pkgs, _, err := tt.cataloger.Catalog(context.Background(), resolver) + + require.NoError(t, err) + require.Len(t, pkgs, 1) + assert.Equal(t, tt.want, pkgs[0].Name) + }) + } +} + func Test_CatalogerWithParserByMediaType(t *testing.T) { allParsedPaths := make(map[string]bool) parser := func(_ context.Context, resolver file.Resolver, env *Environment, reader file.LocationReadCloser) ([]pkg.Package, []artifact.Relationship, error) { diff --git a/syft/pkg/cataloger/gentoo/cataloger_test.go b/syft/pkg/cataloger/gentoo/cataloger_test.go index bff79a412..e35abac8d 100644 --- a/syft/pkg/cataloger/gentoo/cataloger_test.go +++ b/syft/pkg/cataloger/gentoo/cataloger_test.go @@ -3,6 +3,8 @@ package gentoo import ( "testing" + "github.com/stretchr/testify/require" + "github.com/anchore/syft/syft/artifact" "github.com/anchore/syft/syft/file" "github.com/anchore/syft/syft/pkg" @@ -170,3 +172,16 @@ func TestCataloger_Globs(t *testing.T) { }) } } + +func TestPortageCataloger_ShortObjLine(t *testing.T) { + // obj lines missing their path or digest are skipped rather than indexed past their end + pkgtest.NewCatalogTester(). + FromDirectory(t, "testdata/short-obj-line"). + ExpectsAssertion(func(t *testing.T, pkgs []pkg.Package, _ []artifact.Relationship) { + require.Len(t, pkgs, 1) + m := pkgs[0].Metadata.(pkg.PortageEntry) + require.Len(t, m.Files, 1) + require.Equal(t, "/usr/bin/bar", m.Files[0].Path) + }). + TestCataloger(t, NewPortageCataloger()) +} diff --git a/syft/pkg/cataloger/gentoo/license.go b/syft/pkg/cataloger/gentoo/license.go index ee23c0372..4fedf5938 100644 --- a/syft/pkg/cataloger/gentoo/license.go +++ b/syft/pkg/cataloger/gentoo/license.go @@ -5,7 +5,6 @@ import ( "bytes" "fmt" "io" - "slices" "strings" "github.com/scylladb/go-set/strset" @@ -35,7 +34,7 @@ import ( func extractLicenses(resolver file.Resolver, closestLocation *file.Location, reader io.Reader) (string, string) { findings := strset.New() contentsWriter := bytes.Buffer{} - scanner := bufio.NewScanner(io.TeeReader(reader, &contentsWriter)) + scanner := internal.NewLineScanner(io.TeeReader(reader, &contentsWriter)) scanner.Split(bufio.ScanWords) var ( mandatoryLicenses, conditionalLicenses, useflagLicenses []string @@ -69,6 +68,13 @@ func extractLicenses(resolver file.Resolver, closestLocation *file.Location, rea } } } + if err := scanner.Err(); err != nil { + fields := []any{"error", err} + if closestLocation != nil { + fields = append(fields, "path", closestLocation.RealPath) + } + log.WithFields(fields...).Debug("failed to fully read portage LICENSE") + } var licenseGroups map[string][]string if usesGroups { @@ -80,31 +86,22 @@ func extractLicenses(resolver file.Resolver, closestLocation *file.Location, rea findings.Add(conditionalLicenses...) findings.Add(useflagLicenses...) - var mandatoryStatement, conditionalStatement string + return strings.TrimSpace(contentsWriter.String()), licenseExpression(mandatoryLicenses, conditionalLicenses) +} - // attempt to build valid SPDX license expression - if len(mandatoryLicenses) > 0 { - mandatoryStatement = strings.Join(mandatoryLicenses, " AND ") +// licenseExpression attempts to build a valid SPDX license expression +func licenseExpression(mandatoryLicenses, conditionalLicenses []string) string { + mandatoryStatement := strings.Join(mandatoryLicenses, " AND ") + conditionalStatement := strings.Join(conditionalLicenses, " OR ") + + switch { + case mandatoryStatement != "" && conditionalStatement != "": + return mandatoryStatement + " AND (" + conditionalStatement + ")" + case mandatoryStatement != "": + return mandatoryStatement + default: + return conditionalStatement } - if len(conditionalLicenses) > 0 { - conditionalStatement = strings.Join(conditionalLicenses, " OR ") - } - - contents := strings.TrimSpace(contentsWriter.String()) - - if mandatoryStatement != "" && conditionalStatement != "" { - return contents, mandatoryStatement + " AND (" + conditionalStatement + ")" - } - - if mandatoryStatement != "" { - return contents, mandatoryStatement - } - - if conditionalStatement != "" { - return contents, conditionalStatement - } - - return contents, "" } func readLicenseGroups(resolver file.Resolver, closestLocation *file.Location) map[string][]string { @@ -161,10 +158,9 @@ func replaceLicenseGroups(licenses []string, groups map[string][]string) []strin } func parseLicenseGroups(reader io.Reader) (map[string][]string, error) { - result := make(map[string][]string) rawGroups := make(map[string][]string) - scanner := bufio.NewScanner(reader) + scanner := internal.NewLineScanner(reader) // first collect all raw groups for scanner.Scan() { @@ -191,58 +187,69 @@ func parseLicenseGroups(reader io.Reader) (map[string][]string, error) { } // next process each group to expand nested references - for groupName, licenses := range rawGroups { - expanded, err := expandLicenses(groupName, licenses, rawGroups, make(map[string]bool)) + expanded := make(map[string][]string) + for groupName := range rawGroups { + if _, err := expandLicenses(groupName, rawGroups, expanded, make(map[string]bool)); err != nil { + return nil, err + } + } + + return expanded, nil +} + +// expandLicenses handles the recursive expansion of license groups. Each group is expanded once and memoized in +// 'expanded' (the file is untrusted, so re-expanding on every reference would be exponential), and 'visiting' holds +// the groups on the current path to detect cycles. We are always in terms of slices instead of sets to ensure +// original ordering is preserved. +func expandLicenses(currentGroup string, rawGroups, expanded map[string][]string, visiting map[string]bool) ([]string, error) { + if result, ok := expanded[currentGroup]; ok { + return result, nil + } + if visiting[currentGroup] { + return nil, fmt.Errorf("cycle detected in license group definitions for group: %s", currentGroup) + } + visiting[currentGroup] = true + defer delete(visiting, currentGroup) + + result := make([]string, 0) + // sets keep dedup linear, since a single group line can hold hundreds of thousands of tokens + seen := strset.New() + seenRefs := strset.New() + + for _, item := range rawGroups[currentGroup] { + refGroupName, isRef := strings.CutPrefix(item, "@") + if !isRef { + // this is a regular license + if !seen.Has(item) { + seen.Add(item) + result = append(result, item) + } + continue + } + + // this is a reference to another group + if seenRefs.Has(refGroupName) { + continue + } + seenRefs.Add(refGroupName) + + if _, exists := rawGroups[refGroupName]; !exists { + return nil, fmt.Errorf("referenced group not found: %s", refGroupName) + } + + refLicenses, err := expandLicenses(refGroupName, rawGroups, expanded, visiting) if err != nil { return nil, err } - result[groupName] = expanded - } - return result, nil -} - -// expandLicenses handles the recursive expansion of license groups, 'visited' is used to detect cycles. We are always -// in terms of slices instead of sets to ensure original ordering is preserved. -func expandLicenses(currentGroup string, licenses []string, rawGroups map[string][]string, visited map[string]bool) ([]string, error) { - if visited[currentGroup] { - return nil, fmt.Errorf("cycle detected in license group definitions for group: %s", currentGroup) - } - - visited[currentGroup] = true - - result := make([]string, 0) - - for _, item := range licenses { - if strings.HasPrefix(item, "@") { - // this is a reference to another group - refGroupName := item[1:] // remove '@' prefix - - refLicenses, exists := rawGroups[refGroupName] - if !exists { - return nil, fmt.Errorf("referenced group not found: %s", refGroupName) + for _, license := range refLicenses { + if !seen.Has(license) { + seen.Add(license) + result = append(result, license) } - - newVisited := make(map[string]bool) - for k, v := range visited { - newVisited[k] = v - } - - expanded, err := expandLicenses(refGroupName, refLicenses, rawGroups, newVisited) - if err != nil { - return nil, err - } - - for _, license := range expanded { - if !slices.Contains(result, license) { - result = append(result, license) - } - } - } else if !slices.Contains(result, item) { - // ...this is a regular license - result = append(result, item) } } + expanded[currentGroup] = result return result, nil } diff --git a/syft/pkg/cataloger/gentoo/license_test.go b/syft/pkg/cataloger/gentoo/license_test.go index e533c2636..47b0cfda7 100644 --- a/syft/pkg/cataloger/gentoo/license_test.go +++ b/syft/pkg/cataloger/gentoo/license_test.go @@ -2,6 +2,7 @@ package gentoo import ( "bytes" + "fmt" "os" "strings" "testing" @@ -181,3 +182,31 @@ func TestReplaceLicenseGroups(t *testing.T) { }) } } + +func Test_extractLicenses_tokenTooLong(t *testing.T) { + // a nil location must not panic when the scan fails partway through + license := "MIT " + strings.Repeat("a", 2<<20) + _, expression := extractLicenses(nil, nil, strings.NewReader(license)) + assert.Equal(t, "MIT", expression) +} + +func TestParseLicenseGroups_hostileInput(t *testing.T) { + // a single wide group was quadratic to dedup, and a chain where each group references the next + // twice was exponential to expand; both must now finish quickly + var wide strings.Builder + wide.WriteString("WIDE") + for i := range 120_000 { + fmt.Fprintf(&wide, " L%d", i) + } + + var chain strings.Builder + chain.WriteString("G0 MIT\n") + for i := 1; i <= 64; i++ { + fmt.Fprintf(&chain, "G%d @G%d @G%d\n", i, i-1, i-1) + } + + groups, err := parseLicenseGroups(strings.NewReader(wide.String() + "\n" + chain.String())) + require.NoError(t, err) + assert.Len(t, groups["WIDE"], 120_000) + assert.Equal(t, []string{"MIT"}, groups["G64"]) +} diff --git a/syft/pkg/cataloger/gentoo/parse_portage_contents.go b/syft/pkg/cataloger/gentoo/parse_portage_contents.go index 30ba84049..db9d68035 100644 --- a/syft/pkg/cataloger/gentoo/parse_portage_contents.go +++ b/syft/pkg/cataloger/gentoo/parse_portage_contents.go @@ -1,7 +1,6 @@ package gentoo import ( - "bufio" "context" "fmt" "path" @@ -70,12 +69,13 @@ func addFiles(resolver file.Resolver, dbLocation file.Location, entry *pkg.Porta } defer internal.CloseAndLogError(contentsReader, dbLocation.RealPath) - scanner := bufio.NewScanner(contentsReader) + scanner := internal.NewLineScanner(contentsReader) for scanner.Scan() { line := strings.Trim(scanner.Text(), "\n") fields := strings.Split(line, " ") - if fields[0] == "obj" { + // an obj line is "obj " + if len(fields) >= 3 && fields[0] == "obj" { record := pkg.PortageFileRecord{ Path: fields[1], } @@ -86,6 +86,9 @@ func addFiles(resolver file.Resolver, dbLocation file.Location, entry *pkg.Porta entry.Files = append(entry.Files, record) } } + if err := scanner.Err(); err != nil { + log.WithFields("path", dbLocation.RealPath, "error", err).Debug("failed to fully read portage contents") + } } func addLicenses(ctx context.Context, resolver file.Resolver, dbLocation file.Location, entry *pkg.PortageEntry) (pkg.LicenseSet, []file.Location) { @@ -127,7 +130,7 @@ func addSize(resolver file.Resolver, dbLocation file.Location, entry *pkg.Portag } defer internal.CloseAndLogError(sizeReader, location.RealPath) - scanner := bufio.NewScanner(sizeReader) + scanner := internal.NewLineScanner(sizeReader) for scanner.Scan() { line := strings.Trim(scanner.Text(), "\n") size, err := strconv.Atoi(line) @@ -135,6 +138,9 @@ func addSize(resolver file.Resolver, dbLocation file.Location, entry *pkg.Portag entry.InstalledSize = size } } + if err := scanner.Err(); err != nil { + log.WithFields("path", location.RealPath, "error", err).Debug("failed to fully read portage SIZE") + } return []file.Location{location.WithAnnotation(pkg.EvidenceAnnotationKey, pkg.SupportingEvidenceAnnotation)} } diff --git a/syft/pkg/cataloger/gentoo/testdata/short-obj-line/var/db/pkg/app-misc/foo-1.0/CONTENTS b/syft/pkg/cataloger/gentoo/testdata/short-obj-line/var/db/pkg/app-misc/foo-1.0/CONTENTS new file mode 100644 index 000000000..dd84618ef --- /dev/null +++ b/syft/pkg/cataloger/gentoo/testdata/short-obj-line/var/db/pkg/app-misc/foo-1.0/CONTENTS @@ -0,0 +1,4 @@ +dir /usr +obj +obj /usr/bin/foo +obj /usr/bin/bar abc123 1600000000 diff --git a/syft/pkg/cataloger/githubactions/package.go b/syft/pkg/cataloger/githubactions/package.go index 179957510..49c5c83e4 100644 --- a/syft/pkg/cataloger/githubactions/package.go +++ b/syft/pkg/cataloger/githubactions/package.go @@ -88,6 +88,13 @@ func packageURL(name, version string) string { var subPath string var namespace string + // docker:// references are OCI images, not GitHub repos. + // They don't fit the pkg:github PURL shape, so skip them entirely + // (same treatment as local composite actions below). + if strings.HasPrefix(name, "docker:") { + return "" + } + fields := strings.SplitN(name, "/", 3) switch len(fields) { case 1: diff --git a/syft/pkg/cataloger/githubactions/packageurl_test.go b/syft/pkg/cataloger/githubactions/packageurl_test.go new file mode 100644 index 000000000..4174c2026 --- /dev/null +++ b/syft/pkg/cataloger/githubactions/packageurl_test.go @@ -0,0 +1,23 @@ +package githubactions + +import "testing" + +func Test_packageURL_skipsDockerPrefix(t *testing.T) { + // docker:// references should return empty PURL (not a valid pkg:github) + got := packageURL("docker://ghcr.io/sethvargo/ratchet:latest", "sha256:527e78e6d29a9ac306e843cf766afe0604a0b35633f16913d85dd522218e8ca1") + if got != "" { + t.Errorf("packageURL(digest-form) = %q, want empty string", got) + } + + got = packageURL("docker://ghcr.io/some-org/some-action", "v1") + if got != "" { + t.Errorf("packageURL(tag-form) = %q, want empty string", got) + } + + // Sanity: regular github actions still work + got = packageURL("actions/checkout", "v4") + want := "pkg:github/actions/checkout@v4" + if got != want { + t.Errorf("packageURL(regular) = %q, want %q", got, want) + } +} diff --git a/syft/pkg/cataloger/githubactions/testdata/workflow-with-docker-actions.yaml b/syft/pkg/cataloger/githubactions/testdata/workflow-with-docker-actions.yaml new file mode 100644 index 000000000..c33a3d54b --- /dev/null +++ b/syft/pkg/cataloger/githubactions/testdata/workflow-with-docker-actions.yaml @@ -0,0 +1,10 @@ +name: "Docker Actions Test" +on: [push] +jobs: + build: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: docker://ghcr.io/sethvargo/ratchet:latest@sha256:527e78e6d29a9ac306e843cf766afe0604a0b35633f16913d85dd522218e8ca1 + - uses: docker://ghcr.io/some-org/some-action@v1 + - uses: actions/setup-go@v5 diff --git a/syft/pkg/cataloger/golang/cataloger_test.go b/syft/pkg/cataloger/golang/cataloger_test.go index 860171f70..4d504ff65 100644 --- a/syft/pkg/cataloger/golang/cataloger_test.go +++ b/syft/pkg/cataloger/golang/cataloger_test.go @@ -59,9 +59,10 @@ func Test_PackageCataloger_Binary(t *testing.T) { { name: "upx compressed binary", fixture: "image-small-upx", + // v1.0.11, not v1.0.0: see the fixture's Dockerfile wantErr: require.NoError, expectedPkgs: []string{ - "anchore.io/not/real @ v1.0.0 (/run-me)", + "anchore.io/not/real @ v1.0.11 (/run-me)", "github.com/andybalholm/brotli @ v1.1.1 (/run-me)", "github.com/dsnet/compress @ v0.0.2-0.20210315054119-f66993602bf5 (/run-me)", "github.com/golang/snappy @ v0.0.4 (/run-me)", @@ -75,17 +76,17 @@ func Test_PackageCataloger_Binary(t *testing.T) { "github.com/anchore/archiver/v3 @ v3.5.3-0.20241210171143-5b1d8d1c7c51 (/run-me)", }, expectedRels: []string{ - "github.com/andybalholm/brotli @ v1.1.1 (/run-me) [dependency-of] anchore.io/not/real @ v1.0.0 (/run-me)", - "github.com/dsnet/compress @ v0.0.2-0.20210315054119-f66993602bf5 (/run-me) [dependency-of] anchore.io/not/real @ v1.0.0 (/run-me)", - "github.com/golang/snappy @ v0.0.4 (/run-me) [dependency-of] anchore.io/not/real @ v1.0.0 (/run-me)", - "github.com/klauspost/compress @ v1.17.11 (/run-me) [dependency-of] anchore.io/not/real @ v1.0.0 (/run-me)", - "github.com/klauspost/pgzip @ v1.2.6 (/run-me) [dependency-of] anchore.io/not/real @ v1.0.0 (/run-me)", - "github.com/anchore/archiver/v3 @ v3.5.3-0.20241210171143-5b1d8d1c7c51 (/run-me) [dependency-of] anchore.io/not/real @ v1.0.0 (/run-me)", - "github.com/nwaples/rardecode @ v1.1.3 (/run-me) [dependency-of] anchore.io/not/real @ v1.0.0 (/run-me)", - "github.com/pierrec/lz4/v4 @ v4.1.21 (/run-me) [dependency-of] anchore.io/not/real @ v1.0.0 (/run-me)", - "github.com/ulikunitz/xz @ v0.5.12 (/run-me) [dependency-of] anchore.io/not/real @ v1.0.0 (/run-me)", - "github.com/xi2/xz @ v0.0.0-20171230120015-48954b6210f8 (/run-me) [dependency-of] anchore.io/not/real @ v1.0.0 (/run-me)", - "stdlib @ go1.23.2 (/run-me) [dependency-of] anchore.io/not/real @ v1.0.0 (/run-me)", + "github.com/andybalholm/brotli @ v1.1.1 (/run-me) [dependency-of] anchore.io/not/real @ v1.0.11 (/run-me)", + "github.com/dsnet/compress @ v0.0.2-0.20210315054119-f66993602bf5 (/run-me) [dependency-of] anchore.io/not/real @ v1.0.11 (/run-me)", + "github.com/golang/snappy @ v0.0.4 (/run-me) [dependency-of] anchore.io/not/real @ v1.0.11 (/run-me)", + "github.com/klauspost/compress @ v1.17.11 (/run-me) [dependency-of] anchore.io/not/real @ v1.0.11 (/run-me)", + "github.com/klauspost/pgzip @ v1.2.6 (/run-me) [dependency-of] anchore.io/not/real @ v1.0.11 (/run-me)", + "github.com/anchore/archiver/v3 @ v3.5.3-0.20241210171143-5b1d8d1c7c51 (/run-me) [dependency-of] anchore.io/not/real @ v1.0.11 (/run-me)", + "github.com/nwaples/rardecode @ v1.1.3 (/run-me) [dependency-of] anchore.io/not/real @ v1.0.11 (/run-me)", + "github.com/pierrec/lz4/v4 @ v4.1.21 (/run-me) [dependency-of] anchore.io/not/real @ v1.0.11 (/run-me)", + "github.com/ulikunitz/xz @ v0.5.12 (/run-me) [dependency-of] anchore.io/not/real @ v1.0.11 (/run-me)", + "github.com/xi2/xz @ v0.0.0-20171230120015-48954b6210f8 (/run-me) [dependency-of] anchore.io/not/real @ v1.0.11 (/run-me)", + "stdlib @ go1.23.2 (/run-me) [dependency-of] anchore.io/not/real @ v1.0.11 (/run-me)", }, }, { diff --git a/syft/pkg/cataloger/golang/parse_go_mod.go b/syft/pkg/cataloger/golang/parse_go_mod.go index 4a40403b1..e9d021729 100644 --- a/syft/pkg/cataloger/golang/parse_go_mod.go +++ b/syft/pkg/cataloger/golang/parse_go_mod.go @@ -1,7 +1,6 @@ package golang import ( - "bufio" "context" "fmt" "go/build" @@ -437,7 +436,7 @@ func parseGoSumFile(resolver file.Resolver, reader file.LocationReadCloser) (map // github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= // github.com/BurntSushi/toml v0.4.1 h1:GaI7EiDXDRfa8VshkTj7Fym7ha+y8/XxIgD2okUIjLw= // github.com/BurntSushi/toml v0.4.1/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ= - scanner := bufio.NewScanner(contents) + scanner := internal.NewLineScanner(contents) // optionally, resize scanner's capacity for lines over 64K, see next example for scanner.Scan() { line := scanner.Text() @@ -449,6 +448,9 @@ func parseGoSumFile(resolver file.Resolver, reader file.LocationReadCloser) (map hash := parts[2] out[nameVersion] = hash } + if err := scanner.Err(); err != nil { + return out, fmt.Errorf("unable to read go.sum: %w", err) + } return out, nil } diff --git a/syft/pkg/cataloger/golang/scan_binary_bomb_test.go b/syft/pkg/cataloger/golang/scan_binary_bomb_test.go index 6369bbda7..d11ffe7fa 100644 --- a/syft/pkg/cataloger/golang/scan_binary_bomb_test.go +++ b/syft/pkg/cataloger/golang/scan_binary_bomb_test.go @@ -20,19 +20,25 @@ import ( // Test_getBuildInfo_compressedSectionBomb covers the reason readBuildInfo exists: debug/buildinfo opens // ELF files with debug/elf itself, and elf.NewFile expands the section-name string table as it parses, // so an oversized compression header there is an unbounded allocation on a path elfutil.NewFile never -// sees. The fixture is a real zlib stream, so it delivers every byte its header promises. +// sees. The fixture is a real zlib stream, but it stops short of what its header promises: delivering +// the full 256MB only made it slower to build and inflate, mostly under -race. func Test_getBuildInfo_compressedSectionBomb(t *testing.T) { - const declared = 256 << 20 // comfortably over elfutil's bound, small enough to allocate in a test + const ( + declared = 256 << 20 // comfortably over elfutil's bound, which only reads the header + // debug/elf allocates the declared size up front today, so delivery barely matters. Twice the + // guarded budget keeps the fixture a bomb if it ever switches to reading in chunks. + delivered = 64 << 20 + ) - bomb := elfWithCompressedNameTable(t, declared) - t.Logf("%d byte fixture declares a %d byte section name table", len(bomb), declared) + bomb := elfWithCompressedNameTable(t, declared, delivered) + t.Logf("%d byte fixture declares a %d byte section name table and delivers %d", len(bomb), declared, delivered) // the unguarded path is the thing being defended against: prove the fixture really is a bomb unguarded := testutils.MeasureAlloc(t, func() { _, err := buildinfo.Read(bytes.NewReader(bomb)) t.Logf("buildinfo.Read err: %v", err) }) - assert.Greater(t, unguarded, uint64(declared), "fixture did not actually deliver the declared bytes") + assert.Greater(t, unguarded, uint64(delivered), "fixture did not actually deliver its bytes") guarded := testutils.MeasureAlloc(t, func() { _, err := getBuildInfo(bytes.NewReader(bomb)) @@ -47,17 +53,22 @@ func Test_getBuildInfo_compressedSectionBomb(t *testing.T) { // goversion opens the file itself and reads .symtab plus the string table it links, and those are // expanded lazily, so the section-name table bound getBuildInfo applies never reaches them. func Test_getCryptoInformation_compressedSymtabBomb(t *testing.T) { - const declared = 256 << 20 // comfortably over elfutil's bound, small enough to allocate in a test + const ( + declared = 256 << 20 // comfortably over elfutil's bound, which only reads the header + // debug/elf allocates the declared size up front today, so delivery barely matters. Twice the + // guarded budget keeps the fixture a bomb if it ever switches to reading in chunks. + delivered = 64 << 20 + ) - bomb := elfWithCompressedSymtab(t, declared) - t.Logf("%d byte fixture declares a %d byte symbol table", len(bomb), declared) + bomb := elfWithCompressedSymtab(t, declared, delivered) + t.Logf("%d byte fixture declares a %d byte symbol table and delivers %d", len(bomb), declared, delivered) // the unguarded path is the thing being defended against: prove the fixture really is a bomb unguarded := testutils.MeasureAlloc(t, func() { _, err := version.ReadExeFromReader(bytes.NewReader(bomb)) t.Logf("goversion err: %v", err) }) - assert.Greater(t, unguarded, uint64(declared), "fixture did not actually deliver the declared bytes") + assert.Greater(t, unguarded, uint64(delivered), "fixture did not actually deliver its bytes") guarded := testutils.MeasureAlloc(t, func() { _, err := getCryptoInformation(bytes.NewReader(bomb)) @@ -85,12 +96,12 @@ func Test_getCryptoInformation_passesThroughNonELF(t *testing.T) { } // elfWithCompressedSymtab builds a minimal ELF64 whose .symtab is SHF_COMPRESSED, declaring `declared` -// decompressed bytes and genuinely delivering them. The name table is left uncompressed so the file gets +// decompressed bytes and genuinely delivering `delivered` of them. The name table is left uncompressed so the file gets // past the bound getBuildInfo already applies, which is the point: this is the section that bound misses. -func elfWithCompressedSymtab(t *testing.T, declared uint64) []byte { +func elfWithCompressedSymtab(t *testing.T, declared, delivered uint64) []byte { t.Helper() - payload := make([]byte, declared) + payload := make([]byte, delivered) var compressed bytes.Buffer zw := zlib.NewWriter(&compressed) _, err := zw.Write(payload) @@ -150,12 +161,12 @@ func elfWithCompressedSymtab(t *testing.T, declared uint64) []byte { } // elfWithCompressedNameTable builds a minimal ELF64 whose only real section is a SHF_COMPRESSED .shstrtab -// declaring `declared` decompressed bytes and genuinely delivering them. Only the test that runs the +// declaring `declared` decompressed bytes and genuinely delivering `delivered` of them. Only the test that runs the // unguarded path needs delivery; use elfDeclaringNameTable everywhere else, since building this one costs -// `declared` bytes of allocation plus a zlib compress of them. -func elfWithCompressedNameTable(t *testing.T, declared uint64) []byte { +// `delivered` bytes of allocation plus a zlib compress of them. +func elfWithCompressedNameTable(t *testing.T, declared, delivered uint64) []byte { t.Helper() - return elfNameTableFixture(t, declared, true) + return elfNameTableFixture(t, declared, delivered) } // elfDeclaringNameTable declares `declared` bytes without delivering them. CheckSectionNameTable reads the @@ -163,19 +174,15 @@ func elfWithCompressedNameTable(t *testing.T, declared uint64) []byte { // bound itself never needs the stream to be real. func elfDeclaringNameTable(t *testing.T, declared uint64) []byte { t.Helper() - return elfNameTableFixture(t, declared, false) + return elfNameTableFixture(t, declared, 64) } -func elfNameTableFixture(t *testing.T, declared uint64, deliver bool) []byte { +func elfNameTableFixture(t *testing.T, declared, delivered uint64) []byte { t.Helper() // the decompressed name table only has to start with the section names; the rest is padding that // exists purely to make the declared size real - size := declared - if !deliver { - size = 64 - } - payload := make([]byte, size) + payload := make([]byte, delivered) copy(payload, "\x00.shstrtab\x00") var compressed bytes.Buffer diff --git a/syft/pkg/cataloger/golang/testdata/image-small-upx/Dockerfile b/syft/pkg/cataloger/golang/testdata/image-small-upx/Dockerfile index bdd8901d7..7ee71b619 100644 --- a/syft/pkg/cataloger/golang/testdata/image-small-upx/Dockerfile +++ b/syft/pkg/cataloger/golang/testdata/image-small-upx/Dockerfile @@ -9,7 +9,9 @@ COPY go.mod go.sum ./ RUN go mod download COPY main.go main.go -RUN CGO_ENABLED=0 GOOS=linux go build -ldflags "-X main.Version=1.0.0" -o run-me . +# 1.0.11 is chosen so UPX has to pad before its loader stub (1.0.0 needed none, which hid a bug). +# TestImageSmallUPXNeedsLoaderPadding checks the padding is still there. +RUN CGO_ENABLED=0 GOOS=linux go build -ldflags "-X main.Version=1.0.11" -o run-me . RUN upx --best --lzma --exact run-me FROM scratch diff --git a/syft/pkg/cataloger/golang/upx.go b/syft/pkg/cataloger/golang/upx.go index 9d3a4ae8c..4b80a26a7 100644 --- a/syft/pkg/cataloger/golang/upx.go +++ b/syft/pkg/cataloger/golang/upx.go @@ -502,13 +502,14 @@ type loaderSkip struct { } // past reports the offset the chain resumes at, and false once the jump has been spent or there is no -// stub to jump over. +// stub to jump over. UPX pads to a 4 byte boundary before it writes the stub, and l_lsize counts from that +// boundary rather than from where the last extent ended. func (l *loaderSkip) past(offset int64) (int64, bool) { if l.done || l.size == 0 { return offset, false } l.done = true - return offset + int64(l.size), true + return (offset+3)&^3 + int64(l.size), true } // decompressUPXBlocks walks the b_info chain, decoding each block into its place in the reconstructed diff --git a/syft/pkg/cataloger/golang/upx_security_test.go b/syft/pkg/cataloger/golang/upx_security_test.go index f4d4f105d..dd703ec6d 100644 --- a/syft/pkg/cataloger/golang/upx_security_test.go +++ b/syft/pkg/cataloger/golang/upx_security_test.go @@ -718,28 +718,57 @@ func TestDecompressUPX_TailExtentsArePlacedPastTheLoader(t *testing.T) { // front of it. Every other crafted fixture here has l_lsize == 0, so without this skipLoader was only // reachable through the Docker-backed fixture. // + // UPX pads to a 4 byte boundary before it writes the stub and l_lsize counts from there, so each case + // ends the head extent the given number of bytes short of a boundary. The head is a stored block so its + // length alone decides where it ends; a compressed one ends wherever the encoder happens to stop. + // // note: block 1 here is not an ELF, so there are no PT_LOAD offsets and the tail lands sequentially. // The hole-filling path this feeds in production is covered by TestFirstHole and by the sparse // fixtures in upx_bounds_regression_test.go. const loaderSize = 64 - head := bytes.Repeat([]byte("H"), 128) tail := bytes.Repeat([]byte("T"), 32) - data := buildUPXHeaderWithLoader(4096, 4096, loaderSize) - data = append(data, blockFor(t, head)...) - data = append(data, make([]byte, 12)...) // end marker: closes the first run of extents - data = append(data, make([]byte, loaderSize-12)...) // the loader stub the chain skips over - data = append(data, blockFor(t, tail)...) - data = append(data, make([]byte, 12)...) // end marker for the tail run - data = padTo(data, 1024) + tests := []struct { + name string + padding int + }{ + {name: "no padding", padding: 0}, + {name: "1 byte of padding", padding: 1}, + {name: "2 bytes of padding", padding: 2}, + {name: "3 bytes of padding", padding: 3}, + } - out, err := unpack(t, data) - // 160 bytes of a declared 4096, short by construction like every other crafted fixture here - require.ErrorIs(t, err, errUPXPartial) - got := readAll(t, out) + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + data := buildUPXHeaderWithLoader(4096, 4096, loaderSize) - assert.Equal(t, head, got[:len(head)], "the head extent stays where it was placed") - assert.Contains(t, string(got), string(tail), "the extent behind the loader must be placed too") + // trim a 128 byte head so that it, plus its b_info, ends tt.padding bytes short of a boundary + headLen := 128 - (len(data)+12+128+tt.padding)%4 + head := bytes.Repeat([]byte("H"), headLen) + stored := make([]byte, 12) + binary.LittleEndian.PutUint32(stored[0:4], uint32(headLen)) // sz_unc + binary.LittleEndian.PutUint32(stored[4:8], uint32(headLen)) // sz_cpr, equal for a stored block + stored[8] = upxMethodStored + + data = append(data, stored...) + data = append(data, head...) + require.Equal(t, tt.padding, (4-len(data)%4)%4, "the head should end tt.padding bytes short of a boundary") + data = append(data, make([]byte, tt.padding)...) // the padding UPX writes in front of the stub + data = append(data, make([]byte, 12)...) // end marker: closes the first run of extents + data = append(data, make([]byte, loaderSize-12)...) // the loader stub the chain skips over + data = append(data, blockFor(t, tail)...) + data = append(data, make([]byte, 12)...) // end marker for the tail run + data = padTo(data, 1024) + + out, err := unpack(t, data) + // a few hundred bytes of a declared 4096, short by construction like every other crafted fixture here + require.ErrorIs(t, err, errUPXPartial) + got := readAll(t, out) + + assert.Equal(t, head, got[:len(head)], "the head extent stays where it was placed") + assert.Contains(t, string(got), string(tail), "the extent behind the loader must be placed too") + }) + } } func TestDecompressUPX_UnsupportedMethodIsNotReportable(t *testing.T) { diff --git a/syft/pkg/cataloger/golang/upx_test.go b/syft/pkg/cataloger/golang/upx_test.go index 2def6178a..05df518e2 100644 --- a/syft/pkg/cataloger/golang/upx_test.go +++ b/syft/pkg/cataloger/golang/upx_test.go @@ -2,10 +2,14 @@ package golang import ( "bytes" + "io" "testing" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + + stereoscopeFile "github.com/anchore/stereoscope/pkg/file" + "github.com/anchore/stereoscope/pkg/imagetest" ) // every case here is one the magic scan must reject, asserted on parseUPXInfo's errNotUPX result. @@ -104,3 +108,33 @@ func TestParseUPXInfo_ValidHeader(t *testing.T) { assert.Equal(t, uint8(22), info.format) assert.Equal(t, uint32(0x100000), info.originalSize) } + +// TestImageSmallUPXNeedsLoaderPadding checks the fixture rather than the code. UPX pads to a 4 byte +// boundary before its loader stub, and the cataloger tests over image-small-upx only catch an unpacker that +// ignores that padding if there is some. A build that needs none passes either way, which is how the bug +// went unnoticed. The case without padding is covered by TestDecompressUPX_TailExtentsArePlacedPastTheLoader. +func TestImageSmallUPXNeedsLoaderPadding(t *testing.T) { + img := imagetest.GetFixtureImage(t, "docker-archive", "image-small-upx") + rc, err := img.OpenPathFromSquash(stereoscopeFile.Path("/run-me")) + require.NoError(t, err) + data, err := io.ReadAll(rc) + require.NoError(t, rc.Close()) + require.NoError(t, err) + + r := bytes.NewReader(data) + info, err := parseUPXInfo(r, int64(len(data))) + require.NoError(t, err) + require.NotZero(t, info.loaderSize, "real UPX output carries a loader stub") + + // walk the first run of extents to the end marker in front of the stub + extentsEnd := info.firstBlockOff + for blockNum := 0; ; blockNum++ { + block, _, err := readChainBlock(r, extentsEnd, blockNum, info.inputLen) + require.NoError(t, err) + if block == nil { + break + } + extentsEnd = block.dataOffset + int64(block.compressedSize) + } + require.NotZero(t, extentsEnd%4, "the fixture no longer needs padding: change the version string in its Dockerfile") +} diff --git a/syft/pkg/cataloger/haskell/parse_stack_lock.go b/syft/pkg/cataloger/haskell/parse_stack_lock.go index 90c25d02e..1f36ea424 100644 --- a/syft/pkg/cataloger/haskell/parse_stack_lock.go +++ b/syft/pkg/cataloger/haskell/parse_stack_lock.go @@ -91,8 +91,12 @@ func parseStackPackageEncoding(pkgEncoding string) (name, version, hash string) encodingSplits := strings.Split(remainingEncoding, "@") version = encodingSplits[0] if len(encodingSplits) > 1 { - startHash, endHash := strings.Index(encodingSplits[1], ":")+1, strings.Index(encodingSplits[1], ",") - hash = encodingSplits[1][startHash:endHash] + // a pin is either "@sha256:," or "@rev:", and only the first carries a + // hash. without the comma the end index is -1 and the slice below goes out of range. + colonIdx, commaIdx := strings.Index(encodingSplits[1], ":"), strings.Index(encodingSplits[1], ",") + if colonIdx != -1 && commaIdx > colonIdx { + hash = encodingSplits[1][colonIdx+1 : commaIdx] + } } return } diff --git a/syft/pkg/cataloger/haskell/parse_stack_yaml_test.go b/syft/pkg/cataloger/haskell/parse_stack_yaml_test.go index 853381d07..449276af2 100644 --- a/syft/pkg/cataloger/haskell/parse_stack_yaml_test.go +++ b/syft/pkg/cataloger/haskell/parse_stack_yaml_test.go @@ -102,6 +102,17 @@ func TestParseStackYaml(t *testing.T) { PkgHash: "e4dc6473faaa75fbd7eccab4e3ee1d651d75bb0e49946ef0b8b751ccde771a55", }, }, + { + Name: "acme-missiles", + Version: "0.3", + PURL: "pkg:hackage/acme-missiles@0.3", + Locations: locationSet, + Language: pkg.Haskell, + Type: pkg.HackagePkg, + Metadata: pkg.HackageStackYamlEntry{ + PkgHash: "", + }, + }, } // TODO: relationships are not under test yet diff --git a/syft/pkg/cataloger/haskell/testdata/stack.yaml b/syft/pkg/cataloger/haskell/testdata/stack.yaml index 8c05906e1..b6a63c4bd 100644 --- a/syft/pkg/cataloger/haskell/testdata/stack.yaml +++ b/syft/pkg/cataloger/haskell/testdata/stack.yaml @@ -12,5 +12,6 @@ extra-deps: - hspec-core-2.9.4@sha256:a126e9087409fef8dcafcd2f8656456527ac7bb163ed4d9cb3a57589042a5fe8,6498 - hspec-discover-2.9.4@sha256:fbcf49ecfc3d4da53e797fd0275264cba776ffa324ee223e2a3f4ec2d2c9c4a6,2165 - stm-2.5.0.2@sha256:e4dc6473faaa75fbd7eccab4e3ee1d651d75bb0e49946ef0b8b751ccde771a55,2314 + - acme-missiles-0.3@rev:0 ghc-options: "$everything": -haddock diff --git a/syft/pkg/cataloger/homebrew/parse_homebrew_formula.go b/syft/pkg/cataloger/homebrew/parse_homebrew_formula.go index 1255ccbc6..93d866746 100644 --- a/syft/pkg/cataloger/homebrew/parse_homebrew_formula.go +++ b/syft/pkg/cataloger/homebrew/parse_homebrew_formula.go @@ -1,11 +1,11 @@ package homebrew import ( - "bufio" "context" "path" "strings" + "github.com/anchore/syft/internal" "github.com/anchore/syft/internal/log" "github.com/anchore/syft/syft/artifact" "github.com/anchore/syft/syft/file" @@ -46,7 +46,7 @@ func parseHomebrewFormula(ctx context.Context, resolver file.Resolver, _ *generi func parseFormulaFile(reader file.LocationReadCloser) (*parsedHomebrewData, error) { pd := parsedHomebrewData{} - scanner := bufio.NewScanner(reader) + scanner := internal.NewLineScanner(reader) for scanner.Scan() { line := strings.TrimSpace(scanner.Text()) if strings.Contains(line, "class ") && strings.Contains(line, " < Formula") { diff --git a/syft/pkg/cataloger/internal/cpegenerate/candidate_by_package_type.go b/syft/pkg/cataloger/internal/cpegenerate/candidate_by_package_type.go index b43906c8b..7a702c071 100644 --- a/syft/pkg/cataloger/internal/cpegenerate/candidate_by_package_type.go +++ b/syft/pkg/cataloger/internal/cpegenerate/candidate_by_package_type.go @@ -518,6 +518,11 @@ var defaultCandidateAdditions = buildCandidateLookup( candidateKey{PkgName: "libxml2"}, candidateAddition{AdditionalVendors: []string{"xmlsoft"}}, }, + { + pkg.ConanPkg, + candidateKey{PkgName: "expat"}, + candidateAddition{AdditionalVendors: []string{"libexpat_project"}, AdditionalProducts: []string{"libexpat"}}, + }, }) var defaultCandidateRemovals = buildCandidateRemovalLookup( diff --git a/syft/pkg/cataloger/internal/cpegenerate/candidate_by_package_type_test.go b/syft/pkg/cataloger/internal/cpegenerate/candidate_by_package_type_test.go index 8f21eeae4..fb73ea799 100644 --- a/syft/pkg/cataloger/internal/cpegenerate/candidate_by_package_type_test.go +++ b/syft/pkg/cataloger/internal/cpegenerate/candidate_by_package_type_test.go @@ -245,3 +245,10 @@ func Test_defaultCandidateAdditions_conan_libxml2(t *testing.T) { vendors := findAdditionalVendors(defaultCandidateAdditions, pkg.ConanPkg, "libxml2", "") assert.Contains(t, vendors, "xmlsoft") } + +func Test_defaultCandidateAdditions_conan_expat(t *testing.T) { + vendors := findAdditionalVendors(defaultCandidateAdditions, pkg.ConanPkg, "expat", "") + assert.Contains(t, vendors, "libexpat_project") + products := findAdditionalProducts(defaultCandidateAdditions, pkg.ConanPkg, "expat") + assert.Contains(t, products, "libexpat") +} diff --git a/syft/pkg/cataloger/internal/cpegenerate/dictionary/data/cpe-index.json b/syft/pkg/cataloger/internal/cpegenerate/dictionary/data/cpe-index.json index 30ca6c481..baa74273d 100644 --- a/syft/pkg/cataloger/internal/cpegenerate/dictionary/data/cpe-index.json +++ b/syft/pkg/cataloger/internal/cpegenerate/dictionary/data/cpe-index.json @@ -4,6 +4,9 @@ "aahframe.work": [ "cpe:2.3:a:aahframework:aah:*:*:*:*:*:go:*:*" ], + "cel.dev/cel-go": [ + "cpe:2.3:a:google:common_expression_language:*:*:*:*:*:go:*:*" + ], "chainguard.dev/apko": [ "cpe:2.3:a:chainguard:apko:*:*:*:*:*:go:*:*" ], @@ -288,6 +291,9 @@ "golang.org/x/text": [ "cpe:2.3:a:golang:text:*:*:*:*:*:*:*:*" ], + "golang.org/x/tools/gopls": [ + "cpe:2.3:a:golang:gopls:*:*:*:*:*:go:*:*" + ], "gopkg.in/yaml.v3": [ "cpe:2.3:a:yaml_project:yaml:*:*:*:*:*:go:*:*" ], @@ -2338,6 +2344,9 @@ "626": [ "cpe:2.3:a:626_project:626:*:*:*:*:*:node.js:*:*" ], + "@a2ui/web_core": [ + "cpe:2.3:a:google:a2ui\\/web_core:*:*:*:*:*:node.js:*:*" + ], "@aborruso/ckan-mcp-server": [ "cpe:2.3:a:ondata:ckan_mcp_server:*:*:*:*:*:node.js:*:*" ], @@ -2524,7 +2533,8 @@ "cpe:2.3:a:matheuspergoli:draftlab\\/db:*:*:*:*:*:node.js:*:*" ], "@electron/packager": [ - "cpe:2.3:a:electron:packager:*:*:*:*:*:node.js:*:*" + "cpe:2.3:a:electron:packager:*:*:*:*:*:node.js:*:*", + "cpe:2.3:a:openjsf:packager:*:*:*:*:*:node.js:*:*" ], "@ensdomains/ens-contracts": [ "cpe:2.3:a:ens.domains:ethereum_name_service:*:*:*:*:*:node.js:*:*" @@ -2538,6 +2548,9 @@ "@fastify/aws-lambda": [ "cpe:2.3:a:fastify:fastify\\/aws-lambda:*:*:*:*:*:node.js:*:*" ], + "@fastify/busboy": [ + "cpe:2.3:a:fastify:fastify\\/busyboy:*:*:*:*:*:node.js:*:*" + ], "@fastify/express": [ "cpe:2.3:a:fastify:fastify\\/express:*:*:*:*:*:node.js:*:*" ], @@ -2547,6 +2560,9 @@ "@fastify/http-proxy": [ "cpe:2.3:a:fastify:fastify\\/http-proxy:*:*:*:*:*:node.js:*:*" ], + "@fastify/jwt": [ + "cpe:2.3:a:fastify:fastify\\/jwt:*:*:*:*:*:node.js:*:*" + ], "@fastify/middie": [ "cpe:2.3:a:fastify:fastify\\/middie:*:*:*:*:*:node.js:*:*" ], @@ -2650,6 +2666,9 @@ "@nestjs/core": [ "cpe:2.3:a:nestjs:nest:*:*:*:*:*:node.js:*:*" ], + "@nestjs/devtools-integration": [ + "cpe:2.3:a:nestjs:devtools-integration:*:*:*:*:*:node.js:*:*" + ], "@nextcloud/dialogs": [ "cpe:2.3:a:nextcloud\\/dialogs_project:nextcloud\\/dialogs:*:*:*:*:*:node.js:*:*" ], @@ -2747,6 +2766,9 @@ "@shopify/hydrogen": [ "cpe:2.3:a:shopify:hydrogen:*:*:*:*:*:node.js:*:*" ], + "@shtse8/filesystem-mcp": [ + "cpe:2.3:a:sylphx:filesystem_mcp:*:*:*:*:*:node.js:*:*" + ], "@simonsmith/cypress-image-snapshot": [ "cpe:2.3:a:simonsmith:cypress_image_snapshot:*:*:*:*:*:node.js:*:*" ], @@ -2847,6 +2869,9 @@ "adb-driver": [ "cpe:2.3:a:adb-driver_project:adb-driver:*:*:*:*:*:node.js:*:*" ], + "adb-mcp": [ + "cpe:2.3:a:srmorete:adb_mcp_server:*:*:*:*:*:node.js:*:*" + ], "adm-zip": [ "cpe:2.3:a:adm-zip_project:adm-zip:*:*:*:*:*:node.js:*:*" ], @@ -3080,6 +3105,9 @@ "bson-objectid": [ "cpe:2.3:a:bson-objectid_project:bson-objectid:*:*:*:*:*:node.js:*:*" ], + "builder-util-runtime": [ + "cpe:2.3:a:electron:builder-util-runtime:*:*:*:*:*:node.js:*:*" + ], "buns": [ "cpe:2.3:a:buns_project:buns:*:*:*:*:*:node.js:*:*" ], @@ -3279,6 +3307,9 @@ "cordova-plugin-file-transfer": [ "cpe:2.3:a:apache:cordova_file_transfer:*:*:*:*:*:android:*:*" ], + "cordova-plugin-x-socialsharing": [ + "cpe:2.3:a:eddyverbruggen:cordova-plugin-x-socialsharing:*:*:*:*:*:node.js:*:*" + ], "corenlp-js-interface": [ "cpe:2.3:a:corenlp-js-interface_project:corenlp-js-interface:*:*:*:*:*:node.js:*:*" ], @@ -3313,6 +3344,9 @@ "csv-parse": [ "cpe:2.3:a:csv-parse_project:csv-parse:*:*:*:*:*:node.js:*:*" ], + "csvjson": [ + "cpe:2.3:a:pradeep-mishra:csvjson:*:*:*:*:*:node.js:*:*" + ], "cuciuci": [ "cpe:2.3:a:cuciuci_project:cuciuci:*:*:*:*:*:node.js:*:*" ], @@ -4758,7 +4792,8 @@ "cpe:2.3:a:mongo-express_project:mongo-express:*:*:*:*:*:node.js:*:*" ], "mongodb-client-encryption": [ - "cpe:2.3:a:mongodb:libmongocrypt:*:*:*:*:*:node.js:*:*" + "cpe:2.3:a:mongodb:libmongocrypt:*:*:*:*:*:node.js:*:*", + "cpe:2.3:a:mongodb:mongodb_client_encryption:*:*:*:*:*:node.js:*:*" ], "mongodb-instance": [ "cpe:2.3:a:mongodb-instance_project:mongodb-instance:*:*:*:*:*:node.js:*:*" @@ -4793,6 +4828,9 @@ "mppx": [ "cpe:2.3:a:wevm:mppx:*:*:*:*:*:node.js:*:*" ], + "mpregular": [ + "cpe:2.3:a:regularjs:mpregular:*:*:*:*:*:node.js:*:*" + ], "mqtt-packet": [ "cpe:2.3:a:mqtt-packet_project:mqtt-packet:*:*:*:*:*:node.js:*:*" ], @@ -5459,6 +5497,9 @@ "quickserver": [ "cpe:2.3:a:quickserver_project:quickserver:*:*:*:*:*:node.js:*:*" ], + "quill": [ + "cpe:2.3:a:slab:quill:*:*:*:*:*:node.js:*:*" + ], "quill-mention": [ "cpe:2.3:a:quill-mention:quill_mention:*:*:*:*:*:node.js:*:*" ], @@ -6074,6 +6115,9 @@ "trailing-slash": [ "cpe:2.3:a:trailing-slash_project:trailing-slash:*:*:*:*:*:node.js:*:*" ], + "translate": [ + "cpe:2.3:a:francisco:translate:*:*:*:*:*:node.js:*:*" + ], "transpile": [ "cpe:2.3:a:transpile_project:transpile:*:*:*:*:*:node.js:*:*" ], @@ -6098,6 +6142,9 @@ "ts-process-promises": [ "cpe:2.3:a:ts-process-promises_project:ts-process-promises:*:*:*:*:*:node.js:*:*" ], + "ts-tree": [ + "cpe:2.3:a:mbovel:ts-tree:*:*:*:*:*:wordpress:*:*" + ], "tsup": [ "cpe:2.3:a:egoist:tsup:*:*:*:*:*:node.js:*:*" ], @@ -6557,6 +6604,9 @@ "apache-airflow-providers-cncf-kubernetes": [ "cpe:2.3:a:apache:apache-airflow-providers-cncf-kubernetes:*:*:*:*:*:*:*:*" ], + "apache-airflow-providers-fab": [ + "cpe:2.3:a:apache:apache-airflow-providers-fab:*:*:*:*:*:*:*:*" + ], "api-res-py": [ "cpe:2.3:a:api-res-py_project:api-res-py:*:*:*:*:*:python:*:*" ], @@ -6880,6 +6930,9 @@ "mltable": [ "cpe:2.3:a:microsoft:azure_machine_learning_software_development_kit:*:*:*:*:*:*:*:*" ], + "mopidy": [ + "cpe:2.3:a:mopidy:mopidy:*:*:*:*:*:python:*:*" + ], "mpxj": [ "cpe:2.3:a:mpxj:mpxj:*:*:*:*:*:python:*:*" ], @@ -6946,6 +6999,9 @@ "passeo": [ "cpe:2.3:a:passeo_project:passeo:*:*:*:*:*:python:*:*" ], + "pdf-oxide": [ + "cpe:2.3:a:oxide:pdf_oxide:*:*:*:*:*:python:*:*" + ], "pdm": [ "cpe:2.3:a:frostming:pdm:*:*:*:*:*:python:*:*" ], @@ -7042,6 +7098,9 @@ "pymetasploit3": [ "cpe:2.3:a:danmcinerney:pymetasploit3:*:*:*:*:*:*:*:*" ], + "pymongo": [ + "cpe:2.3:a:mongodb:python_driver:*:*:*:*:*:mongodb:*:*" + ], "pyo": [ "cpe:2.3:a:pyo_project:pyo:*:*:*:*:*:*:*:*" ], @@ -7198,6 +7257,9 @@ "tuf": [ "cpe:2.3:a:linuxfoundation:the_update_framework:*:*:*:*:*:python:*:*" ], + "tweepy": [ + "cpe:2.3:a:tweepy:tweepy:*:*:*:*:*:python:*:*" + ], "uamqp": [ "cpe:2.3:a:microsoft:azure_uamqp:*:*:*:*:*:python:*:*" ], @@ -9000,6 +9062,9 @@ "affiliate-ads-builder-for-clickbank-products": [ "cpe:2.3:a:clickbank:affiliate_ads_for_clickbank_products:*:*:*:*:*:wordpress:*:*" ], + "affiliate-links": [ + "cpe:2.3:a:wecantrack:affiliate_links:*:*:*:*:*:wordpress:*:*" + ], "affiliate-toolkit-starter": [ "cpe:2.3:a:servit:affiliate-toolkit:*:*:*:*:*:wordpress:*:*" ], @@ -9629,6 +9694,9 @@ "cpe:2.3:a:awesomesupport:awesome_support_wordpress_helpdesk_\\\u0026_support:*:*:*:*:*:wordpress:*:*", "cpe:2.3:a:getawesomesupport:awesome_support:*:*:*:*:*:wordpress:*:*" ], + "awesome-surveys": [ + "cpe:2.3:a:willbrubaker:awesome_surveys:*:*:*:*:*:wordpress:*:*" + ], "awesome-weather": [ "cpe:2.3:a:awesome_weather_widget_project:awesome_weather_widget:*:*:*:*:*:wordpress:*:*" ], @@ -9994,7 +10062,8 @@ "cpe:2.3:a:wordpress:blogger_importer:*:*:*:*:*:wordpress:*:*" ], "blogintroduction-wordpress-plugin": [ - "cpe:2.3:a:kimhuebel:blog_introduction:*:*:*:*:*:wordpress:*:*" + "cpe:2.3:a:kimhuebel:blog_introduction:*:*:*:*:*:wordpress:*:*", + "cpe:2.3:a:kimhuebel:blogintroduction-wordpress-plugin:*:*:*:*:*:wordpress:*:*" ], "blogmentor": [ "cpe:2.3:a:auburnforest:blogmentor:*:*:*:*:*:wordpress:*:*" @@ -10850,6 +10919,9 @@ "cpe:2.3:a:code_snippets:code_snippets:*:*:*:*:*:wordpress:*:*", "cpe:2.3:a:codesnippets:code_snippets:*:*:*:*:*:wordpress:*:*" ], + "code-snippets-cpt": [ + "cpe:2.3:a:jtsternberg:code_snippets_cpt:*:*:*:*:*:wordpress:*:*" + ], "code-snippets-extended": [ "cpe:2.3:a:code_snippets_extended_project:code_snippets_extended:*:*:*:*:*:wordpress:*:*", "cpe:2.3:a:codesnippets:code_snippets:*:*:*:*:*:wordpress:*:*" @@ -11993,7 +12065,7 @@ "cpe:2.3:a:download_zip_attachments_project:download_zip_attachments:*:*:*:*:*:wordpress:*:*" ], "drag-and-drop-multiple-file-upload-contact-form-7": [ - "cpe:2.3:a:codedropz:drag_and_drop_multiple_file_upload_-_contact_form_7:*:*:*:*:*:wordpress:*:*", + "cpe:2.3:a:codedropz:drag_and_drop_multiple_file_upload_-_contact_form_7:*:*:*:*:free:wordpress:*:*", "cpe:2.3:a:codedropz:drag_and_drop_multiple_file_upload_-_contact_form_7:*:*:*:*:pro:wordpress:*:*", "cpe:2.3:a:codedropz:drag_and_drop_multiple_file_upload_-_contact_form_7:*:*:*:*:standard:wordpress:*:*", "cpe:2.3:a:codedropz:drag_and_drop_multiple_file_upload_for_contact_form_7:*:*:*:*:free:wordpress:*:*" @@ -12007,6 +12079,9 @@ "drive-folder-embeder": [ "cpe:2.3:a:azumbro:drive_folder_embedder:*:*:*:*:*:wordpress:*:*" ], + "drm-protected-video-streaming": [ + "cpe:2.3:a:s3bubble:s3player:*:*:*:*:*:wordpress:*:*" + ], "droit-dark-mode": [ "cpe:2.3:a:droitthemes:droit_dark_mode:*:*:*:*:*:wordpress:*:*" ], @@ -12345,7 +12420,8 @@ "cpe:2.3:a:nicheaddons:education_addon:*:*:*:*:*:wordpress:*:*" ], "edwiser-bridge": [ - "cpe:2.3:a:edwiser:bridge:*:*:*:*:*:wordpress:*:*" + "cpe:2.3:a:edwiser:bridge:*:*:*:*:*:wordpress:*:*", + "cpe:2.3:a:wisdmlabs:edwiser_bridge:*:*:*:*:*:wordpress:*:*" ], "eelv-newsletter": [ "cpe:2.3:a:eelv_newsletter_project:eelv_newsletter:*:*:*:*:*:wordpress:*:*" @@ -13696,6 +13772,9 @@ "google-cse": [ "cpe:2.3:a:erikeng:google_cse:*:*:*:*:*:wordpress:*:*" ], + "google-distance-calculator": [ + "cpe:2.3:a:webtamarin:mk_google_directions:*:*:*:*:*:wordpress:*:*" + ], "google-document-embedder": [ "cpe:2.3:a:google_doc_embedder:google_doc_embedder:*:*:*:*:*:wordpress:*:*", "cpe:2.3:a:google_doc_embedder_project:google_doc_embedder:*:*:*:*:*:wordpress:*:*" @@ -13971,6 +14050,9 @@ "header-enhancement": [ "cpe:2.3:a:catchplugins:header_enhancement:*:*:*:*:*:wordpress:*:*" ], + "header-footer": [ + "cpe:2.3:a:satollo:head\\,_footer\\,_and_post_injections:*:*:*:*:*:wordpress:*:*" + ], "header-footer-code": [ "cpe:2.3:a:ninjateam:header_footer_custom_code:*:*:*:*:*:wordpress:*:*" ], @@ -15046,6 +15128,9 @@ "lh-password-changer": [ "cpe:2.3:a:shawfactor:lh-password-changer:*:*:*:*:*:wordpress:*:*" ], + "library-bookshelves": [ + "cpe:2.3:a:photonicgnostic:library_bookshelves:*:*:*:*:*:wordpress:*:*" + ], "library-viewer": [ "cpe:2.3:a:pexlechris:library_viewer:*:*:*:*:*:wordpress:*:*" ], @@ -15124,6 +15209,9 @@ "linker": [ "cpe:2.3:a:pojo:linker:*:*:*:*:*:wordpress:*:*" ], + "linkmyposts": [ + "cpe:2.3:a:gunnettmd:linkmyposts:*:*:*:*:*:wordpress:*:*" + ], "linkworth-wp-plugin": [ "cpe:2.3:a:linkworth:linkworth:*:*:*:*:*:wordpress:*:*" ], @@ -15501,6 +15589,9 @@ "market-exporter": [ "cpe:2.3:a:vanyukov:market_exporter:*:*:*:*:*:wordpress:*:*" ], + "marketing-and-seo-booster": [ + "cpe:2.3:a:secretlab:marketing_and_seo_booster:*:*:*:*:*:wordpress:*:*" + ], "marketo-forms-and-tracking": [ "cpe:2.3:a:hutchhouse:marketo_forms_and_tracking:*:*:*:*:*:wordpress:*:*" ], @@ -17028,6 +17119,9 @@ "points-and-rewards-for-woocommerce": [ "cpe:2.3:a:wpswings:points_and_rewards_for_woocommerce:*:*:*:*:*:wordpress:*:*" ], + "policy-genius": [ + "cpe:2.3:a:waelhassan:privacy_policy_genius:*:*:*:*:*:wordpress:*:*" + ], "poll-maker": [ "cpe:2.3:a:ays-pro:poll_maker:*:*:*:*:free:wordpress:*:*" ], @@ -17713,6 +17807,9 @@ "rays-grid": [ "cpe:2.3:a:rays_grid_project:rays_grid:*:*:*:*:*:wordpress:*:*" ], + "razorpay-payment-button": [ + "cpe:2.3:a:razorpay:razorpay_payment_button:*:*:*:*:*:wordpress:*:*" + ], "rb-internal-links": [ "cpe:2.3:a:rb_internal_links_project:rb_internal_links:*:*:*:*:*:wordpress:*:*" ], @@ -18215,6 +18312,9 @@ "schreikasten": [ "cpe:2.3:a:schreikasten_project:schreikasten:*:*:*:*:*:wordpress:*:*" ], + "scottcart": [ + "cpe:2.3:a:wpplugin:scottcart:*:*:*:*:*:wordpress:*:*" + ], "scoutnet-kalender": [ "cpe:2.3:a:scoutnet:kalender:*:*:*:*:*:wordpress:*:*" ], @@ -18337,6 +18437,9 @@ "sender": [ "cpe:2.3:a:bestwebsoft:sender:*:*:*:*:*:wordpress:*:*" ], + "sender-net-automated-emails": [ + "cpe:2.3:a:sender:sender:*:*:*:*:*:wordpress:*:*" + ], "sendgrid-email-delivery-simplified": [ "cpe:2.3:a:sendgrid:sendgrid:*:*:*:*:*:wordpress:*:*" ], @@ -18398,6 +18501,9 @@ "sermon-browser": [ "cpe:2.3:a:sermon_browser_project:sermon_browser:*:*:*:*:*:wordpress:*:*" ], + "sermonaudio-widgets": [ + "cpe:2.3:a:sermonaudio:sermonaudio_widgets:*:*:*:*:*:wordpress:*:*" + ], "sermone-online-sermons-management": [ "cpe:2.3:a:bearthemes:sermon\\'e_-_sermons_online:*:*:*:*:*:wordpress:*:*" ], @@ -18445,6 +18551,9 @@ "shared-files": [ "cpe:2.3:a:tammersoft:shared_files:*:*:*:*:*:wordpress:*:*" ], + "sharethis-share-buttons": [ + "cpe:2.3:a:sharethis:sharethis_share_buttons:*:*:*:*:*:wordpress:*:*" + ], "shariff": [ "cpe:2.3:a:datenverwurstungszentrale:shariff_wrapper:*:*:*:*:*:wordpress:*:*" ], @@ -18747,6 +18856,9 @@ "simple-mobile-url-redirect": [ "cpe:2.3:a:ozette:simple_mobile_url_redirect:*:*:*:*:*:wordpress:*:*" ], + "simple-nav-archives": [ + "cpe:2.3:a:philipwalton:simple_nav_archives:*:*:*:*:*:wordpress:*:*" + ], "simple-page-access-restriction": [ "cpe:2.3:a:pluginsandsnippets:simple_page_access_restriction:*:*:*:*:*:wordpress:*:*" ], @@ -19412,6 +19524,9 @@ "stars-rating": [ "cpe:2.3:a:stars_rating_project:stars_rating:*:*:*:*:*:wordpress:*:*" ], + "stars-smtp-mailer": [ + "cpe:2.3:a:myriadsolutionz:stars_smtp_mailer:*:*:*:*:*:wordpress:*:*" + ], "starter-templates": [ "cpe:2.3:a:fancywp:starter_templates:*:*:*:*:*:wordpress:*:*" ], @@ -19445,6 +19560,9 @@ "sticky-popup": [ "cpe:2.3:a:sticky_popup_project:sticky_popup:*:*:*:*:*:wordpress:*:*" ], + "sticky-social-icons": [ + "cpe:2.3:a:sanil:sticky_social_icons:*:*:*:*:*:wordpress:*:*" + ], "stklcode-liveticker": [ "cpe:2.3:a:stklcode:liveticker:*:*:*:*:*:wordpress:*:*" ], @@ -19703,6 +19821,9 @@ "tabs-responsive": [ "cpe:2.3:a:wpshopmart:tabs_responsive:*:*:*:*:*:wordpress:*:*" ], + "tabulate": [ + "cpe:2.3:a:samwilson:tabulate:*:*:*:*:*:wordpress:*:*" + ], "tagregator": [ "cpe:2.3:a:tagregator_project:tagregator:*:*:*:*:*:wordpress:*:*" ], @@ -21678,6 +21799,9 @@ "wp-cleanfix": [ "cpe:2.3:a:undolog:wp_cleanfix:*:*:*:*:*:wordpress:*:*" ], + "wp-click-info": [ + "cpe:2.3:a:s-a:wp_click_info:*:*:*:*:*:wordpress:*:*" + ], "wp-client-logo-carousel": [ "cpe:2.3:a:client_logo_carousel_project:client_logo_carousel:*:*:*:*:*:wordpress:*:*" ], @@ -22303,6 +22427,9 @@ "wp-media-library-categories": [ "cpe:2.3:a:jeffrey-wp:media_library_categories:*:*:*:*:*:wordpress:*:*" ], + "wp-mediatagger": [ + "cpe:2.3:a:phd38:wp_mediatagger:*:*:*:*:*:wordpress:*:*" + ], "wp-members": [ "cpe:2.3:a:butlerblog:wp-members:*:*:*:*:*:wordpress:*:*" ], @@ -23639,6 +23766,9 @@ "chic-lite": [ "cpe:2.3:a:rarathemes:chic:*:*:*:*:lite:wordpress:*:*" ], + "ciyashop": [ + "cpe:2.3:a:potenzaglobalsolutions:ciyashop:*:*:*:*:*:wordpress:*:*" + ], "clean-retina": [ "cpe:2.3:a:themehorse:clean_retina:*:*:*:*:free:wordpress:*:*" ], @@ -23687,6 +23817,9 @@ "drop": [ "cpe:2.3:a:competethemes:drop:*:*:*:*:*:wordpress:*:*" ], + "dwt-listing-2": [ + "cpe:2.3:a:scriptsbundle:dwt_listing:*:*:*:*:*:wordpress:*:*" + ], "education-zone": [ "cpe:2.3:a:rarathemes:education_zone:*:*:*:*:*:wordpress:*:*" ], @@ -23961,6 +24094,9 @@ "travel-agency": [ "cpe:2.3:a:rarathemes:travel_agency:*:*:*:*:*:wordpress:*:*" ], + "traveler": [ + "cpe:2.3:a:shinecommerce:traveler:*:*:*:*:*:wordpress:*:*" + ], "tweaker5": [ "cpe:2.3:a:arnoldgoodway:tweaker5:*:*:*:*:*:wordpress:*:*" ], diff --git a/syft/pkg/cataloger/internal/cpegenerate/generate.go b/syft/pkg/cataloger/internal/cpegenerate/generate.go index 097cf4e32..4495e17c1 100644 --- a/syft/pkg/cataloger/internal/cpegenerate/generate.go +++ b/syft/pkg/cataloger/internal/cpegenerate/generate.go @@ -3,10 +3,12 @@ package cpegenerate import ( "bufio" "bytes" + "cmp" _ "embed" "encoding/json" "fmt" "regexp" + "slices" "sort" "strings" "sync" @@ -145,8 +147,14 @@ func FromPackageAttributes(p pkg.Package) []cpe.CPE { return nil } - vendors := candidateVendors(p) - products := candidateProducts(p) + // real CPE fields are short, and every byte of a field is paid once per generated CPE (key building and regex + // validation), so a hostile name or version is not worth generating candidates for at all. + if len(p.Name) > maxCandidateFieldLength || len(p.Version) > maxCandidateFieldLength { + return nil + } + + vendors := boundCandidates(candidateVendors(p)) + products := boundCandidates(candidateProducts(p)) targetSWs := candidateTargetSw(p) if len(products) == 0 { return nil @@ -183,6 +191,32 @@ func FromPackageAttributes(p pkg.Package) []cpe.CPE { return result } +// maxCandidateFieldLength bounds the name, version, and each vendor and product candidate that reaches the +// cross-product in FromPackageAttributes. Real values are well under this. +const maxCandidateFieldLength = 512 + +// maxCandidates bounds how many vendor and product candidates each feed the cross-product in FromPackageAttributes, +// so a single package yields at most maxCandidates² CPEs. Some sources add a candidate per input element with no bound +// of their own (e.g. one vendor per java groupID segment, one product per .NET executable), and real packages produce +// a dozen or so candidates at most. +const maxCandidates = 64 + +// boundCandidates drops candidates longer than maxCandidateFieldLength and keeps at most maxCandidates of the rest. +// Candidates are ordered shortest first (then lexically) so that truncation is deterministic and keeps the short, +// generic values that tend to be real vendors and products. +func boundCandidates(candidates []string) []string { + candidates = slices.DeleteFunc(candidates, func(c string) bool { + return len(c) > maxCandidateFieldLength + }) + if len(candidates) <= maxCandidates { + return candidates + } + slices.SortFunc(candidates, func(a, b string) int { + return cmp.Or(cmp.Compare(len(a), len(b)), strings.Compare(a, b)) + }) + return candidates[:maxCandidates] +} + func candidateTargetSw(p pkg.Package) []string { switch p.Type { case pkg.WordpressPluginPkg: @@ -359,13 +393,19 @@ func addAllSubSelections(fields fieldCandidateSet) { } } +// maxSubSelections bounds how many prefixes generateSubSelections returns. Each prefix becomes a vendor candidate, so +// without a bound a name with N separators yields O(N) vendors of O(N) length each (quadratic in time and memory). +// Real package names rarely exceed a handful of segments. +const maxSubSelections = 10 + // generateSubSelections attempts to split a field by hyphens and underscores and return a list of sensible sub-selections -// that can be used as product or vendor candidates. E.g. jenkins-ci-tools -> [jenkins-ci-tools, jenkins-ci, jenkins]. +// that can be used as product or vendor candidates. E.g. jenkins-ci-tools -> [jenkins, jenkins-ci, jenkins-ci-tools]. +// At most maxSubSelections prefixes are returned, shortest first. func generateSubSelections(field string) (results []string) { scanner := bufio.NewScanner(strings.NewReader(field)) scanner.Split(scanByHyphenOrUnderscore) var lastToken uint8 - for scanner.Scan() { + for len(results) < maxSubSelections && scanner.Scan() { rawCandidate := scanner.Text() if len(rawCandidate) == 0 { break diff --git a/syft/pkg/cataloger/internal/cpegenerate/generate_test.go b/syft/pkg/cataloger/internal/cpegenerate/generate_test.go index 704468780..a4e6d3d08 100644 --- a/syft/pkg/cataloger/internal/cpegenerate/generate_test.go +++ b/syft/pkg/cataloger/internal/cpegenerate/generate_test.go @@ -182,6 +182,24 @@ func TestGeneratePackageCPEs(t *testing.T) { "cpe:2.3:a:name:name:3.2:*:*:*:*:*:*:*", }, }, + { + // groovy moved to org.apache.groovy at 4.0 and real jars ship no pom metadata, so the known package + // list drives the vendor. See https://github.com/anchore/syft/issues/5311 + name: "groovy 4 module without pom metadata", + p: pkg.Package{ + Name: "groovy-json", + Version: "4.0.33", + FoundBy: "some-analyzer", + Language: pkg.Java, + Type: pkg.JavaPkg, + Metadata: pkg.JavaArchive{Manifest: &pkg.JavaManifest{}}, + }, + expected: []string{ + "cpe:2.3:a:apache:groovy-json:4.0.33:*:*:*:*:*:*:*", + "cpe:2.3:a:apache:groovy_json:4.0.33:*:*:*:*:*:*:*", + "cpe:2.3:a:apache:groovy:4.0.33:*:*:*:*:*:*:*", + }, + }, { name: "java language with groupID", p: pkg.Package{ @@ -874,6 +892,22 @@ func TestGeneratePackageCPEs(t *testing.T) { "cpe:2.3:a:rust_package:rust_package:0.5.0:*:*:*:*:rust:*:*", }, }, + { + name: "conan expat: libexpat_project vendor and libexpat product", + p: pkg.Package{ + Name: "expat", + Version: "2.5.0", + Type: pkg.ConanPkg, + }, + expected: []string{ + "cpe:2.3:a:expat:expat:2.5.0:*:*:*:*:*:*:*", + "cpe:2.3:a:expat:libexpat:2.5.0:*:*:*:*:*:*:*", + "cpe:2.3:a:libexpat:expat:2.5.0:*:*:*:*:*:*:*", + "cpe:2.3:a:libexpat:libexpat:2.5.0:*:*:*:*:*:*:*", + "cpe:2.3:a:libexpat_project:expat:2.5.0:*:*:*:*:*:*:*", + "cpe:2.3:a:libexpat_project:libexpat:2.5.0:*:*:*:*:*:*:*", + }, + }, } for _, test := range tests { @@ -1103,6 +1137,11 @@ func Test_generateSubSelections(t *testing.T) { field: "_", expected: nil, }, + { + // capped at maxSubSelections, keeping the shortest prefixes (the full name is not returned) + field: "a_b_c_d_e_f_g_h_i_j_k_l", + expected: []string{"a", "a_b", "a_b_c", "a_b_c_d", "a_b_c_d_e", "a_b_c_d_e_f", "a_b_c_d_e_f_g", "a_b_c_d_e_f_g_h", "a_b_c_d_e_f_g_h_i", "a_b_c_d_e_f_g_h_i_j"}, + }, } for _, test := range tests { t.Run(test.field, func(t *testing.T) { @@ -1247,3 +1286,59 @@ func TestAddBinaryPackageDigitVariations(t *testing.T) { }) } } + +func TestFromPackageAttributes_boundsSubSelectionBlowup(t *testing.T) { + // a name with many separators used to produce a vendor candidate per segment, each O(name length), which was + // quadratic in time and memory. + name := strings.Repeat("a_", 200) + "a" + p := pkg.Package{Name: name, Version: "1.0", Type: pkg.PythonPkg, Language: pkg.Python} + + cpes := FromPackageAttributes(p) + assert.NotEmpty(t, cpes) + assert.LessOrEqual(t, len(cpes), 1000) +} + +func TestFromPackageAttributes_boundsCandidateCrossProduct(t *testing.T) { + // every groupID segment becomes a vendor, and with no artifactID every segment past the second also becomes a + // product, so the vendor x product cross-product is quadratic in segment count without a bound. + var segments []string + for i := range 1000 { + segments = append(segments, fmt.Sprintf("s%d", i)) + } + p := pkg.Package{ + Name: "thing", + Version: "1.0", + Type: pkg.JavaPkg, + Language: pkg.Java, + Metadata: pkg.JavaArchive{ + Manifest: &pkg.JavaManifest{ + Main: pkg.KeyValues{{Key: "Bundle-SymbolicName", Value: "com.x." + strings.Join(segments, ".")}}, + }, + }, + } + + cpes := FromPackageAttributes(p) + assert.NotEmpty(t, cpes) + assert.LessOrEqual(t, len(cpes), maxCandidates*maxCandidates) +} + +func TestFromPackageAttributes_skipsOversizedFields(t *testing.T) { + long := strings.Repeat("a", maxCandidateFieldLength+1) + + assert.Empty(t, FromPackageAttributes(pkg.Package{Name: long, Version: "1.0", Type: pkg.PythonPkg})) + assert.Empty(t, FromPackageAttributes(pkg.Package{Name: "thing", Version: long, Type: pkg.PythonPkg})) + assert.NotEmpty(t, FromPackageAttributes(pkg.Package{Name: "thing", Version: "1.0", Type: pkg.PythonPkg})) +} + +func Test_boundCandidates(t *testing.T) { + long := strings.Repeat("a", maxCandidateFieldLength+1) + assert.Equal(t, []string{"b", "a"}, boundCandidates([]string{"b", long, "a"}), "under the cap order is untouched") + + var many []string + for i := range maxCandidates + 10 { + many = append(many, strings.Repeat("x", maxCandidates+10-i)) + } + got := boundCandidates(many) + assert.Len(t, got, maxCandidates) + assert.Equal(t, "x", got[0], "shortest candidates are kept") +} diff --git a/syft/pkg/cataloger/internal/cpegenerate/java.go b/syft/pkg/cataloger/internal/cpegenerate/java.go index 5f7129562..69d9f85c2 100644 --- a/syft/pkg/cataloger/internal/cpegenerate/java.go +++ b/syft/pkg/cataloger/internal/cpegenerate/java.go @@ -190,16 +190,16 @@ func GroupIDsFromJavaPackage(p pkg.Package) (groupIDs []string) { return nil } - return GroupIDsFromJavaMetadata(p.Name, metadata) + return GroupIDsFromJavaMetadata(p.Name, p.Version, metadata) } // GroupIDsFromJavaMetadata returns the possible group IDs for a Java package // This function is similar to GroupIDFromJavaPackage, but returns all possible group IDs and is less strict // It is used as a way to generate possible candidates for CPE matching. -func GroupIDsFromJavaMetadata(pkgName string, metadata pkg.JavaArchive) (groupIDs []string) { +func GroupIDsFromJavaMetadata(pkgName, version string, metadata pkg.JavaArchive) (groupIDs []string) { groupIDs = append(groupIDs, groupIDsFromPomProperties(metadata.PomProperties)...) groupIDs = append(groupIDs, groupIDsFromPomProject(metadata.PomProject)...) - groupIDs = append(groupIDs, groupIDsFromJavaManifest(pkgName, metadata.Manifest)...) + groupIDs = append(groupIDs, groupIDsFromJavaManifest(pkgName, version, metadata.Manifest)...) return groupIDs } @@ -253,8 +253,8 @@ func addGroupIDsFromGroupIDsAndArtifactID(groupID, artifactID string) (groupIDs return groupIDs } -func groupIDsFromJavaManifest(pkgName string, manifest *pkg.JavaManifest) []string { - if groupID, ok := DefaultArtifactIDToGroupID[pkgName]; ok { +func groupIDsFromJavaManifest(pkgName, version string, manifest *pkg.JavaManifest) []string { + if groupID, ok := ArtifactIDToGroupID(pkgName, version); ok { return []string{groupID} } @@ -303,7 +303,7 @@ func GetManifestFieldGroupIDs(manifest *pkg.JavaManifest, fields []string) (grou func cleanGroupID(groupID string) string { groupID = strings.TrimSpace(strings.Split(removeOSCIDirectives(groupID), "#")[0]) - if replacement, ok := GroupIDCorrections[groupID]; ok { + if replacement, ok := groupIDCorrections[groupID]; ok { return replacement } return groupID diff --git a/syft/pkg/cataloger/internal/cpegenerate/java_groupid_map.go b/syft/pkg/cataloger/internal/cpegenerate/java_groupid_map.go index 9dcb71567..9cb50ffd6 100644 --- a/syft/pkg/cataloger/internal/cpegenerate/java_groupid_map.go +++ b/syft/pkg/cataloger/internal/cpegenerate/java_groupid_map.go @@ -1,10 +1,17 @@ package cpegenerate -var GroupIDCorrections = map[string]string{ +import ( + "strconv" + "strings" +) + +var groupIDCorrections = map[string]string{ "org.lz4.java": "org.lz4", } -var DefaultArtifactIDToGroupID = map[string]string{ +// defaultArtifactIDToGroupID holds artifacts that always map to the same group ID. Look up group IDs with +// ArtifactIDToGroupID instead of reading this map directly, since migratedArtifactIDToGroupID takes precedence. +var defaultArtifactIDToGroupID = map[string]string{ "ant": "org.apache.ant", "ant-antlr": "org.apache.ant", "ant-antunit": "org.apache.ant", @@ -1832,104 +1839,83 @@ var DefaultArtifactIDToGroupID = map[string]string{ "commons-jelly-tags-velocity": "commons-jelly", "commons-jelly-tags-xml": "commons-jelly", "commons-jelly-tags-xmlunit": "commons-jelly", - "groovy": "org.codehaus.groovy", - "groovy-all": "org.codehaus.groovy", - "groovy-all-jdk14": "org.codehaus.groovy", - "groovy-all-minimal": "org.codehaus.groovy", - "groovy-all-tests": "org.codehaus.groovy", - "groovy-ant": "org.codehaus.groovy", - "groovy-astbuilder": "org.codehaus.groovy", - "groovy-backports-compat23": "org.codehaus.groovy", - "groovy-binary": "org.codehaus.groovy", - "groovy-bom": "org.codehaus.groovy", - "groovy-bsf": "org.codehaus.groovy", - "groovy-cli-commons": "org.codehaus.groovy", - "groovy-cli-picocli": "org.codehaus.groovy", - "groovy-console": "org.codehaus.groovy", - "groovy-datetime": "org.codehaus.groovy", - "groovy-dateutil": "org.codehaus.groovy", - "groovy-docgenerator": "org.codehaus.groovy", - "groovy-eclipse-batch": "org.codehaus.groovy", - "groovy-eclipse-compiler": "org.codehaus.groovy", - "groovy-groovydoc": "org.codehaus.groovy", - "groovy-groovysh": "org.codehaus.groovy", - "groovy-jaxb": "org.codehaus.groovy", - "groovy-jdk14": "org.codehaus.groovy", - "groovy-jmx": "org.codehaus.groovy", - "groovy-json": "org.codehaus.groovy", - "groovy-json-direct": "org.codehaus.groovy", - "groovy-jsr223": "org.codehaus.groovy", - "groovy-macro": "org.codehaus.groovy", - "groovy-nio": "org.codehaus.groovy", - "groovy-servlet": "org.codehaus.groovy", - "groovy-sql": "org.codehaus.groovy", - "groovy-swing": "org.codehaus.groovy", - "groovy-templates": "org.codehaus.groovy", - "groovy-test": "org.codehaus.groovy", - "groovy-test-junit5": "org.codehaus.groovy", - "groovy-testng": "org.codehaus.groovy", - "groovy-tests-vm8": "org.codehaus.groovy", - "groovy-xml": "org.codehaus.groovy", - "groovy-xmlrpc": "org.codehaus.groovy", - "groovy-yaml": "org.codehaus.groovy", - "kafka-clients": "org.apache.kafka", - "kafka-examples": "org.apache.kafka", - "kafka-group-coordinator": "org.apache.kafka", - "kafka-hadoop-consumer": "org.apache.kafka", - "kafka-hadoop-producer": "org.apache.kafka", - "kafka-java-examples": "org.apache.kafka", - "kafka-log4j-appender": "org.apache.kafka", - "kafka-metadata": "org.apache.kafka", - "kafka-perf_2.10": "org.apache.kafka", - "kafka-perf_2.8.0": "org.apache.kafka", - "kafka-perf_2.8.2": "org.apache.kafka", - "kafka-perf_2.9.1": "org.apache.kafka", - "kafka-perf_2.9.2": "org.apache.kafka", - "kafka-raft": "org.apache.kafka", - "kafka-server-common": "org.apache.kafka", - "kafka-shell": "org.apache.kafka", - "kafka-storage": "org.apache.kafka", - "kafka-storage-api": "org.apache.kafka", - "kafka-streams": "org.apache.kafka", - "kafka-streams-examples": "org.apache.kafka", - "kafka-streams-scala_2.11": "org.apache.kafka", - "kafka-streams-scala_2.12": "org.apache.kafka", - "kafka-streams-scala_2.13": "org.apache.kafka", - "kafka-streams-test-utils": "org.apache.kafka", - "kafka-streams-upgrade-system-tests-0100": "org.apache.kafka", - "kafka-streams-upgrade-system-tests-0101": "org.apache.kafka", - "kafka-streams-upgrade-system-tests-0102": "org.apache.kafka", - "kafka-streams-upgrade-system-tests-0110": "org.apache.kafka", - "kafka-streams-upgrade-system-tests-10": "org.apache.kafka", - "kafka-streams-upgrade-system-tests-11": "org.apache.kafka", - "kafka-streams-upgrade-system-tests-20": "org.apache.kafka", - "kafka-streams-upgrade-system-tests-21": "org.apache.kafka", - "kafka-streams-upgrade-system-tests-22": "org.apache.kafka", - "kafka-streams-upgrade-system-tests-23": "org.apache.kafka", - "kafka-streams-upgrade-system-tests-24": "org.apache.kafka", - "kafka-streams-upgrade-system-tests-25": "org.apache.kafka", - "kafka-streams-upgrade-system-tests-26": "org.apache.kafka", - "kafka-streams-upgrade-system-tests-27": "org.apache.kafka", - "kafka-streams-upgrade-system-tests-28": "org.apache.kafka", - "kafka-streams-upgrade-system-tests-30": "org.apache.kafka", - "kafka-streams-upgrade-system-tests-31": "org.apache.kafka", - "kafka-streams-upgrade-system-tests-32": "org.apache.kafka", - "kafka-streams-upgrade-system-tests-33": "org.apache.kafka", - "kafka-streams-upgrade-system-tests-34": "org.apache.kafka", - "kafka-streams-upgrade-system-tests-35": "org.apache.kafka", - "kafka-tools": "org.apache.kafka", - "kafka-tools-api": "org.apache.kafka", - "kafka_2.10": "org.apache.kafka", - "kafka_2.11": "org.apache.kafka", - "kafka_2.12": "org.apache.kafka", - "kafka_2.13": "org.apache.kafka", - "kafka_2.8.0": "org.apache.kafka", - "kafka_2.8.2": "org.apache.kafka", - "kafka_2.9.1": "org.apache.kafka", - "kafka_2.9.2": "org.apache.kafka", + // the Eclipse-hosted groovy-eclipse-* artifacts stayed on codehaus past 4.0, and the org.apache.groovy modules + // below were added in 4.0+ and only exist under apache. See https://github.com/anchore/syft/issues/5311 + "groovy-eclipse-batch": "org.codehaus.groovy", + "groovy-eclipse-compiler": "org.codehaus.groovy", + "groovy-callsite": "org.apache.groovy", + "groovy-concurrent-java": "org.apache.groovy", + "groovy-contracts": "org.apache.groovy", + "groovy-csv": "org.apache.groovy", + "groovy-ginq": "org.apache.groovy", + "groovy-grape-ivy": "org.apache.groovy", + "groovy-grape-maven": "org.apache.groovy", + "groovy-http-builder": "org.apache.groovy", + "groovy-macro-library": "org.apache.groovy", + "groovy-markdown": "org.apache.groovy", + "groovy-reactor": "org.apache.groovy", + "groovy-rxjava": "org.apache.groovy", + "groovy-test-junit6": "org.apache.groovy", + "groovy-toml": "org.apache.groovy", + "groovy-typecheckers": "org.apache.groovy", + "kafka-clients": "org.apache.kafka", + "kafka-examples": "org.apache.kafka", + "kafka-group-coordinator": "org.apache.kafka", + "kafka-hadoop-consumer": "org.apache.kafka", + "kafka-hadoop-producer": "org.apache.kafka", + "kafka-java-examples": "org.apache.kafka", + "kafka-log4j-appender": "org.apache.kafka", + "kafka-metadata": "org.apache.kafka", + "kafka-perf_2.10": "org.apache.kafka", + "kafka-perf_2.8.0": "org.apache.kafka", + "kafka-perf_2.8.2": "org.apache.kafka", + "kafka-perf_2.9.1": "org.apache.kafka", + "kafka-perf_2.9.2": "org.apache.kafka", + "kafka-raft": "org.apache.kafka", + "kafka-server-common": "org.apache.kafka", + "kafka-shell": "org.apache.kafka", + "kafka-storage": "org.apache.kafka", + "kafka-storage-api": "org.apache.kafka", + "kafka-streams": "org.apache.kafka", + "kafka-streams-examples": "org.apache.kafka", + "kafka-streams-scala_2.11": "org.apache.kafka", + "kafka-streams-scala_2.12": "org.apache.kafka", + "kafka-streams-scala_2.13": "org.apache.kafka", + "kafka-streams-test-utils": "org.apache.kafka", + "kafka-streams-upgrade-system-tests-0100": "org.apache.kafka", + "kafka-streams-upgrade-system-tests-0101": "org.apache.kafka", + "kafka-streams-upgrade-system-tests-0102": "org.apache.kafka", + "kafka-streams-upgrade-system-tests-0110": "org.apache.kafka", + "kafka-streams-upgrade-system-tests-10": "org.apache.kafka", + "kafka-streams-upgrade-system-tests-11": "org.apache.kafka", + "kafka-streams-upgrade-system-tests-20": "org.apache.kafka", + "kafka-streams-upgrade-system-tests-21": "org.apache.kafka", + "kafka-streams-upgrade-system-tests-22": "org.apache.kafka", + "kafka-streams-upgrade-system-tests-23": "org.apache.kafka", + "kafka-streams-upgrade-system-tests-24": "org.apache.kafka", + "kafka-streams-upgrade-system-tests-25": "org.apache.kafka", + "kafka-streams-upgrade-system-tests-26": "org.apache.kafka", + "kafka-streams-upgrade-system-tests-27": "org.apache.kafka", + "kafka-streams-upgrade-system-tests-28": "org.apache.kafka", + "kafka-streams-upgrade-system-tests-30": "org.apache.kafka", + "kafka-streams-upgrade-system-tests-31": "org.apache.kafka", + "kafka-streams-upgrade-system-tests-32": "org.apache.kafka", + "kafka-streams-upgrade-system-tests-33": "org.apache.kafka", + "kafka-streams-upgrade-system-tests-34": "org.apache.kafka", + "kafka-streams-upgrade-system-tests-35": "org.apache.kafka", + "kafka-tools": "org.apache.kafka", + "kafka-tools-api": "org.apache.kafka", + "kafka_2.10": "org.apache.kafka", + "kafka_2.11": "org.apache.kafka", + "kafka_2.12": "org.apache.kafka", + "kafka_2.13": "org.apache.kafka", + "kafka_2.8.0": "org.apache.kafka", + "kafka_2.8.2": "org.apache.kafka", + "kafka_2.9.1": "org.apache.kafka", + "kafka_2.9.2": "org.apache.kafka", // legacy "Jackson 1.x" (aka "jackson-asl") artifacts predate the convention of embedding - // META-INF/maven/.../pom.properties in the jar, so groupIDFromKnownPackageList is the only + // META-INF/maven/.../pom.properties in the jar, so ArtifactIDToGroupID is the only // way to recover the correct group ID for jars built without that metadata (e.g. Ant-built // jars from before ~2014). Without this, the group ID falls back to the artifact name itself, // producing purls that do not match the vulnerability database's namespace. @@ -1940,3 +1926,83 @@ var DefaultArtifactIDToGroupID = map[string]string{ "jackson-xc": "org.codehaus.jackson", "jackson-smile": "org.codehaus.jackson", } + +// groupIDMigration describes an artifact that moved from one group ID to another at a major version. +type groupIDMigration struct { + before, after string + atMajor int +} + +// groovy moved from org.codehaus.groovy to org.apache.groovy at 4.0 +var groovyMigration = groupIDMigration{before: "org.codehaus.groovy", after: "org.apache.groovy", atMajor: 4} + +// migratedArtifactIDToGroupID holds artifacts whose group ID depends on the version. It takes precedence over +// defaultArtifactIDToGroupID; use ArtifactIDToGroupID rather than reading it directly. +var migratedArtifactIDToGroupID = map[string]groupIDMigration{ + // groovy artifacts published under both coordinates. Without these, jars that ship no pom metadata get a + // stale codehaus group or one guessed from the per-module Automatic-Module-Name (e.g. org.apache.groovy.json). + // See https://github.com/anchore/syft/issues/5311 + "groovy": groovyMigration, + "groovy-all": groovyMigration, + "groovy-all-jdk14": groovyMigration, + "groovy-all-minimal": groovyMigration, + "groovy-all-tests": groovyMigration, + "groovy-ant": groovyMigration, + "groovy-astbuilder": groovyMigration, + "groovy-backports-compat23": groovyMigration, + "groovy-binary": groovyMigration, + "groovy-bom": groovyMigration, + "groovy-bsf": groovyMigration, + "groovy-cli-commons": groovyMigration, + "groovy-cli-picocli": groovyMigration, + "groovy-console": groovyMigration, + "groovy-datetime": groovyMigration, + "groovy-dateutil": groovyMigration, + "groovy-docgenerator": groovyMigration, + "groovy-groovydoc": groovyMigration, + "groovy-groovysh": groovyMigration, + "groovy-jaxb": groovyMigration, + "groovy-jdk14": groovyMigration, + "groovy-jmx": groovyMigration, + "groovy-json": groovyMigration, + "groovy-json-direct": groovyMigration, + "groovy-jsr223": groovyMigration, + "groovy-macro": groovyMigration, + "groovy-nio": groovyMigration, + "groovy-servlet": groovyMigration, + "groovy-sql": groovyMigration, + "groovy-swing": groovyMigration, + "groovy-templates": groovyMigration, + "groovy-test": groovyMigration, + "groovy-test-junit5": groovyMigration, + "groovy-testng": groovyMigration, + "groovy-tests-vm8": groovyMigration, + "groovy-xml": groovyMigration, + "groovy-xmlrpc": groovyMigration, + "groovy-yaml": groovyMigration, +} + +// ArtifactIDToGroupID returns the known group ID for the given artifact ID at the given version. For migrated +// artifacts an empty or unparseable version resolves to the pre-migration group ID. +func ArtifactIDToGroupID(artifactID, version string) (string, bool) { + if m, ok := migratedArtifactIDToGroupID[artifactID]; ok { + if majorVersion(version) >= m.atMajor { + return m.after, true + } + return m.before, true + } + groupID, ok := defaultArtifactIDToGroupID[artifactID] + return groupID, ok +} + +// majorVersion returns the leading numeric component of a version (e.g. "4.0.33" and "4-SNAPSHOT" +// resolve to 4), or 0 when the version does not start with a number. +func majorVersion(version string) int { + leading, _, _ := strings.Cut(version, ".") + leading, _, _ = strings.Cut(leading, "-") + n, err := strconv.Atoi(leading) + if err != nil { + return 0 + } + return n +} diff --git a/syft/pkg/cataloger/internal/cpegenerate/java_groupid_map_test.go b/syft/pkg/cataloger/internal/cpegenerate/java_groupid_map_test.go new file mode 100644 index 000000000..8232f0e93 --- /dev/null +++ b/syft/pkg/cataloger/internal/cpegenerate/java_groupid_map_test.go @@ -0,0 +1,49 @@ +package cpegenerate + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestArtifactIDToGroupID(t *testing.T) { + tests := []struct { + name string + artifactID string + version string + want string + wantOK bool + }{ + {name: "unconditional entry", artifactID: "ant-antlr", version: "1.10.0", want: "org.apache.ant", wantOK: true}, + {name: "unknown artifact", artifactID: "not-a-real-artifact", version: "1.0.0"}, + {name: "groovy 4 migrates to apache", artifactID: "groovy", version: "4.0.33", want: "org.apache.groovy", wantOK: true}, + {name: "groovy 3 stays on codehaus", artifactID: "groovy", version: "3.0.22", want: "org.codehaus.groovy", wantOK: true}, + {name: "groovy without a version falls back to codehaus", artifactID: "groovy-json", want: "org.codehaus.groovy", wantOK: true}, + {name: "groovy-eclipse stays on codehaus at 4.x", artifactID: "groovy-eclipse-batch", version: "4.0.33-02", want: "org.codehaus.groovy", wantOK: true}, + {name: "apache-only groovy module", artifactID: "groovy-ginq", version: "4.0.33", want: "org.apache.groovy", wantOK: true}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, ok := ArtifactIDToGroupID(tt.artifactID, tt.version) + assert.Equal(t, tt.wantOK, ok) + assert.Equal(t, tt.want, got) + }) + } +} + +func Test_majorVersion(t *testing.T) { + tests := map[string]int{ + "4.0.33": 4, + "4.0.0-alpha-1": 4, + "4-SNAPSHOT": 4, + "4": 4, + "3.0.22": 3, + "v4.0.33": 0, + "": 0, + } + for version, expect := range tests { + t.Run(version, func(t *testing.T) { + assert.Equal(t, expect, majorVersion(version)) + }) + } +} diff --git a/syft/pkg/cataloger/internal/cpegenerate/java_test.go b/syft/pkg/cataloger/internal/cpegenerate/java_test.go index 6529598c7..6b9211a41 100644 --- a/syft/pkg/cataloger/internal/cpegenerate/java_test.go +++ b/syft/pkg/cataloger/internal/cpegenerate/java_test.go @@ -400,6 +400,16 @@ func Test_groupIDsFromJavaPackage(t *testing.T) { }, expects: nil, }, + { + // pom-less groovy 4 jars (all real ones) must resolve to the apache group, same as the purl + name: "groovy 4 without pom metadata", + pkg: pkg.Package{ + Name: "groovy", + Version: "4.0.33", + Metadata: pkg.JavaArchive{Manifest: &pkg.JavaManifest{}}, + }, + expects: []string{"org.apache.groovy"}, + }, { name: "no manifest or pom info", pkg: pkg.Package{ @@ -562,7 +572,7 @@ func Test_groupIDsFromJavaManifest(t *testing.T) { for _, test := range tests { t.Run(test.name, func(t *testing.T) { - got := groupIDsFromJavaManifest(test.name, &test.manifest) + got := groupIDsFromJavaManifest(test.name, "", &test.manifest) require.Equal(t, test.expected, got) }) } diff --git a/syft/pkg/cataloger/internal/dotnet/bundle/bundle.go b/syft/pkg/cataloger/internal/dotnet/bundle/bundle.go index 3bead6fb9..434b1d2ac 100644 --- a/syft/pkg/cataloger/internal/dotnet/bundle/bundle.go +++ b/syft/pkg/cataloger/internal/dotnet/bundle/bundle.go @@ -209,7 +209,7 @@ func read7BitEncodedInt(r io.Reader) (int, error) { shift := 0 for { var b [1]byte - if _, err := r.Read(b[:]); err != nil { + if _, err := io.ReadFull(r, b[:]); err != nil { return 0, err } result |= int(b[0]&0x7F) << shift diff --git a/syft/pkg/cataloger/java/archive_parser.go b/syft/pkg/cataloger/java/archive_parser.go index 4232d07a1..2e520594b 100644 --- a/syft/pkg/cataloger/java/archive_parser.go +++ b/syft/pkg/cataloger/java/archive_parser.go @@ -359,7 +359,7 @@ func (j *archiveParser) discoverNameVersionLicense(ctx context.Context, manifest // findLicenseFromJavaMetadata attempts to find license information from all available maven metadata properties and pom info func (j *archiveParser) findLicenseFromJavaMetadata(ctx context.Context, groupID, artifactID, version string, parsedPom *parsedPomProject, manifest *pkg.JavaManifest) []pkg.License { if groupID == "" { - if gID := groupIDFromJavaMetadata(artifactID, pkg.JavaArchive{Manifest: manifest}); gID != "" { + if gID := groupIDFromJavaMetadata(artifactID, version, pkg.JavaArchive{Manifest: manifest}); gID != "" { groupID = gID } } @@ -825,7 +825,7 @@ func newPackageFromMavenData(ctx context.Context, r *maven.Resolver, pomProperti vPathSuffix := "" groupID := "" if parentMetadata, ok := parentPkg.Metadata.(pkg.JavaArchive); ok { - groupID = groupIDFromJavaMetadata(parentPkg.Name, parentMetadata) + groupID = groupIDFromJavaMetadata(parentPkg.Name, parentPkg.Version, parentMetadata) } parentKey := fmt.Sprintf("%s:%s:%s", groupID, parentPkg.Name, parentPkg.Version) @@ -902,8 +902,8 @@ func packageIdentitiesMatch(p pkg.Package, parentPkg *pkg.Package) bool { } // try to determine identity with the metadata - groupID := groupIDFromJavaMetadata(p.Name, metadata) - parentGroupID := groupIDFromJavaMetadata(parentPkg.Name, parentMetadata) + groupID := groupIDFromJavaMetadata(p.Name, p.Version, metadata) + parentGroupID := groupIDFromJavaMetadata(parentPkg.Name, parentPkg.Version, parentMetadata) if uniquePkgKey(groupID, &p) == uniquePkgKey(parentGroupID, parentPkg) { return true } diff --git a/syft/pkg/cataloger/java/package_url.go b/syft/pkg/cataloger/java/package_url.go index e47e9a7e6..477b89a46 100644 --- a/syft/pkg/cataloger/java/package_url.go +++ b/syft/pkg/cataloger/java/package_url.go @@ -12,7 +12,7 @@ import ( func packageURL(name, version string, metadata pkg.JavaArchive) string { var groupID = name - if gID := groupIDFromJavaMetadata(name, metadata); gID != "" { + if gID := groupIDFromJavaMetadata(name, version, metadata); gID != "" { groupID = gID } @@ -30,9 +30,9 @@ func packageURL(name, version string, metadata pkg.JavaArchive) string { // The order of precedence is: // 1. The group ID from the POM properties // 2. The group ID from the POM project -// 3. The group ID from a select map of known group IDs +// 3. The group ID from a select map of known group IDs (see cpegenerate.ArtifactIDToGroupID) // 4. The group ID from the Java manifest -func groupIDFromJavaMetadata(pkgName string, metadata pkg.JavaArchive) (groupID string) { +func groupIDFromJavaMetadata(pkgName, version string, metadata pkg.JavaArchive) (groupID string) { if groupID = groupIDFromPomProperties(metadata.PomProperties); groupID != "" { return groupID } @@ -41,8 +41,8 @@ func groupIDFromJavaMetadata(pkgName string, metadata pkg.JavaArchive) (groupID return groupID } - if groupID = groupIDFromKnownPackageList(pkgName); groupID != "" { - return groupID + if gID, ok := cpegenerate.ArtifactIDToGroupID(pkgName, version); ok { + return gID } if groupID = groupIDFromJavaManifest(metadata.Manifest); groupID != "" { @@ -52,13 +52,6 @@ func groupIDFromJavaMetadata(pkgName string, metadata pkg.JavaArchive) (groupID return groupID } -func groupIDFromKnownPackageList(pkgName string) (groupID string) { - if groupID, ok := cpegenerate.DefaultArtifactIDToGroupID[pkgName]; ok { - return groupID - } - return groupID -} - func groupIDFromJavaManifest(manifest *pkg.JavaManifest) (groupID string) { if manifest == nil { return groupID diff --git a/syft/pkg/cataloger/java/package_url_test.go b/syft/pkg/cataloger/java/package_url_test.go index 5bb92fd00..a6fa7d973 100644 --- a/syft/pkg/cataloger/java/package_url_test.go +++ b/syft/pkg/cataloger/java/package_url_test.go @@ -132,6 +132,26 @@ func Test_packageURL(t *testing.T) { }, expect: "pkg:maven/parent/example-java-app-maven@0.1.0", }, + { + // regression for github.com/anchore/syft/issues/5311 + name: "groovy 4 without pom metadata", + pkg: pkg.Package{ + Name: "groovy", + Version: "4.0.33", + Language: pkg.Java, + Type: pkg.JavaPkg, + Metadata: pkg.JavaArchive{ + VirtualPath: "groovy-4.0.33.jar", + Manifest: &pkg.JavaManifest{ + Main: []pkg.KeyValue{ + {Key: "Bundle-SymbolicName", Value: "groovy"}, + {Key: "Automatic-Module-Name", Value: "org.apache.groovy.core"}, + }, + }, + }, + }, + expect: "pkg:maven/org.apache.groovy/groovy@4.0.33", + }, } for _, tt := range tests { t.Run(tt.expect, func(t *testing.T) { @@ -144,6 +164,7 @@ func Test_groupIDFromJavaMetadata(t *testing.T) { tests := []struct { name string pkgName string + version string metadata pkg.JavaArchive expect string }{ @@ -218,6 +239,38 @@ func Test_groupIDFromJavaMetadata(t *testing.T) { metadata: pkg.JavaArchive{}, expect: "org.apache.derby", }, + { + // regression for github.com/anchore/syft/issues/5311: groovy 4+ moved to org.apache.groovy, but + // jars without pom metadata fall through to the known package list, which keeps the codehaus-era + // coordinates. The map must also beat the per-module Automatic-Module-Name in the manifest. + name: "known package list groovy 4 beats the manifest", + pkgName: "groovy", + version: "4.0.33", + metadata: pkg.JavaArchive{ + Manifest: &pkg.JavaManifest{ + Main: []pkg.KeyValue{ + {Key: "Bundle-SymbolicName", Value: "groovy"}, + {Key: "Automatic-Module-Name", Value: "org.apache.groovy.core"}, + }, + }, + }, + expect: "org.apache.groovy", + }, + { + // modules only published under the apache coordinates would otherwise take the group ID + // from the per-module Automatic-Module-Name (org.apache.groovy.ginq) + name: "known package list apache-only groovy module", + pkgName: "groovy-ginq", + version: "4.0.33", + metadata: pkg.JavaArchive{ + Manifest: &pkg.JavaManifest{ + Main: []pkg.KeyValue{ + {Key: "Automatic-Module-Name", Value: "org.apache.groovy.ginq"}, + }, + }, + }, + expect: "org.apache.groovy", + }, { name: "java manifest", metadata: pkg.JavaArchive{ @@ -241,7 +294,7 @@ func Test_groupIDFromJavaMetadata(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - assert.Equal(t, tt.expect, groupIDFromJavaMetadata(tt.pkgName, tt.metadata)) + assert.Equal(t, tt.expect, groupIDFromJavaMetadata(tt.pkgName, tt.version, tt.metadata)) }) } } diff --git a/syft/pkg/cataloger/java/parse_gradle_lockfile.go b/syft/pkg/cataloger/java/parse_gradle_lockfile.go index 4ba86afb3..4e9e8c694 100644 --- a/syft/pkg/cataloger/java/parse_gradle_lockfile.go +++ b/syft/pkg/cataloger/java/parse_gradle_lockfile.go @@ -1,10 +1,11 @@ package java import ( - "bufio" "context" + "fmt" "strings" + "github.com/anchore/syft/internal" "github.com/anchore/syft/syft/artifact" "github.com/anchore/syft/syft/file" "github.com/anchore/syft/syft/pkg" @@ -22,7 +23,7 @@ func parseGradleLockfile(_ context.Context, _ file.Resolver, _ *generic.Environm var pkgs []pkg.Package // Create a new scanner to read the file - scanner := bufio.NewScanner(reader) + scanner := internal.NewLineScanner(reader) // Create slices to hold the dependencies and plugins dependencies := []lockfileDependency{} @@ -45,6 +46,7 @@ func parseGradleLockfile(_ context.Context, _ file.Resolver, _ *generic.Environm dependencies = append(dependencies, dep) } } + scanErr := scanner.Err() // map the dependencies for _, dep := range dependencies { @@ -72,5 +74,8 @@ func parseGradleLockfile(_ context.Context, _ file.Resolver, _ *generic.Environm pkgs = append(pkgs, mappedPkg) } + if scanErr != nil { + return pkgs, nil, fmt.Errorf("unable to read gradle lockfile: %w", scanErr) + } return pkgs, nil, nil } diff --git a/syft/pkg/cataloger/java/parse_gradle_lockfile_test.go b/syft/pkg/cataloger/java/parse_gradle_lockfile_test.go index d6fc4c2a8..a9208a1b8 100644 --- a/syft/pkg/cataloger/java/parse_gradle_lockfile_test.go +++ b/syft/pkg/cataloger/java/parse_gradle_lockfile_test.go @@ -1,8 +1,15 @@ package java import ( + "bufio" + "context" + "io" + "strings" "testing" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/anchore/syft/syft/file" "github.com/anchore/syft/syft/pkg" "github.com/anchore/syft/syft/pkg/cataloger/internal/pkgtest" @@ -69,3 +76,14 @@ func Test_parserGradleLockfile(t *testing.T) { }) } } + +func Test_parserGradleLockfile_lineTooLong(t *testing.T) { + // packages before an oversized line are kept, and the scan error is still reported + contents := "org.example:before:1.0=compileClasspath\n" + strings.Repeat("a", 2<<20) + "\norg.example:after:2.0=compileClasspath\n" + reader := file.NewLocationReadCloser(file.NewLocation("gradle.lockfile"), io.NopCloser(strings.NewReader(contents))) + + pkgs, _, err := parseGradleLockfile(context.Background(), nil, nil, reader) + require.ErrorIs(t, err, bufio.ErrTooLong) + require.Len(t, pkgs, 1) + assert.Equal(t, "before", pkgs[0].Name) +} diff --git a/syft/pkg/cataloger/java/parse_java_manifest.go b/syft/pkg/cataloger/java/parse_java_manifest.go index cb7945eca..0c1e3129a 100644 --- a/syft/pkg/cataloger/java/parse_java_manifest.go +++ b/syft/pkg/cataloger/java/parse_java_manifest.go @@ -1,12 +1,12 @@ package java import ( - "bufio" "fmt" "io" "strings" "unicode" + "github.com/anchore/syft/internal" "github.com/anchore/syft/internal/log" "github.com/anchore/syft/syft/pkg" ) @@ -26,7 +26,7 @@ func parseJavaManifest(path string, reader io.Reader) (*pkg.JavaManifest, error) } var lastKey string - scanner := bufio.NewScanner(reader) + scanner := internal.NewLineScanner(reader) for scanner.Scan() { line := scanner.Text() diff --git a/syft/pkg/cataloger/java/parse_pom_properties.go b/syft/pkg/cataloger/java/parse_pom_properties.go index 8ef3676be..b4704b099 100644 --- a/syft/pkg/cataloger/java/parse_pom_properties.go +++ b/syft/pkg/cataloger/java/parse_pom_properties.go @@ -1,13 +1,13 @@ package java import ( - "bufio" "fmt" "io" "strings" "github.com/go-viper/mapstructure/v2" + "github.com/anchore/syft/internal" "github.com/anchore/syft/syft/pkg" ) @@ -16,7 +16,7 @@ const pomPropertiesGlob = "**/*pom.properties" func parsePomProperties(path string, reader io.Reader) (*pkg.JavaPomProperties, error) { var props pkg.JavaPomProperties propMap := make(map[string]string) - scanner := bufio.NewScanner(reader) + scanner := internal.NewLineScanner(reader) for scanner.Scan() { line := scanner.Text() diff --git a/syft/pkg/cataloger/javascript/parse_yarn_lock.go b/syft/pkg/cataloger/javascript/parse_yarn_lock.go index 9213be760..7e65acf68 100644 --- a/syft/pkg/cataloger/javascript/parse_yarn_lock.go +++ b/syft/pkg/cataloger/javascript/parse_yarn_lock.go @@ -206,28 +206,16 @@ func parseYarnV1LockFile(reader io.ReadCloser) ([]yarnPackage, error) { continue } if strings.HasPrefix(line, " ") && !strings.HasPrefix(line, " ") { - line = strings.Trim(line, " ") - array := strings.Split(line, " ") - switch array[0] { - case "version": - pkg.Version = strings.Trim(array[1], "\"") - case "resolved": - name, version, resolved := findResolvedPackageAndVersion(line) - if name != "" && version != "" && resolved != "" { - pkg.Name = name - pkg.Version = version - pkg.Resolved = resolved - } else { - pkg.Resolved = strings.Trim(array[1], "\"") - } - case "integrity": - pkg.Integrity = strings.Trim(array[1], "\"") - } + applyYarnV1Field(&pkg, strings.Trim(line, " ")) continue } if strings.HasPrefix(line, " ") { line = strings.Trim(line, " ") array := strings.Split(line, " ") + if len(array) < 2 { + // a dependency without a version constraint is malformed; skip it rather than guess + continue + } dependencyName := strings.Trim(array[0], "\"") dependencyVersion := strings.Trim(array[1], "\"") dependencies[dependencyName] = dependencyVersion @@ -244,6 +232,31 @@ func parseYarnV1LockFile(reader io.ReadCloser) ([]yarnPackage, error) { return pkgs, nil } +// applyYarnV1Field sets the package field described by a single (already trimmed) yarn v1 field line. +func applyYarnV1Field(pkg *yarnPackage, line string) { + array := strings.Split(line, " ") + // a field line may be missing its value in a malformed lockfile + var value string + if len(array) > 1 { + value = strings.Trim(array[1], "\"") + } + switch array[0] { + case "version": + pkg.Version = value + case "resolved": + name, version, resolved := findResolvedPackageAndVersion(line) + if name != "" && version != "" && resolved != "" { + pkg.Name = name + pkg.Version = version + pkg.Resolved = resolved + } else { + pkg.Resolved = value + } + case "integrity": + pkg.Integrity = value + } +} + func parseYarnLockYaml(reader io.ReadCloser) ([]yarnPackage, error) { var lockfile = map[string]yarnV2PackageEntry{} if err := yaml.NewDecoder(reader, yaml.AllowDuplicateMapKey()).Decode(&lockfile); err != nil { diff --git a/syft/pkg/cataloger/javascript/parse_yarn_lock_test.go b/syft/pkg/cataloger/javascript/parse_yarn_lock_test.go index 93a24c725..7c18eab01 100644 --- a/syft/pkg/cataloger/javascript/parse_yarn_lock_test.go +++ b/syft/pkg/cataloger/javascript/parse_yarn_lock_test.go @@ -6,9 +6,11 @@ import ( "net/http" "net/http/httptest" "os" + "strings" "testing" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" "github.com/anchore/syft/syft/artifact" "github.com/anchore/syft/syft/file" @@ -1091,3 +1093,35 @@ func setupYarnRegistry() (mux *http.ServeMux, serverURL string, teardown func()) return mux, server.URL, server.Close } + +func TestParseYarnV1LockFile_missingValues(t *testing.T) { + tests := []struct { + name string + content string + want []yarnPackage + }{ + { + // a dependency line without a constraint is dropped rather than indexed past its end + name: "dependency without version", + content: "# yarn lockfile v1\n\na@1:\n version \"1\"\n dependencies:\n b\n\n", + want: []yarnPackage{{Name: "a", Version: "1", Dependencies: map[string]string{}}}, + }, + { + // a field line without a value leaves the field empty + name: "field without value", + content: "# yarn lockfile v1\n\na@1:\n version\n\n", + want: []yarnPackage{{Name: "a", Dependencies: map[string]string{}}}, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + var got []yarnPackage + var err error + require.NotPanics(t, func() { + got, err = parseYarnV1LockFile(io.NopCloser(strings.NewReader(tt.content))) + }) + require.NoError(t, err) + assert.Equal(t, tt.want, got) + }) + } +} diff --git a/syft/pkg/cataloger/lua/rockspec_parser.go b/syft/pkg/cataloger/lua/rockspec_parser.go index 14317b5f0..8c42d8f8e 100644 --- a/syft/pkg/cataloger/lua/rockspec_parser.go +++ b/syft/pkg/cataloger/lua/rockspec_parser.go @@ -2,6 +2,7 @@ package lua import ( "bytes" + "errors" "fmt" "io" @@ -33,6 +34,10 @@ func (r rockspecNode) String() string { return "" } +var errUnexpectedEnd = errors.New("unexpected end of input") + +var errTooDeep = fmt.Errorf("nesting deeper than %d levels", parsing.MaxDepth) + var noReturn = rockspec{ value: nil, } @@ -46,10 +51,11 @@ func parseRockspecData(reader io.Reader) (rockspec, error) { i := 0 locals := make(map[string]string) - blocks, err := parseRockspecBlock(data, &i, locals) + blocks, err := parseRockspecBlock(data, &i, locals, 0) if err != nil { - return noReturn, err + // decorate once here rather than at every nesting level, which copied the input once per level + return noReturn, fmt.Errorf("%w\n%s", err, parsing.PrintError(data, i)) } return rockspec{ @@ -57,9 +63,12 @@ func parseRockspecData(reader io.Reader) (rockspec, error) { }, nil } -func parseRockspecBlock(data []byte, i *int, locals map[string]string) ([]rockspecNode, error) { +func parseRockspecBlock(data []byte, i *int, locals map[string]string, depth int) ([]rockspecNode, error) { + if depth > parsing.MaxDepth { + return nil, errTooDeep + } var out []rockspecNode - var iterator func(data []byte, i *int, locals map[string]string) (*rockspecNode, error) + var iterator func(data []byte, i *int, locals map[string]string, depth int) (*rockspecNode, error) parsing.SkipWhitespace(data, i) @@ -92,9 +101,9 @@ func parseRockspecBlock(data []byte, i *int, locals map[string]string) ([]rocksp } for *i < len(data) { - item, err := iterator(data, i, locals) + item, err := iterator(data, i, locals, depth) if err != nil { - return nil, fmt.Errorf("%w\n%s", err, parsing.PrintError(data, *i)) + return nil, err } parsing.SkipWhitespace(data, i) @@ -114,7 +123,7 @@ func parseRockspecBlock(data []byte, i *int, locals map[string]string) ([]rocksp } //nolint:funlen, gocognit -func parseRockspecNode(data []byte, i *int, locals map[string]string) (*rockspecNode, error) { +func parseRockspecNode(data []byte, i *int, locals map[string]string, depth int) (*rockspecNode, error) { parsing.SkipWhitespace(data, i) if *i >= len(data) { @@ -212,7 +221,7 @@ func parseRockspecNode(data []byte, i *int, locals map[string]string) (*rockspec *i = offset parsing.SkipWhitespace(data, i) - obj, err := parseRockspecBlock(data, i, locals) + obj, err := parseRockspecBlock(data, i, locals, depth+1) if err != nil { return nil, err @@ -247,6 +256,9 @@ func parseRockspecNode(data []byte, i *int, locals map[string]string) (*rockspec } value := str.String() + if *i >= len(data) { + return nil, errUnexpectedEnd + } c = data[*i] if c != ']' { @@ -271,7 +283,7 @@ func parseRockspecNode(data []byte, i *int, locals map[string]string) (*rockspec }, nil } -func parseRockspecListItem(data []byte, i *int, locals map[string]string) (*rockspecNode, error) { +func parseRockspecListItem(data []byte, i *int, locals map[string]string, _ int) (*rockspecNode, error) { parsing.SkipWhitespace(data, i) if *i >= len(data) { @@ -378,6 +390,9 @@ out: if err != nil { return "", err } + if *i >= len(data) { + return "", errUnexpectedEnd + } c = data[*i] if c != ']' { return "", fmt.Errorf("unterminated literal at %d", *i) @@ -394,6 +409,9 @@ out: } func parseRockspecString(data []byte, i *int, _ map[string]string) (*rockspecNode, error) { + if *i >= len(data) { + return nil, errUnexpectedEnd + } delim := data[*i] var endDelim byte switch delim { @@ -434,6 +452,39 @@ func parseComment(data []byte, i *int) { } } +// parseLocalValue parses a single value on the right-hand side of a local assignment. Expressions +// resolve to an empty string, and bare references resolve through locals. The caller must ensure *i < len(data). +func parseLocalValue(data []byte, i *int, locals map[string]string) (string, error) { + c := data[*i] + + switch c { + case '"', '\'': + value, err := parseRockspecString(data, i, locals) + if err != nil { + return "", err + } + return value.value.(string), nil + default: + ref, err := parseRockspecLiteral(data, i, locals) + if err != nil { + return "", err + } + + // skip if it's an expression + skipWhitespaceNoNewLine(data, i) + if *i >= len(data) { + return "", errUnexpectedEnd + } + c := data[*i] + + if c != '\n' && c != '\r' { + skipExpression(data, i) + return "", nil + } + return locals[ref], nil + } +} + //nolint:funlen func parseLocal(data []byte, i *int, locals map[string]string) error { keys := []string{} @@ -459,6 +510,9 @@ keys: keys = append(keys, key) parsing.SkipWhitespace(data, i) + if *i >= len(data) { + return errUnexpectedEnd + } c := data[*i] @@ -477,42 +531,22 @@ keys: values: for { skipWhitespaceNoNewLine(data, i) - - c := data[*i] - - switch c { - case '"', '\'': - value, err := parseRockspecString(data, i, locals) - - if err != nil { - return err - } - values = append(values, value.value.(string)) - default: - ref, err := parseRockspecLiteral(data, i, locals) - if err != nil { - return err - } - - // Skip if it's an expression - skipWhitespaceNoNewLine(data, i) - c := data[*i] - - var value string - - if c != '\n' && c != '\r' { - skipExpression(data, i) - value = "" - } else { - value = locals[ref] - } - - values = append(values, value) + if *i >= len(data) { + return errUnexpectedEnd } - skipWhitespaceNoNewLine(data, i) + value, err := parseLocalValue(data, i, locals) + if err != nil { + return err + } + values = append(values, value) - c = data[*i] + skipWhitespaceNoNewLine(data, i) + if *i >= len(data) { + return errUnexpectedEnd + } + + c := data[*i] switch c { case ',': diff --git a/syft/pkg/cataloger/lua/rockspec_parser_test.go b/syft/pkg/cataloger/lua/rockspec_parser_test.go index aca545e00..6a7bd70f7 100644 --- a/syft/pkg/cataloger/lua/rockspec_parser_test.go +++ b/syft/pkg/cataloger/lua/rockspec_parser_test.go @@ -2,6 +2,7 @@ package lua import ( "bytes" + "strings" "testing" "github.com/stretchr/testify/assert" @@ -282,3 +283,35 @@ hello = foo.. `, }) } } + +func TestParseRockspec_deepNesting(t *testing.T) { + // deep input overflowed the goroutine stack (fatal, not recoverable) before nesting was bounded, and each level + // used to append its own copy of the input to the error + _, err := parseRockspecData(strings.NewReader(strings.Repeat("a={", 3<<20))) + require.ErrorIs(t, err, errTooDeep) + assert.Less(t, len(err.Error()), 1024) +} + +func Test_parseRockspecData_truncated(t *testing.T) { + // each of these ends mid-statement, and used to read past the end of the input + tests := []struct { + name string + content string + }{ + {name: "unterminated long string value", content: "a=[[x]"}, + {name: "unterminated index literal", content: `a["x"`}, + {name: "local missing equals", content: "local a"}, + {name: "local missing value", content: "local a="}, + {name: "local reference at end of input", content: "local a=b"}, + {name: "local string at end of input", content: `local a="x"`}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + var err error + require.NotPanics(t, func() { + _, err = parseRockspecData(strings.NewReader(tt.content)) + }) + require.Error(t, err) + }) + } +} diff --git a/syft/pkg/cataloger/nix/db_cataloger.go b/syft/pkg/cataloger/nix/db_cataloger.go index 615e213ef..825895b69 100644 --- a/syft/pkg/cataloger/nix/db_cataloger.go +++ b/syft/pkg/cataloger/nix/db_cataloger.go @@ -95,7 +95,7 @@ func (c dbCataloger) selectDBParser(dbLocation file.Location, resolver file.Reso scanner := bufio.NewScanner(schemaContents) if !scanner.Scan() { - log.WithFields("path", loc.RealPath).Tracef("failed to read Nix database schema file, assuming %d", defaultSchema) + log.WithFields("path", loc.RealPath, "error", scanner.Err()).Tracef("failed to read Nix database schema file, assuming %d", defaultSchema) return c.schemaProcessor[defaultSchema], 0 } diff --git a/syft/pkg/cataloger/parsers_fuzz_test.go b/syft/pkg/cataloger/parsers_fuzz_test.go new file mode 100644 index 000000000..2d1ef1056 --- /dev/null +++ b/syft/pkg/cataloger/parsers_fuzz_test.go @@ -0,0 +1,400 @@ +package cataloger_test + +import ( + "context" + "encoding/json" + "errors" + "flag" + "fmt" + "go/ast" + "go/parser" + "go/token" + "io/fs" + "os" + "path/filepath" + "regexp" + "runtime" + "runtime/debug" + "runtime/metrics" + "runtime/pprof" + "sort" + "strconv" + "strings" + "testing" + "time" + + "github.com/bmatcuk/doublestar/v4" + + gosync "github.com/anchore/go-sync" + "github.com/anchore/syft/internal/capabilities" + "github.com/anchore/syft/internal/sbomsync" + "github.com/anchore/syft/internal/task" + "github.com/anchore/syft/internal/unknown" + "github.com/anchore/syft/syft/pkg" + "github.com/anchore/syft/syft/pkg/cataloger/internal/pkgtest" + "github.com/anchore/syft/syft/sbom" + "github.com/anchore/syft/syft/source" + "github.com/anchore/syft/syft/source/directorysource" +) + +const ( + // per-input limits. These are generous on purpose: the net is for inputs that take seconds or + // hundreds of MB, not for small inefficiencies. + fuzzTimeout = 10 * time.Second + // fuzzHangSlack is how long past fuzzTimeout a parser that ignores ctx gets before the process is taken down, + // so a ctx-aware parser always reports its timeout through the normal path first + fuzzHangSlack = 5 * time.Second + fuzzAllocBase = 256 << 20 + fuzzAllocPerByte = 1024 + fuzzMaxSeedBytes = 256 << 10 + fuzzMaxSeedsPerFn = 8 +) + +type fuzzTarget struct { + cataloger string + glob string + path string // a concrete relative path that matches glob + dir string // the cataloger package directory, relative to this package, for seeds + task task.Task +} + +func (ft fuzzTarget) String() string { return ft.cataloger + " " + ft.glob } + +// FuzzGenericParsers feeds one input at a time to each glob-selected parser of every generic cataloger, by writing +// it to a path the parser's glob matches and running the cataloger task over that directory. Catalogers that are not +// generic, and processor panics (which the generic cataloger only logs), are not covered. +// +// # What fuzzing is +// +// A fuzz test is a test whose input is generated. Go's fuzzer starts from a set of example inputs (the seed corpus), +// mutates them (flipping bytes, splicing pieces together, inserting values) and keeps any mutation that reaches code +// it has not reached before, so over time it works its way into parser branches that hand-written tests never try. +// Each input is one parser and one file's worth of bytes. An input fails when the parser: +// +// - panics (including a panic the generic cataloger recovers and reports as an unknown) +// - takes longer than fuzzTimeout +// - allocates much more than the input size warrants (see fuzzAllocBase and fuzzAllocPerByte) +// +// A parser returning an error is fine: rejecting bad input is the correct behavior. +// +// # Plain go test (no fuzzing) +// +// Without -fuzz, go test does not generate anything. It runs every seed once, like a table test, and reports them +// as subtests named seed#0, seed#1, ... (go names seeds by position; the failure message says which parser it was). +// The seeds are each parser's own testdata files plus a few inputs built from the string literals in its package, +// which give the parser its own key names and prefixes to trip over. This is what runs in CI, and takes seconds. +// It also replays every saved failure in testdata/fuzz/FuzzGenericParsers/ (see below). +// +// # Running the fuzzer +// +// Fuzzing is only done on demand, locally. Pick one parser with SYFT_FUZZ_TARGET, a regexp over +// " " (cataloger names are listed by `syft cataloger list`), and give it a time budget: +// +// SYFT_FUZZ_TARGET='^apk-db-cataloger ' go test ./syft/pkg/cataloger -run '^$' -fuzz FuzzGenericParsers -fuzztime 60s +// +// -run '^$' skips the ordinary tests, and -fuzztime bounds the run (without it the fuzzer runs until stopped). +// Without SYFT_FUZZ_TARGET every parser shares one run, so each gets a small slice of the time. The fuzzer's +// "new interesting" inputs are kept in the go build cache, not the repo, so later runs pick up where earlier ones +// left off. +// +// # When the fuzzer finds a failure +// +// It shrinks the input to a small version that still fails, prints the failure, and writes it to +// testdata/fuzz/FuzzGenericParsers/. That file is two lines of text, the parser name and the input bytes, so +// it can be read in review. From then on plain go test replays it as the subtest FuzzGenericParsers/, and it +// can be run alone: +// +// go test ./syft/pkg/cataloger -run 'FuzzGenericParsers/' +// +// The lifecycle of a failure: +// +// 1. fix the parser so the input is rejected with an error (or parsed correctly) rather than panicking or hanging +// 2. commit the saved file in the same change as the fix, as the regression test for it; do not commit one without +// its fix, since it fails every test run until the fix lands +// 3. fuzz that parser again, since one fix often uncovers the next failure behind it +// +// Saved files are kept indefinitely, like any other regression test. Delete one only when it no longer means +// anything (for example, the parser it names was removed). A test case in the parser's own _test.go is an equally +// good home for the input, and reads better there. +func FuzzGenericParsers(f *testing.F) { + targets := fuzzTargets(f) + if len(targets) == 0 { + f.Skip("no targets selected") + } + + // inputs name their target rather than index it, so a saved crasher keeps pointing at the same parser when + // the target list changes or is filtered + byName := map[string]fuzzTarget{} + for _, ft := range targets { + byName[ft.String()] = ft + for _, seed := range fuzzSeeds(ft) { + f.Add(ft.String(), seed) + } + } + + f.Fuzz(func(t *testing.T, name string, data []byte) { + ft, ok := byName[name] + if !ok { + // while fuzzing, the mutator rewrites names too, and a filter deliberately drops targets. Otherwise this + // is a saved input whose target was renamed or removed, and skipping it would silently drop a regression. + if os.Getenv("SYFT_FUZZ_TARGET") == "" && flag.Lookup("test.fuzz").Value.String() == "" { + t.Fatalf("saved fuzz input names unknown target %q: update or remove it", name) + } + t.Skip("not a selected target") + } + runFuzzTarget(t, ft, data) + }) +} + +func runFuzzTarget(t *testing.T, ft fuzzTarget, data []byte) { + root := t.TempDir() + p := filepath.Join(root, filepath.FromSlash(ft.path)) + if err := os.MkdirAll(filepath.Dir(p), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(p, data, 0o600); err != nil { + t.Fatal(err) + } + src, err := directorysource.NewFromPath(root) + if err != nil { + t.Fatal(err) + } + resolver, err := src.FileResolver(source.AllLayersScope) + if err != nil { + t.Fatal(err) + } + ctx, cancel := context.WithTimeout(pkgtest.Context(t), fuzzTimeout) + defer cancel() + + var before, after runtime.MemStats + runtime.ReadMemStats(&before) + + // the heap metric is process-wide and counts unswept garbage, so collect first and measure growth from there + heap := []metrics.Sample{{Name: "/memory/classes/heap/objects:bytes"}} + runtime.GC() + metrics.Read(heap) + heapBase := heap[0].Value.Uint64() + + done := make(chan error, 1) + go func() { + defer func() { + if r := recover(); r != nil { + done <- fmt.Errorf("panic: %v\n%s", r, debug.Stack()) + } + }() + done <- ft.task.Execute(ctx, resolver, sbomsync.NewBuilder(&sbom.SBOM{Artifacts: sbom.Artifacts{Packages: pkg.NewCollection()}})) + }() + + limit := uint64(fuzzAllocBase + fuzzAllocPerByte*len(data)) + tick := time.NewTicker(50 * time.Millisecond) + defer tick.Stop() + deadline := time.After(fuzzTimeout + fuzzHangSlack) +wait: + for { + select { + case err = <-done: + break wait + case <-tick.C: + // the after-the-fact TotalAlloc check below can't stop a parser that is on its way to exhausting the + // machine, so watch heap growth while it runs and take the process down once it passes the limit + metrics.Read(heap) + if grown := heap[0].Value.Uint64() - min(heapBase, heap[0].Value.Uint64()); grown > limit { + _ = pprof.Lookup("goroutine").WriteTo(os.Stderr, 2) + panic(fmt.Sprintf("%s: input of %d bytes grew the heap by %d MB (limit %d MB)", ft, len(data), grown>>20, limit>>20)) + } + case <-deadline: + // not every parser watches ctx, so a hung one keeps running and would skew every later input in this + // process. Dump where it is stuck and take the process down; the fuzz engine records that as a crasher. + _ = pprof.Lookup("goroutine").WriteTo(os.Stderr, 2) + panic(fmt.Sprintf("%s: input of %d bytes did not finish within %s", ft, len(data), fuzzTimeout+fuzzHangSlack)) + } + } + + if ctx.Err() != nil { + t.Fatalf("%s: input of %d bytes did not finish within %s", ft, len(data), fuzzTimeout) + } + + // a panic recovered further down (a go-sync PanicError, or the generic per-file recover) still surfaces as an error + if errors.As(err, &gosync.PanicError{}) || hasRecoveredPanic(err) { + t.Fatalf("%s: %v", ft, err) + } + + // TotalAlloc is cumulative, so this bounds churn rather than peak, and it is read after the fact: a single huge + // allocation kills the process before this check, which the fuzz engine still records as a crasher + runtime.ReadMemStats(&after) + if alloc := after.TotalAlloc - before.TotalAlloc; alloc > limit { + t.Fatalf("%s: input of %d bytes allocated %d MB (limit %d MB)", ft, len(data), alloc>>20, limit>>20) + } +} + +// hasRecoveredPanic finds an error anywhere in the tree that starts with "recovered from panic", the wording of the +// generic per-file recover (and of the golang and license recovers). Matching on the start of a node keeps parse +// errors that echo the input from counting. +func hasRecoveredPanic(err error) bool { + if err == nil { + return false + } + if strings.HasPrefix(err.Error(), "recovered from panic") { + return true + } + switch u := err.(type) { + case *unknown.CoordinateError: + // per-file errors arrive wrapped with their location, and CoordinateError does not unwrap + return hasRecoveredPanic(u.Reason) + case interface{ Unwrap() []error }: + for _, e := range u.Unwrap() { + if hasRecoveredPanic(e) { + return true + } + } + case interface{ Unwrap() error }: + return hasRecoveredPanic(u.Unwrap()) + } + return false +} + +// fuzzTargets pairs every glob-selected parser in the capabilities data with the task that runs its cataloger. +func fuzzTargets(tb testing.TB) []fuzzTarget { + tasks, err := task.DefaultPackageTaskFactories().Tasks(task.DefaultCatalogingFactoryConfig()) + if err != nil { + tb.Fatal(err) + } + byName := map[string]task.Task{} + for _, tsk := range tasks { + byName[tsk.Name()] = tsk + } + + entries, err := capabilities.Packages() + if err != nil { + tb.Fatal(err) + } + + var filter *regexp.Regexp + if v := os.Getenv("SYFT_FUZZ_TARGET"); v != "" { + if filter, err = regexp.Compile(v); err != nil { + tb.Fatalf("SYFT_FUZZ_TARGET: %v", err) + } + } + + var targets []fuzzTarget + for _, e := range entries { + tsk, ok := byName[e.Name] + if e.Type != "generic" || !ok { + continue + } + for _, p := range e.Parsers { + // glob parsers only. The mimetype parsers (binary classifiers) need a real executable + // header to be selected at all; add them if a mutating seed turns out to be worth it. + if p.Detector.Method != capabilities.GlobDetection { + continue + } + for _, g := range p.Detector.Criteria { + ft := fuzzTarget{ + cataloger: e.Name, + glob: g, + path: concretePath(g), + dir: strings.TrimPrefix(filepath.Dir(e.Source.File), "syft/pkg/cataloger/"), + task: tsk, + } + if ok, _ := doublestar.Match(g, "/"+ft.path); !ok || !filepath.IsLocal(ft.path) { + tb.Logf("skipping %s: no concrete path for glob", ft) + continue + } + if filter != nil && !filter.MatchString(ft.String()) { + continue + } + targets = append(targets, ft) + } + } + } + if filter != nil && len(targets) == 0 { + tb.Fatalf("SYFT_FUZZ_TARGET %q matches no target", filter) + } + return targets +} + +var braceAlt = regexp.MustCompile(`\{([^,}]*)[^}]*\}`) + +// concretePath turns a glob into one relative path it matches, e.g. `**/*.gguf` -> `x.gguf`. +func concretePath(glob string) string { + s := strings.TrimPrefix(strings.TrimPrefix(glob, "**/"), "/") + s = braceAlt.ReplaceAllString(s, "$1") + s = strings.ReplaceAll(s, "**", "x") + return strings.ReplaceAll(s, "*", "x") +} + +// fuzzSeeds is the parser's own testdata that matches its glob, plus a few inputs built from the string literals in +// its package. Random bytes rarely produce the exact key names a parser branches on, the literals give the mutator +// something to splice. +func fuzzSeeds(ft fuzzTarget) [][]byte { + var seeds [][]byte + // the CLI's malformed fixtures are deliberately not seeds here: before their fixes land some of them allocate + // gigabytes, which in-process means swapping the machine rather than failing a check. The CLI test runs them in + // a subprocess under a memory limit instead. + _ = filepath.WalkDir(ft.dir, func(p string, d fs.DirEntry, err error) error { + if err != nil || len(seeds) >= fuzzMaxSeedsPerFn { + return fs.SkipAll + } + if d.IsDir() || !d.Type().IsRegular() { + return nil + } + if ok, _ := doublestar.Match(ft.glob, "/"+filepath.ToSlash(p)); !ok { + return nil + } + if info, err := d.Info(); err != nil || info.Size() > fuzzMaxSeedBytes { + return nil + } + if b, err := os.ReadFile(p); err == nil { + seeds = append(seeds, b) + } + return nil + }) + + lits := stringLiterals(ft.dir) + if len(lits) == 0 { + return append(seeds, []byte{}) + } + obj := map[string]string{} + var lines, kv strings.Builder + for _, l := range lits { + obj[l] = l + lines.WriteString(l + "\n") + kv.WriteString(strconv.Quote(l) + ": " + strconv.Quote(l) + "\n") + } + js, _ := json.Marshal(obj) + return append(seeds, []byte(lines.String()), []byte(kv.String()), js) +} + +// stringLiterals returns the distinct short string literals in the non-test go files of a package directory. +func stringLiterals(dir string) []string { + entries, err := os.ReadDir(dir) + if err != nil { + return nil + } + set := map[string]struct{}{} + fset := token.NewFileSet() + for _, e := range entries { + name := e.Name() + if e.IsDir() || !strings.HasSuffix(name, ".go") || strings.HasSuffix(name, "_test.go") { + continue + } + f, err := parser.ParseFile(fset, filepath.Join(dir, name), nil, 0) + if err != nil { + continue + } + ast.Inspect(f, func(n ast.Node) bool { + if bl, ok := n.(*ast.BasicLit); ok && bl.Kind == token.STRING { + if s, err := strconv.Unquote(bl.Value); err == nil && len(s) > 1 && len(s) <= 64 { + set[s] = struct{}{} + } + } + return true + }) + } + out := make([]string, 0, len(set)) + for s := range set { + out = append(out, s) + } + sort.Strings(out) + return out +} diff --git a/syft/pkg/cataloger/php/interpreter_cataloger.go b/syft/pkg/cataloger/php/interpreter_cataloger.go index 80ae69d21..f9022d358 100644 --- a/syft/pkg/cataloger/php/interpreter_cataloger.go +++ b/syft/pkg/cataloger/php/interpreter_cataloger.go @@ -236,7 +236,10 @@ func (p interpreterCataloger) getClassifier(realPath string) (string, *binutils. Vendor: fmt.Sprintf("php-%s", name), Product: fmt.Sprintf("php-%s", name), }, - Source: cpe.GeneratedSource, + // note: we must use a declared source here so that this CPE is treated as authoritative and no + // further CPEs are generated from the (misleading) package name alone, e.g. cpe:...:openssl:openssl + // for the openssl PHP extension, which would imply this is the upstream OpenSSL project. + Source: cpe.DeclaredSource, }, }, } diff --git a/syft/pkg/cataloger/php/interpreter_cataloger_test.go b/syft/pkg/cataloger/php/interpreter_cataloger_test.go index 8d6d9e6d7..bff331d5c 100644 --- a/syft/pkg/cataloger/php/interpreter_cataloger_test.go +++ b/syft/pkg/cataloger/php/interpreter_cataloger_test.go @@ -3,6 +3,10 @@ package php import ( "testing" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/anchore/syft/syft/cpe" "github.com/anchore/syft/syft/pkg/cataloger/internal/pkgtest" ) @@ -147,3 +151,16 @@ func Test_InterpreterCataloger(t *testing.T) { }) } } + +func Test_getClassifier_declaredCPESource(t *testing.T) { + // regression test for https://github.com/anchore/syft/issues/5014 + // the CPE assigned to a PHP extension must be declared (not generated) so that downstream CPE + // generation does not also add a misleading CPE derived from the bare extension name alone + // (e.g. cpe:2.3:a:openssl:openssl:... for the openssl extension, implying the upstream OpenSSL project). + c := interpreterCataloger{name: "php-interpreter-cataloger"} + name, cls := c.getClassifier("/usr/lib/php85/modules/openssl.so") + require.Equal(t, "openssl", name) + require.NotNil(t, cls) + require.Len(t, cls.CPEs, 1) + assert.Equal(t, cpe.DeclaredSource, cls.CPEs[0].Source) +} diff --git a/syft/pkg/cataloger/python/dependency.go b/syft/pkg/cataloger/python/dependency.go index 50b7d6c4a..29671ed59 100644 --- a/syft/pkg/cataloger/python/dependency.go +++ b/syft/pkg/cataloger/python/dependency.go @@ -15,7 +15,7 @@ import ( "github.com/anchore/syft/syft/pkg/cataloger/internal/dependency" ) -func poetryLockDependencySpecifier(p pkg.Package) dependency.Specification { //nolint:dupl // this is very similar to the uv lock dependency specifier, but should remain separate +func poetryLockDependencySpecifier(p pkg.Package) dependency.Specification { meta, ok := p.Metadata.(pkg.PythonPoetryLockEntry) if !ok { log.Tracef("cataloger failed to extract poetry lock metadata for package %+v", p.Name) diff --git a/syft/pkg/cataloger/python/parse_requirements.go b/syft/pkg/cataloger/python/parse_requirements.go index f79737e6a..6622496fd 100644 --- a/syft/pkg/cataloger/python/parse_requirements.go +++ b/syft/pkg/cataloger/python/parse_requirements.go @@ -1,7 +1,6 @@ package python import ( - "bufio" "context" "fmt" "regexp" @@ -105,7 +104,7 @@ func (rp requirementsParser) parseRequirementsTxt(ctx context.Context, _ file.Re var errs error var packages []pkg.Package - scanner := bufio.NewScanner(reader) + scanner := internal.NewLineScanner(reader) var lastLine string for scanner.Scan() { line := trimRequirementsTxtLine(scanner.Text()) @@ -214,9 +213,8 @@ func guessVersion(constraint string) string { constraints := strings.Split(constraint, ",") filteredVersions := map[string]struct{}{} for _, part := range constraints { - if strings.Contains(part, "!=") { - parts := strings.Split(part, "!=") - filteredVersions[strings.TrimSpace(parts[1])] = struct{}{} + if _, excluded, ok := strings.Cut(part, "!="); ok { + filteredVersions[strings.TrimSpace(excluded)] = struct{}{} } } @@ -236,9 +234,8 @@ func guessVersion(constraint string) string { continue } - if strings.Contains(part, "==") { - parts := strings.Split(part, "==") - return strings.TrimSpace(parts[1]) + if _, pinned, ok := strings.Cut(part, "=="); ok { + return strings.TrimSpace(pinned) } if closestVersion == nil || version.GreaterThan(*closestVersion) { diff --git a/syft/pkg/cataloger/python/parse_setup.go b/syft/pkg/cataloger/python/parse_setup.go index a42c785ac..d8cff06de 100644 --- a/syft/pkg/cataloger/python/parse_setup.go +++ b/syft/pkg/cataloger/python/parse_setup.go @@ -1,11 +1,12 @@ package python import ( - "bufio" "context" + "fmt" "regexp" "strings" + "github.com/anchore/syft/internal" "github.com/anchore/syft/internal/log" "github.com/anchore/syft/syft/artifact" "github.com/anchore/syft/syft/file" @@ -36,7 +37,7 @@ var unquotedPinnedDependency = regexp.MustCompile(`^\s*(\w+)\s*==\s*([\w\.\-]+)` func (sp setupFileParser) parseSetupFile(ctx context.Context, _ file.Resolver, _ *generic.Environment, reader file.LocationReadCloser) ([]pkg.Package, []artifact.Relationship, error) { var packages []pkg.Package - scanner := bufio.NewScanner(reader) + scanner := internal.NewLineScanner(reader) for scanner.Scan() { line := scanner.Text() @@ -45,20 +46,23 @@ func (sp setupFileParser) parseSetupFile(ctx context.Context, _ file.Resolver, _ packages = sp.processQuotedDependencies(ctx, line, reader, packages) packages = sp.processUnquotedDependency(ctx, line, reader, packages) } + if err := scanner.Err(); err != nil { + return packages, nil, fmt.Errorf("unable to read setup.py: %w", err) + } return packages, nil, nil } func (sp setupFileParser) processQuotedDependencies(ctx context.Context, line string, reader file.LocationReadCloser, packages []pkg.Package) []pkg.Package { for _, match := range pinnedDependency.FindAllString(line, -1) { - if p, ok := sp.parseQuotedDependency(ctx, match, line, reader); ok { + if p, ok := sp.parseQuotedDependency(ctx, match, reader); ok { packages = append(packages, p) } } return packages } -func (sp setupFileParser) parseQuotedDependency(ctx context.Context, match, line string, reader file.LocationReadCloser) (pkg.Package, bool) { +func (sp setupFileParser) parseQuotedDependency(ctx context.Context, match string, reader file.LocationReadCloser) (pkg.Package, bool) { parts := strings.Split(match, "==") if len(parts) != 2 { return pkg.Package{}, false @@ -67,7 +71,7 @@ func (sp setupFileParser) parseQuotedDependency(ctx context.Context, match, line name := cleanDependencyString(parts[0]) version := cleanDependencyString(parts[len(parts)-1]) - return sp.validateAndCreatePackage(ctx, name, version, line, reader) + return sp.validateAndCreatePackage(ctx, name, version, reader) } // processUnquotedDependency extracts and processes an unquoted dependency from a line @@ -80,7 +84,7 @@ func (sp setupFileParser) processUnquotedDependency(ctx context.Context, line st name := strings.TrimSpace(matches[1]) version := strings.TrimSpace(matches[2]) - if p, ok := sp.validateAndCreatePackage(ctx, name, version, line, reader); ok { + if p, ok := sp.validateAndCreatePackage(ctx, name, version, reader); ok { if !isDuplicatePackage(p, packages) { packages = append(packages, p) } @@ -96,14 +100,15 @@ func cleanDependencyString(s string) string { return s } -func (sp setupFileParser) validateAndCreatePackage(ctx context.Context, name, version, line string, reader file.LocationReadCloser) (pkg.Package, bool) { +func (sp setupFileParser) validateAndCreatePackage(ctx context.Context, name, version string, reader file.LocationReadCloser) (pkg.Package, bool) { if hasTemplateDirective(name) || hasTemplateDirective(version) { // this can happen in more dynamic setup.py where there is templating return pkg.Package{}, false } if name == "" || version == "" { - log.WithFields("path", reader.RealPath).Debugf("unable to parse package in setup.py line: %q", line) + // the raw line is not logged since this runs once per match, which is quadratic on a long line + log.WithFields("path", reader.RealPath, "name", name, "version", version).Debug("unable to parse package in setup.py") return pkg.Package{}, false } diff --git a/syft/pkg/cataloger/python/parse_uv_lock.go b/syft/pkg/cataloger/python/parse_uv_lock.go index 5ac0b3a1c..26c2492e6 100644 --- a/syft/pkg/cataloger/python/parse_uv_lock.go +++ b/syft/pkg/cataloger/python/parse_uv_lock.go @@ -5,12 +5,13 @@ import ( "context" "fmt" "io" + "maps" + "slices" "sort" "strings" "github.com/BurntSushi/toml" - "github.com/anchore/syft/internal/log" "github.com/anchore/syft/internal/unknown" "github.com/anchore/syft/syft/artifact" "github.com/anchore/syft/syft/file" @@ -45,8 +46,12 @@ type uvPackage struct { Metadata uvMetadata `toml:"metadata"` } -type uvDependencies []struct { - Name string `toml:"name"` +type uvDependencies []uvDependency + +type uvDependency struct { + Name string `toml:"name"` + // Version is only written by uv when more than one version of this name is locked + Version string `toml:"version"` Extras []string `toml:"extra"` Markers string `toml:"marker"` } @@ -83,12 +88,16 @@ func newUvLockParser(cfg CatalogerConfig) uvLockParser { // parseUvLock is a parser function for uv.lock contents, returning all the pakcages discovered func (ulp uvLockParser) parseUvLock(ctx context.Context, _ file.Resolver, _ *generic.Environment, reader file.LocationReadCloser) ([]pkg.Package, []artifact.Relationship, error) { - pkgs, err := ulp.uvLockPackages(ctx, reader) + pkgs, specs, err := ulp.uvLockPackages(ctx, reader) if err != nil { return nil, nil, err } - return pkgs, dependency.Resolve(uvLockDependencySpecifier, pkgs), err + specifier := func(p pkg.Package) dependency.Specification { + return specs[p.ID()] + } + + return pkgs, dependency.Resolve(specifier, pkgs), err } func extractUvIndex(p uvPackage) string { @@ -139,105 +148,114 @@ func newPythonUvLockEntry(p uvPackage) pkg.PythonUvLockEntry { } } -func (ulp uvLockParser) uvLockPackages(ctx context.Context, reader file.LocationReadCloser) ([]pkg.Package, error) { +// uvLockPackages returns the packages in the lock along with the dependency specification for each (by package ID). +func (ulp uvLockParser) uvLockPackages(ctx context.Context, reader file.LocationReadCloser) ([]pkg.Package, map[artifact.ID]dependency.Specification, error) { var parsedLockFileVersion uvLockFileVersion // we cannot use the reader twice, so we read the contents first --uv.lock files tend to be small enough contents, err := io.ReadAll(reader) //nolint:gocritic // multi-pass parse requires []byte if err != nil { - return nil, unknown.New(reader.Location, fmt.Errorf("failed to read uv lock file: %w", err)) + return nil, nil, unknown.New(reader.Location, fmt.Errorf("failed to read uv lock file: %w", err)) } _, err = toml.NewDecoder(bytes.NewReader(contents)).Decode(&parsedLockFileVersion) if err != nil { - return nil, fmt.Errorf("failed to read uv lock version: %w", err) + return nil, nil, fmt.Errorf("failed to read uv lock version: %w", err) } // We will need to add some logic to parse and branch on different // lock file versions should they arise, but this gets us // started down this road for now. if parsedLockFileVersion.Version > 1 { - return nil, fmt.Errorf("could not parse uv lock file version %d", parsedLockFileVersion.Version) + return nil, nil, fmt.Errorf("could not parse uv lock file version %d", parsedLockFileVersion.Version) } var parsedLockFile uvLockFile _, err = toml.NewDecoder(bytes.NewReader(contents)).Decode(&parsedLockFile) if err != nil { - return nil, fmt.Errorf("failed to parse uv lock packages: %w", err) - } - - // The uv lock file doesn't store the dependency version in the dependency structure. - // Thus, we need a name -> version map for invoking extractUvDependencies. - // We then, of course, have to pass it down the call stack. - var pkgVerMap = make(map[string]string) - for _, p := range parsedLockFile.Packages { - pkgVerMap[p.Name] = p.Version + return nil, nil, fmt.Errorf("failed to parse uv lock packages: %w", err) } var pkgs []pkg.Package + specs := make(map[artifact.ID]dependency.Specification) for _, p := range parsedLockFile.Packages { - pkgs = append(pkgs, - newPackageForIndexWithMetadata( - ctx, - ulp.licenseResolver, - p.Name, - p.Version, - newPythonUvLockEntry(p), - reader.WithAnnotation(pkg.EvidenceAnnotationKey, pkg.PrimaryEvidenceAnnotation), - ), + np := newPackageForIndexWithMetadata( + ctx, + ulp.licenseResolver, + p.Name, + p.Version, + newPythonUvLockEntry(p), + reader.WithAnnotation(pkg.EvidenceAnnotationKey, pkg.PrimaryEvidenceAnnotation), ) + pkgs = append(pkgs, np) + specs[np.ID()] = uvLockDependencySpecification(p) } - return pkgs, unknown.IfEmptyf(pkgs, "unable to determine packages") + return pkgs, specs, unknown.IfEmptyf(pkgs, "unable to determine packages") } -func isDependencyForUvExtra(dep pkg.PythonUvLockDependencyEntry) bool { +func isDependencyForUvExtra(dep uvDependency) bool { return strings.Contains(dep.Markers, "extra ==") } -// This is identical to poetryLockDependencySpecifier since it operates on identical -// data structures. Keeping it separate for now since it's always possible for data -// structures to change down the line. -// It *is* possible we may be able to merge the Uv and Poetry data structures -func uvLockDependencySpecifier(p pkg.Package) dependency.Specification { //nolint:dupl // this is very similar to the poetry lock dependency specifier, but should remain separate - meta, ok := p.Metadata.(pkg.PythonUvLockEntry) - if !ok { - log.Tracef("cataloger failed to extract UV lock metadata for package %+v", p.Name) - return dependency.Specification{} - } - - provides := []string{packageRef(p.Name, "")} - +// uvLockDependencySpecification is built from the raw lock entry rather than the package metadata. When a lock +// holds several versions of one name (a forked resolution), uv records which version each dependency entry means, +// and that is the only way to pair a dependent with the right one. The pairing is carried by the resulting +// relationships, so the version does not need to live on the metadata. +func uvLockDependencySpecification(p uvPackage) dependency.Specification { var requires []string - - for _, dep := range meta.Dependencies { + for _, dep := range p.Dependencies { if isDependencyForUvExtra(dep) { continue } - - requires = append(requires, packageRef(dep.Name, "")) - - for _, extra := range dep.Extras { - requires = append(requires, packageRef(dep.Name, extra)) - } + requires = append(requires, uvLockRequires(dep)...) } var variants []dependency.ProvidesRequires - for _, extra := range meta.Extras { + for _, extra := range slices.Sorted(maps.Keys(p.OptionalDependencies)) { + var extraRequires []string + for _, dep := range p.OptionalDependencies[extra] { + extraRequires = append(extraRequires, uvLockRequires(dep)...) + } variants = append(variants, dependency.ProvidesRequires{ - Provides: []string{packageRef(p.Name, extra.Name)}, - Requires: extractPackageNames(extra.Dependencies), + Provides: uvLockProvides(p, extra), + Requires: extraRequires, }, ) } return dependency.Specification{ ProvidesRequires: dependency.ProvidesRequires{ - Provides: provides, + Provides: uvLockProvides(p, ""), Requires: requires, }, Variants: variants, } } + +// uvLockProvides offers both the bare ref (for dependency entries without a version) and the versioned ref (for +// dependency entries uv pinned to this version). +func uvLockProvides(p uvPackage, extra string) []string { + return []string{packageRef(p.Name, extra), uvLockPackageRef(p.Name, p.Version, extra)} +} + +// uvLockRequires always requires the base package, plus each extra individually (name[extra1] and name[extra2], +// never name[extra1,extra2]). +func uvLockRequires(dep uvDependency) []string { + refs := []string{uvLockPackageRef(dep.Name, dep.Version, "")} + for _, extra := range dep.Extras { + refs = append(refs, uvLockPackageRef(dep.Name, dep.Version, extra)) + } + return refs +} + +func uvLockPackageRef(name, version, extra string) string { + ref := packageRef(name, extra) + version = strings.TrimSpace(version) + if version == "" { + return ref + } + return ref + "@" + version +} diff --git a/syft/pkg/cataloger/python/parse_uv_lock_test.go b/syft/pkg/cataloger/python/parse_uv_lock_test.go index 582ac9eb6..9aafc2471 100644 --- a/syft/pkg/cataloger/python/parse_uv_lock_test.go +++ b/syft/pkg/cataloger/python/parse_uv_lock_test.go @@ -129,6 +129,79 @@ func TestParseUvLock(t *testing.T) { pkgtest.TestFileParser(t, fixture, uvLockParser.parseUvLock, expectedPkgs, expectedRelationships) } +func TestParseUvLockForkedVersions(t *testing.T) { + fixture := "testdata/uv/forked-versions/uv.lock" + locations := file.NewLocationSet(file.NewLocation(fixture)) + index := "https://pypi.org/simple" + + newPkg := func(name, version string, meta pkg.PythonUvLockEntry) pkg.Package { + return pkg.Package{ + Name: name, + Version: version, + Locations: locations, + PURL: "pkg:pypi/" + name + "@" + version, + Language: pkg.Python, + Type: pkg.PythonPkg, + Metadata: meta, + } + } + + forkdemo := newPkg("forkdemo", "0.1.0", pkg.PythonUvLockEntry{ + Index: ".", + Dependencies: []pkg.PythonUvLockDependencyEntry{ + {Name: "pandas", Markers: "python_full_version < '3.11'"}, + {Name: "pandas", Markers: "python_full_version >= '3.11'"}, + }, + Extras: []pkg.PythonUvLockExtraEntry{ + {Name: "legacy", Dependencies: []string{"numpy"}}, + }, + }) + numpy1264 := newPkg("numpy", "1.26.4", pkg.PythonUvLockEntry{Index: index}) + numpy226 := newPkg("numpy", "2.2.6", pkg.PythonUvLockEntry{Index: index}) + numpy246 := newPkg("numpy", "2.4.6", pkg.PythonUvLockEntry{Index: index}) + numpy253 := newPkg("numpy", "2.5.3", pkg.PythonUvLockEntry{Index: index}) + pandas233 := newPkg("pandas", "2.3.3", pkg.PythonUvLockEntry{ + Index: index, + Dependencies: []pkg.PythonUvLockDependencyEntry{ + {Name: "numpy", Markers: "python_full_version < '3.10'"}, + {Name: "numpy", Markers: "python_full_version == '3.10.*'"}, + }, + }) + pandas306 := newPkg("pandas", "3.0.6", pkg.PythonUvLockEntry{ + Index: index, + Dependencies: []pkg.PythonUvLockDependencyEntry{ + {Name: "numpy", Markers: "python_full_version == '3.11.*'"}, + {Name: "numpy", Markers: "python_full_version >= '3.12'"}, + }, + }) + + dependencyOf := func(from, to pkg.Package) artifact.Relationship { + return artifact.Relationship{From: from, To: to, Type: artifact.DependencyOfRelationship} + } + + // each dependent links only to the versions its entries name, including through an optional dependency + expectedRelationships := []artifact.Relationship{ + dependencyOf(pandas233, forkdemo), + dependencyOf(pandas306, forkdemo), + dependencyOf(numpy1264, forkdemo), + dependencyOf(numpy1264, pandas233), + dependencyOf(numpy226, pandas233), + dependencyOf(numpy246, pandas306), + dependencyOf(numpy253, pandas306), + } + + uvLockParser := newUvLockParser(DefaultCatalogerConfig()) + pkgtest.TestFileParser(t, fixture, uvLockParser.parseUvLock, []pkg.Package{ + forkdemo, + numpy1264, + numpy226, + numpy246, + numpy253, + pandas233, + pandas306, + }, expectedRelationships) +} + func TestParseUvLockWithLicenseEnrichment(t *testing.T) { ctx := context.TODO() fixture := "testdata/pypi-remote/uv.lock" diff --git a/syft/pkg/cataloger/python/parse_wheel_egg_metadata.go b/syft/pkg/cataloger/python/parse_wheel_egg_metadata.go index 0cb060a23..5b0572e3a 100644 --- a/syft/pkg/cataloger/python/parse_wheel_egg_metadata.go +++ b/syft/pkg/cataloger/python/parse_wheel_egg_metadata.go @@ -1,7 +1,6 @@ package python import ( - "bufio" "fmt" "path/filepath" "strings" @@ -9,6 +8,7 @@ import ( "github.com/bmatcuk/doublestar/v4" "github.com/go-viper/mapstructure/v2" + "github.com/anchore/syft/internal" "github.com/anchore/syft/internal/log" "github.com/anchore/syft/syft/file" "github.com/anchore/syft/syft/pkg" @@ -68,7 +68,7 @@ func extractRFC5322Fields(locationReader file.LocationReadCloser) (map[string]an // though this spec is governed by RFC 5322 (mail message), the metadata files are not guaranteed to be compliant. // We must survive parsing as much info as possible without failing and dropping the data. - scanner := bufio.NewScanner(locationReader) + scanner := internal.NewLineScanner(locationReader) for scanner.Scan() { line := scanner.Text() line = strings.TrimRight(line, "\n") @@ -108,6 +108,10 @@ func extractRFC5322Fields(locationReader file.LocationReadCloser) (map[string]an } } } + if err := scanner.Err(); err != nil { + // the caller discards everything on error, so keep the partial fields and log instead + log.WithFields("path", locationReader.Path(), "error", err).Debug("failed to fully read python wheel/egg metadata") + } return fields, nil } diff --git a/syft/pkg/cataloger/python/testdata/uv/forked-versions/uv.lock b/syft/pkg/cataloger/python/testdata/uv/forked-versions/uv.lock new file mode 100644 index 000000000..bcc73c082 --- /dev/null +++ b/syft/pkg/cataloger/python/testdata/uv/forked-versions/uv.lock @@ -0,0 +1,54 @@ +version = 1 +requires-python = ">=3.9" + +[[package]] +name = "forkdemo" +version = "0.1.0" +source = { virtual = "." } +dependencies = [ + { name = "pandas", version = "2.3.3", marker = "python_full_version < '3.11'" }, + { name = "pandas", version = "3.0.6", marker = "python_full_version >= '3.11'" }, +] + +[package.optional-dependencies] +legacy = [ + { name = "numpy", version = "1.26.4" }, +] + +[[package]] +name = "numpy" +version = "1.26.4" +source = { registry = "https://pypi.org/simple" } + +[[package]] +name = "numpy" +version = "2.2.6" +source = { registry = "https://pypi.org/simple" } + +[[package]] +name = "numpy" +version = "2.4.6" +source = { registry = "https://pypi.org/simple" } + +[[package]] +name = "numpy" +version = "2.5.3" +source = { registry = "https://pypi.org/simple" } + +[[package]] +name = "pandas" +version = "2.3.3" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "numpy", version = "1.26.4", marker = "python_full_version < '3.10'" }, + { name = "numpy", version = "2.2.6", marker = "python_full_version == '3.10.*'" }, +] + +[[package]] +name = "pandas" +version = "3.0.6" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "numpy", version = "2.4.6", marker = "python_full_version == '3.11.*'" }, + { name = "numpy", version = "2.5.3", marker = "python_full_version >= '3.12'" }, +] diff --git a/syft/pkg/cataloger/r/parse_description.go b/syft/pkg/cataloger/r/parse_description.go index 01aa89f99..340e395ec 100644 --- a/syft/pkg/cataloger/r/parse_description.go +++ b/syft/pkg/cataloger/r/parse_description.go @@ -1,12 +1,13 @@ package r import ( - "bufio" "context" "io" "regexp" "strings" + "github.com/anchore/syft/internal" + "github.com/anchore/syft/internal/log" "github.com/anchore/syft/syft/artifact" "github.com/anchore/syft/syft/file" "github.com/anchore/syft/syft/pkg" @@ -105,7 +106,7 @@ func extractFieldsFromDescriptionFile(reader io.Reader) map[string]string { result := make(map[string]string) key := "" var valueFragment strings.Builder - scanner := bufio.NewScanner(reader) + scanner := internal.NewLineScanner(reader) for scanner.Scan() { line := scanner.Text() @@ -139,6 +140,9 @@ func extractFieldsFromDescriptionFile(reader io.Reader) map[string]string { if key != "" { result[key] = valueFragment.String() } + if err := scanner.Err(); err != nil { + log.WithFields("error", err).Debug("failed to fully read R DESCRIPTION file") + } return result } diff --git a/syft/pkg/cataloger/ruby/parse_gemfile_lock.go b/syft/pkg/cataloger/ruby/parse_gemfile_lock.go index d8fe50850..29bc1cac3 100644 --- a/syft/pkg/cataloger/ruby/parse_gemfile_lock.go +++ b/syft/pkg/cataloger/ruby/parse_gemfile_lock.go @@ -1,12 +1,12 @@ package ruby import ( - "bufio" "context" "strings" "github.com/scylladb/go-set/strset" + "github.com/anchore/syft/internal" "github.com/anchore/syft/internal/unknown" "github.com/anchore/syft/syft/artifact" "github.com/anchore/syft/syft/file" @@ -21,7 +21,7 @@ var sectionsOfInterest = strset.New("GEM", "GIT", "PATH", "PLUGIN SOURCE") // parseGemFileLockEntries is a parser function for Gemfile.lock contents, returning all Gems discovered. func parseGemFileLockEntries(_ context.Context, _ file.Resolver, _ *generic.Environment, reader file.LocationReadCloser) ([]pkg.Package, []artifact.Relationship, error) { var pkgs []pkg.Package - scanner := bufio.NewScanner(reader) + scanner := internal.NewLineScanner(reader) var currentSection string diff --git a/syft/pkg/cataloger/ruby/parse_gemspec.go b/syft/pkg/cataloger/ruby/parse_gemspec.go index 92b586987..144fe7878 100644 --- a/syft/pkg/cataloger/ruby/parse_gemspec.go +++ b/syft/pkg/cataloger/ruby/parse_gemspec.go @@ -1,7 +1,6 @@ package ruby import ( - "bufio" "context" "encoding/json" "fmt" @@ -75,7 +74,7 @@ func processList(s string) []string { func parseGemSpecEntries(ctx context.Context, resolver file.Resolver, _ *generic.Environment, reader file.LocationReadCloser) ([]pkg.Package, []artifact.Relationship, error) { var pkgs []pkg.Package var fields = make(map[string]any) - scanner := bufio.NewScanner(reader) + scanner := internal.NewLineScanner(reader) for scanner.Scan() { line := scanner.Text() @@ -101,6 +100,7 @@ func parseGemSpecEntries(ctx context.Context, resolver file.Resolver, _ *generic } } } + scanErr := scanner.Err() resolveRubyInterpolationsInFields(fields) @@ -121,6 +121,9 @@ func parseGemSpecEntries(ctx context.Context, resolver file.Resolver, _ *generic ) } + if scanErr != nil { + return pkgs, nil, fmt.Errorf("unable to read gemspec: %w", scanErr) + } return pkgs, nil, nil } diff --git a/syft/pkg/cataloger/snap/parse_base_dpkg.go b/syft/pkg/cataloger/snap/parse_base_dpkg.go index b73f82555..9cc958583 100644 --- a/syft/pkg/cataloger/snap/parse_base_dpkg.go +++ b/syft/pkg/cataloger/snap/parse_base_dpkg.go @@ -5,7 +5,7 @@ import ( "fmt" "strings" - "gopkg.in/yaml.v3" + "go.yaml.in/yaml/v3" "github.com/anchore/syft/syft/artifact" "github.com/anchore/syft/syft/file" @@ -60,10 +60,9 @@ func parseBaseDpkgYaml(_ context.Context, _ file.Resolver, _ *generic.Environmen } // Handle architecture suffixes (e.g., "libssl1.1:amd64") - if strings.Contains(name, ":") { - archParts := strings.SplitN(name, ":", 2) - name = archParts[0] - snapMetadata.Architecture = archParts[1] + if n, arch, ok := strings.Cut(name, ":"); ok { + name = n + snapMetadata.Architecture = arch } debPkg := newDebianPackageFromSnap( diff --git a/syft/pkg/cataloger/snap/parse_kernel_changelog.go b/syft/pkg/cataloger/snap/parse_kernel_changelog.go index 5b5a81b4f..7ad37f300 100644 --- a/syft/pkg/cataloger/snap/parse_kernel_changelog.go +++ b/syft/pkg/cataloger/snap/parse_kernel_changelog.go @@ -1,13 +1,13 @@ package snap import ( - "bufio" "compress/gzip" "context" "fmt" "regexp" "strings" + "github.com/anchore/syft/internal" "github.com/anchore/syft/syft/artifact" "github.com/anchore/syft/syft/file" "github.com/anchore/syft/syft/pkg" @@ -32,7 +32,7 @@ func parseKernelChangelog(_ context.Context, _ file.Resolver, _ *generic.Environ } defer gzReader.Close() - scanner := bufio.NewScanner(gzReader) + scanner := internal.NewLineScanner(gzReader) // read the first line to extract kernel version // Format: "linux (5.4.0-195.215) focal; urgency=medium" @@ -65,6 +65,9 @@ func parseKernelChangelog(_ context.Context, _ file.Resolver, _ *generic.Environ break } } + if err := scanner.Err(); err != nil { + return packages, nil, fmt.Errorf("failed to read changelog content: %w", err) + } return packages, nil, nil } diff --git a/syft/pkg/cataloger/snap/parse_kernel_changelog_test.go b/syft/pkg/cataloger/snap/parse_kernel_changelog_test.go index 063161d19..0e3c17c1a 100644 --- a/syft/pkg/cataloger/snap/parse_kernel_changelog_test.go +++ b/syft/pkg/cataloger/snap/parse_kernel_changelog_test.go @@ -318,7 +318,9 @@ func TestParseKernelChangelog(t *testing.T) { // thing holding this path, and it is incidental: switching to io.ReadAll or a bufio.Reader to pick up // long lines would remove it silently, which is what this test is here to catch. func TestParseKernelChangelog_gzipBombStaysBounded(t *testing.T) { - const expanded = 512 * 1024 * 1024 + // 4x the 8MB budget below is plenty to tell bounded from unbounded. Going bigger only makes the + // fixture slower to build and inflate, which is most of this test's time under -race. + const expanded = 32 * 1024 * 1024 // one enormous line with no newline, the worst case for a line scanner var gz bytes.Buffer @@ -330,7 +332,9 @@ func TestParseKernelChangelog_gzipBombStaysBounded(t *testing.T) { remaining -= n } require.NoError(t, gw.Close()) - require.Less(t, gz.Len(), 1024*1024, "compressed payload should be tiny relative to what it expands to") + // compress/flate output for this input varies by Go version (about 510KB on go1.26, 1MB on go1.27), so + // bound the fixture by ratio rather than a fixed size + require.Less(t, gz.Len(), expanded/100, "compressed payload should be tiny relative to what it expands to") // prove the fixture is a bomb: the same stream read without a line bound costs its full size unbounded := testutils.MeasureAlloc(t, func() { diff --git a/syft/pkg/cataloger/snap/parse_snapd_snapcraft.go b/syft/pkg/cataloger/snap/parse_snapd_snapcraft.go index 8a085f9ab..90dccead7 100644 --- a/syft/pkg/cataloger/snap/parse_snapd_snapcraft.go +++ b/syft/pkg/cataloger/snap/parse_snapd_snapcraft.go @@ -5,7 +5,7 @@ import ( "fmt" "strings" - "gopkg.in/yaml.v3" + "go.yaml.in/yaml/v3" "github.com/anchore/syft/syft/artifact" "github.com/anchore/syft/syft/file" diff --git a/syft/pkg/cataloger/snap/parse_system_manifest.go b/syft/pkg/cataloger/snap/parse_system_manifest.go index ee7159c83..72c934647 100644 --- a/syft/pkg/cataloger/snap/parse_system_manifest.go +++ b/syft/pkg/cataloger/snap/parse_system_manifest.go @@ -5,7 +5,7 @@ import ( "fmt" "strings" - "gopkg.in/yaml.v3" + "go.yaml.in/yaml/v3" "github.com/anchore/syft/syft/artifact" "github.com/anchore/syft/syft/file" @@ -82,10 +82,9 @@ func parseSystemManifest(_ context.Context, _ file.Resolver, _ *generic.Environm // Handle architecture suffixes if present currentMetadata := snapMetadata - if strings.Contains(name, ":") { - archParts := strings.SplitN(name, ":", 2) - name = archParts[0] - currentMetadata.Architecture = archParts[1] + if n, arch, ok := strings.Cut(name, ":"); ok { + name = n + currentMetadata.Architecture = arch } debPkg := newDebianPackageFromSnap( diff --git a/syft/pkg/cataloger/swift/parse_podfile_lock.go b/syft/pkg/cataloger/swift/parse_podfile_lock.go index d8d094186..e571ca436 100644 --- a/syft/pkg/cataloger/swift/parse_podfile_lock.go +++ b/syft/pkg/cataloger/swift/parse_podfile_lock.go @@ -2,6 +2,7 @@ package swift import ( "context" + "errors" "fmt" "strings" @@ -45,9 +46,11 @@ func parsePodfileLock(_ context.Context, _ file.Resolver, _ *generic.Environment default: return nil, nil, fmt.Errorf("malformed podfile.lock") } - splits := strings.Split(podBlob, " ") - podName := splits[0] - podVersion := strings.TrimSuffix(strings.TrimPrefix(splits[1], "("), ")") + podName, podVersion, ok := strings.Cut(podBlob, " ") + if !ok { + return nil, nil, errors.New("malformed podfile.lock: pod has no version") + } + podVersion = strings.TrimSuffix(strings.TrimPrefix(podVersion, "("), ")") podRootPkg := strings.Split(podName, "/")[0] var pkgHash string diff --git a/syft/pkg/cataloger/swift/parse_podfile_lock_test.go b/syft/pkg/cataloger/swift/parse_podfile_lock_test.go index 7769d85c5..e01ce3ec0 100644 --- a/syft/pkg/cataloger/swift/parse_podfile_lock_test.go +++ b/syft/pkg/cataloger/swift/parse_podfile_lock_test.go @@ -1,8 +1,13 @@ package swift import ( + "context" + "io" + "strings" "testing" + "github.com/stretchr/testify/require" + "github.com/anchore/syft/syft/artifact" "github.com/anchore/syft/syft/file" "github.com/anchore/syft/syft/pkg" @@ -280,3 +285,15 @@ func Test_corruptPodfile(t *testing.T) { WithError(). TestParser(t, parsePodfileLock) } + +func TestParsePodfileLock_podWithoutVersion(t *testing.T) { + // a pod entry is "Name (version)"; one without the version used to index past the end of the split + contents := "PODS:\n - Foo\nSPEC CHECKSUMS:\n Foo: abc\n" + reader := file.NewLocationReadCloser(file.NewLocation("Podfile.lock"), io.NopCloser(strings.NewReader(contents))) + + var err error + require.NotPanics(t, func() { + _, _, err = parsePodfileLock(context.Background(), nil, nil, reader) + }) + require.ErrorContains(t, err, "has no version") +} diff --git a/syft/pkg/cataloger/testdata/fuzz/FuzzGenericParsers/e6edf122b3cfae05 b/syft/pkg/cataloger/testdata/fuzz/FuzzGenericParsers/e6edf122b3cfae05 new file mode 100644 index 000000000..fa31971c5 --- /dev/null +++ b/syft/pkg/cataloger/testdata/fuzz/FuzzGenericParsers/e6edf122b3cfae05 @@ -0,0 +1,3 @@ +go test fuzz v1 +string("lua-rock-cataloger **/*.rockspec") +[]byte("[") diff --git a/syft/pkg/cataloger/wordpress/parse_plugin.go b/syft/pkg/cataloger/wordpress/parse_plugin.go index 4956dc6f8..a3de5060c 100644 --- a/syft/pkg/cataloger/wordpress/parse_plugin.go +++ b/syft/pkg/cataloger/wordpress/parse_plugin.go @@ -2,7 +2,9 @@ package wordpress import ( "context" + "errors" "fmt" + "io" "path/filepath" "regexp" "strings" @@ -42,12 +44,13 @@ func parseWordpressPluginFiles(ctx context.Context, resolver file.Resolver, _ *g var pkgs []pkg.Package buffer := make([]byte, contentBufferSize) - _, err := reader.Read(buffer) - if err != nil { + // a file shorter than the buffer is expected, so only the header it does have is parsed + n, err := io.ReadFull(reader, buffer) + if err != nil && !errors.Is(err, io.ErrUnexpectedEOF) { return nil, nil, fmt.Errorf("failed to read %s file: %w", reader.Path(), err) } - fields := extractFields(string(buffer)) + fields := extractFields(string(buffer[:n])) name, nameOk := fields["name"] version, versionOk := fields["version"] diff --git a/syft/pkg/cataloger/wordpress/parse_plugin_test.go b/syft/pkg/cataloger/wordpress/parse_plugin_test.go index 640f1d42b..97a3941c4 100644 --- a/syft/pkg/cataloger/wordpress/parse_plugin_test.go +++ b/syft/pkg/cataloger/wordpress/parse_plugin_test.go @@ -2,9 +2,13 @@ package wordpress import ( "context" + "io" + "os" "testing" + "testing/iotest" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" "github.com/anchore/syft/syft/file" "github.com/anchore/syft/syft/pkg" @@ -58,3 +62,18 @@ func Test_extractFields(t *testing.T) { }) } } + +func TestParseWordpressPluginFiles_shortReads(t *testing.T) { + // a reader that returns one byte per Read must still yield the full header + fixture := "testdata/glob-paths/wp-content/plugins/akismet/akismet.php" + f, err := os.Open(fixture) + require.NoError(t, err) + t.Cleanup(func() { _ = f.Close() }) + reader := file.NewLocationReadCloser(file.NewLocation(fixture), io.NopCloser(iotest.OneByteReader(f))) + + pkgs, _, err := parseWordpressPluginFiles(context.Background(), nil, nil, reader) + require.NoError(t, err) + require.Len(t, pkgs, 1) + assert.Equal(t, "Akismet Anti-spam: Spam Protection", pkgs[0].Name) + assert.Equal(t, "5.3", pkgs[0].Version) +} diff --git a/syft/source/directorysource/directory_source.go b/syft/source/directorysource/directory_source.go index 0b270b466..c6eb43a16 100644 --- a/syft/source/directorysource/directory_source.go +++ b/syft/source/directorysource/directory_source.go @@ -232,7 +232,8 @@ func cleanDirPath(path, base string) string { relPath, err := filepath.Rel(cleanBase, cleanRoot) if err == nil { - path = relPath + // filepath.Rel yields a native path, but source names are posix like every other path syft reports + path = filepath.ToSlash(relPath) } // this is odd, but this means we can't use base } diff --git a/syft/source/directorysource/directory_source_test.go b/syft/source/directorysource/directory_source_test.go index fc4160e58..88e7b3430 100644 --- a/syft/source/directorysource/directory_source_test.go +++ b/syft/source/directorysource/directory_source_test.go @@ -4,6 +4,7 @@ import ( "io/fs" "os" "path/filepath" + "runtime" "testing" "github.com/google/go-cmp/cmp" @@ -325,6 +326,9 @@ func Test_DirectorySource_Exclusions(t *testing.T) { } func Test_getDirectoryExclusionFunctions_crossPlatform(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("emulates windows paths on a posix host, see directory_source_win_test.go for the windows equivalent") + } testCases := []struct { desc string root string @@ -465,10 +469,11 @@ func Test_DirectorySource_ID(t *testing.T) { testutil.Chdir(t, "..") // run with source/testdata tests := []struct { - name string - cfg Config - want artifact.ID - wantErr require.ErrorAssertionFunc + name string + cfg Config + want artifact.ID + wantErr require.ErrorAssertionFunc + skipOnWindows bool }{ { name: "empty", @@ -486,6 +491,8 @@ func Test_DirectorySource_ID(t *testing.T) { name: "with odd unclean path through non-existent directory", cfg: Config{Path: "testdata/does-not-exist/../"}, wantErr: require.Error, + // windows cleans the path lexically, so the non-existent directory is never visited + skipOnWindows: true, }, { name: "to a file (not a directory)", @@ -543,6 +550,9 @@ func Test_DirectorySource_ID(t *testing.T) { } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { + if tt.skipOnWindows && runtime.GOOS == "windows" { + t.Skip("posix-specific path behavior") + } if tt.wantErr == nil { tt.wantErr = require.NoError } diff --git a/syft/source/directorysource/directory_source_win_test.go b/syft/source/directorysource/directory_source_win_test.go index 8f99a59d3..d7d38f86a 100644 --- a/syft/source/directorysource/directory_source_win_test.go +++ b/syft/source/directorysource/directory_source_win_test.go @@ -56,8 +56,9 @@ func Test_DirectorySource_crossPlatformExclusions(t *testing.T) { require.NoError(t, err) for _, f := range fns { - result := f(test.path, nil, nil) - require.Equal(t, test.match, result) + // a matched exclusion is signaled by a skip error + err := f("", test.path, nil, nil) + require.Equal(t, test.match, err != nil) } }) } diff --git a/syft/source/filesource/file_source.go b/syft/source/filesource/file_source.go index 63c34c1c8..4beb0925e 100644 --- a/syft/source/filesource/file_source.go +++ b/syft/source/filesource/file_source.go @@ -223,6 +223,11 @@ func fileAnalysisPath(path string, skipExtractArchive bool) (string, func() erro envelopedUnarchiver, _, err := intFile.IdentifyArchive(context.Background(), path, nil) if unarchiver, ok := envelopedUnarchiver.(archives.Extractor); err == nil && ok { + if isZipWithRarName(path, envelopedUnarchiver) { + log.Debugf("source path has a .rar name but zip content (e.g. a Java resource adapter archive); analyzing it as a single file") + return analysisPath, cleanupFn, nil + } + analysisPath, cleanupFn, err = unarchiveToTmp(path, unarchiver) if err != nil { return "", cleanupFn, fmt.Errorf("unable to unarchive source file: %w", err) @@ -234,6 +239,25 @@ func fileAnalysisPath(path string, skipExtractArchive bool) (string, func() erro return analysisPath, cleanupFn, nil } +// isZipWithRarName reports whether the archive format was identified as RAR from the file name while the file +// content is a zip archive. Archive identification above uses the file name only, and ".rar" is also the extension +// of Java resource adapter archives (JCA), which are zip files. Such a file is left for the java cataloger to read, +// the same way .jar, .war and .ear files are, instead of being handed to the RAR extractor, which cannot read it. +func isZipWithRarName(path string, format archives.Format) bool { + if _, ok := format.(archives.Rar); !ok { + return false + } + + f, err := os.Open(path) + if err != nil { + return false + } + defer f.Close() + + match, err := archives.Zip{}.Match(context.Background(), "", f) + return err == nil && match.ByStream +} + func digestOfFileContents(path string) string { f, err := os.Open(path) if err != nil { diff --git a/syft/source/filesource/file_source_test.go b/syft/source/filesource/file_source_test.go index 87fe2a795..1866dd668 100644 --- a/syft/source/filesource/file_source_test.go +++ b/syft/source/filesource/file_source_test.go @@ -1,11 +1,14 @@ package filesource import ( + "archive/zip" + "bytes" "io" "os" "os/exec" "path" "path/filepath" + "runtime" "syscall" "testing" @@ -96,6 +99,9 @@ func TestNewFromFile(t *testing.T) { } func TestNewFromFile_WithArchive(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("archive fixtures are generated with a shell script") + } testutil.Chdir(t, "..") // run with source/testdata testCases := []struct { @@ -169,6 +175,123 @@ func TestNewFromFile_WithArchive(t *testing.T) { } } +func TestNewFromFile_RarExtension(t *testing.T) { + testutil.Chdir(t, "..") // run with source/testdata + + testCases := []struct { + desc string + input string + wantExtracted bool + wantPaths []string + wantNoPaths []string + }{ + { + // a Java resource adapter archive is a zip file with a .rar extension: it must not be handed to the RAR + // extractor, but left for the java cataloger to read as a single file (like a .jar) + desc: "zip archive with a .rar name is analyzed as a single file", + input: createResourceAdapterArchive(t, t.TempDir()), + wantExtracted: false, + wantPaths: []string{"example-ra-1.0.0.rar"}, + wantNoPaths: []string{"META-INF/ra.xml", "example-lib-1.0.0.jar"}, + }, + { + // a self-extracting RAR has an executable stub before the RAR signature. The stub holds no "R" bytes, + // which is the input that made rardecode v2.2.0 loop forever in its signature search + desc: "self-extracting RAR archive is extracted", + input: writeSfxRar(t, t.TempDir()), + wantExtracted: true, + wantPaths: []string{"inside.txt"}, + wantNoPaths: []string{"example-sfx.rar"}, + }, + { + // a RAR 5.0 archive holding one stored file (inside.txt) + desc: "RAR archive is extracted", + input: "testdata/rar-archive/example.rar", + wantExtracted: true, + wantPaths: []string{"inside.txt"}, + wantNoPaths: []string{"example.rar"}, + }, + } + for _, test := range testCases { + t.Run(test.desc, func(t *testing.T) { + src, err := New(Config{ + Path: test.input, + }) + require.NoError(t, err) + t.Cleanup(func() { + require.NoError(t, src.Close()) + }) + + assert.Equal(t, test.wantExtracted, src.(*fileSource).analysisPath != test.input) + + res, err := src.FileResolver(source.SquashedScope) + require.NoError(t, err) + + for _, p := range test.wantPaths { + refs, err := res.FilesByPath(p) + require.NoError(t, err) + assert.Len(t, refs, 1, "expected to find %q", p) + } + + for _, p := range test.wantNoPaths { + refs, err := res.FilesByPath(p) + require.NoError(t, err) + assert.Empty(t, refs, "expected not to find %q", p) + } + }) + } +} + +// createResourceAdapterArchive writes a minimal Java resource adapter archive (a zip file with a .rar extension that +// holds a deployment descriptor and a nested jar) into dir and returns its path. +func createResourceAdapterArchive(t testing.TB, dir string) string { + t.Helper() + + lib := zipBytes(t, [][2]string{ + {"META-INF/MANIFEST.MF", "Manifest-Version: 1.0\n"}, + {"META-INF/maven/org.example/example-lib/pom.properties", "groupId=org.example\nartifactId=example-lib\nversion=1.0.0\n"}, + }) + + ra := zipBytes(t, [][2]string{ + {"META-INF/MANIFEST.MF", "Manifest-Version: 1.0\n"}, + {"META-INF/ra.xml", "\n"}, + {"example-lib-1.0.0.jar", string(lib)}, + {"META-INF/maven/org.example/example-ra/pom.properties", "groupId=org.example\nartifactId=example-ra\nversion=1.0.0\n"}, + }) + + archivePath := filepath.Join(dir, "example-ra-1.0.0.rar") + require.NoError(t, os.WriteFile(archivePath, ra, 0o600)) + return archivePath +} + +// writeSfxRar writes testdata/rar-archive/example.rar behind a stand-in executable stub into dir and returns its path. +func writeSfxRar(t testing.TB, dir string) string { + t.Helper() + + rar, err := os.ReadFile("testdata/rar-archive/example.rar") + require.NoError(t, err) + + sfxPath := filepath.Join(dir, "example-sfx.rar") + require.NoError(t, os.WriteFile(sfxPath, append(bytes.Repeat([]byte("MZ stub "), 1024), rar...), 0o600)) + return sfxPath +} + +// zipBytes returns a zip archive holding the given (name, contents) entries, in order. +func zipBytes(t testing.TB, entries [][2]string) []byte { + t.Helper() + + var buf bytes.Buffer + w := zip.NewWriter(&buf) + for _, e := range entries { + f, err := w.Create(e[0]) + require.NoError(t, err) + _, err = f.Write([]byte(e[1])) + require.NoError(t, err) + } + require.NoError(t, w.Close()) + return buf.Bytes() +} + // setupArchiveTest encapsulates common test setup work for tar file tests. It returns a cleanup function, // which should be called (typically deferred) by the caller, the path of the created tar archive, and an error, // which should trigger a fatal test failure in the consuming test. The returned cleanup function will never be nil diff --git a/syft/source/ocimodelsource/oci_model_source.go b/syft/source/ocimodelsource/oci_model_source.go index c93e9c3fa..77ab1b187 100644 --- a/syft/source/ocimodelsource/oci_model_source.go +++ b/syft/source/ocimodelsource/oci_model_source.go @@ -135,8 +135,8 @@ func fetchAndStoreModelHeaders(ctx context.Context, client *registryClient, arti } } - // SafeTensors weight-layer headers. We only pull the leading prefix (same - // budget as a GGUF header) + // SafeTensors weight-layer headers. We only pull the leading + // maxSafeTensorsHeaderBytes if artifact.Format == modelFormatSafeTensors { for _, layer := range artifact.SafeTensorsLayers { li, err := fetchSafeTensorsLayerHeader(ctx, client, artifact.Reference, layer, tempDir) @@ -174,7 +174,7 @@ func storeConfigBlobAsLayer(artifact *modelArtifact, tempDir string) (fileresolv // fetchCompanionLayer downloads a companion (non-weight) layer to a temp file func fetchCompanionLayer(ctx context.Context, client *registryClient, ref name.Reference, layer v1.Descriptor, tempDir string) (fileresolver.LayerInfo, error) { - data, err := client.fetchBlobRange(ctx, ref, layer.Digest, maxCompanionBytes) + data, err := client.fetchBlobRange(ctx, ref, layer.Digest, maxCompanionBytes+1) if err != nil { return fileresolver.LayerInfo{}, fmt.Errorf("failed to fetch companion layer: %w", err) } @@ -191,7 +191,7 @@ func fetchCompanionLayer(ctx context.Context, client *registryClient, ref name.R // fetchSingleGGUFHeader fetches a single GGUF layer header and writes it to a temp file func fetchSingleGGUFHeader(ctx context.Context, client *registryClient, ref name.Reference, layer v1.Descriptor, tempDir string) (fileresolver.LayerInfo, error) { - headerData, err := client.fetchBlobRange(ctx, ref, layer.Digest, maxWeightHeaderBytes) + headerData, err := client.fetchBlobRange(ctx, ref, layer.Digest, maxGGUFHeaderBytes) if err != nil { return fileresolver.LayerInfo{}, fmt.Errorf("failed to fetch GGUF layer header: %w", err) } @@ -212,7 +212,7 @@ func fetchSingleGGUFHeader(ctx context.Context, client *registryClient, ref name // fetchSafeTensorsLayerHeader fetches the leading bytes of a SafeTensors weight // layer (enough to cover the JSON header) and writes them to a temp file func fetchSafeTensorsLayerHeader(ctx context.Context, client *registryClient, ref name.Reference, layer v1.Descriptor, tempDir string) (fileresolver.LayerInfo, error) { - headerData, err := client.fetchBlobRange(ctx, ref, layer.Digest, maxWeightHeaderBytes) + headerData, err := client.fetchBlobRange(ctx, ref, layer.Digest, maxSafeTensorsHeaderBytes) if err != nil { return fileresolver.LayerInfo{}, fmt.Errorf("failed to fetch safetensors layer header: %w", err) } diff --git a/syft/source/ocimodelsource/registry_client.go b/syft/source/ocimodelsource/registry_client.go index 8e6c6b9c7..0d4be3a22 100644 --- a/syft/source/ocimodelsource/registry_client.go +++ b/syft/source/ocimodelsource/registry_client.go @@ -16,6 +16,7 @@ import ( "github.com/google/go-containerregistry/pkg/v1/remote" "github.com/anchore/stereoscope/pkg/image" + "github.com/anchore/syft/internal" ) // errNotModelArtifact is returned when a reference does not point to a model artifact. @@ -37,17 +38,20 @@ const ( modelFormatGGUF = "gguf" modelFormatSafeTensors = "safetensors" - // maxWeightHeaderBytes is the leading slice we range-GET from a (multi-GB) - // weight layer — enough to cover the GGUF/safetensors header. Note this is - // smaller than the ai cataloger's own maxSafeTensorsHeaderSize (100 MB) parse - // ceiling: a safetensors header between the two is parseable from a directory - // scan but truncated here, so its shard would go uncounted on an OCI scan. - // Keep this comfortably above real-world header sizes. - maxWeightHeaderBytes = 8 * 1024 * 1024 // 8 MB + // maxGGUFHeaderBytes is the leading slice we range-GET from a (multi-GB) + // GGUF weight layer, enough to cover its header. See + // https://github.com/ggml-org/ggml/blob/master/docs/gguf.md#file-structure + maxGGUFHeaderBytes = 8 * 1024 * 1024 // 8 MB + + // maxSafeTensorsHeaderBytes is the leading slice we range-GET from a + // safetensors weight layer: the 8-byte length prefix plus the same header + // ceiling the ai cataloger applies to directory scans. + maxSafeTensorsHeaderBytes = internal.MaxSafeTensorsHeaderSize + 8 // maxCompanionBytes caps a whole companion blob (README, config.json, - // license); these are small by convention. Matches the 4 MB read cap in - // classifyOCIModelFileLayer. + // license); these are small by convention. It is the upper bound of the ai + // cataloger's per-layer caps. One extra byte is fetched so the cataloger can + // tell an oversized blob from one that is exactly at the cap. maxCompanionBytes = 4 * 1024 * 1024 // 4 MB ) @@ -279,9 +283,6 @@ func (c *registryClient) fetchBlobRange(ctx context.Context, ref name.Reference, } defer reader.Close() - // Note: this is not some arbitrary number picked out of the blue. - // This is based on the specification of header data found here: - // https://github.com/ggml-org/ggml/blob/master/docs/gguf.md#file-structure data := make([]byte, maxBytes) n, err := io.ReadFull(reader, data) diff --git a/syft/source/testdata/rar-archive/example.rar b/syft/source/testdata/rar-archive/example.rar new file mode 100644 index 000000000..f2641fd4c Binary files /dev/null and b/syft/source/testdata/rar-archive/example.rar differ diff --git a/test/cli/archive_test.go b/test/cli/archive_test.go index 3c00c1d46..843401b29 100644 --- a/test/cli/archive_test.go +++ b/test/cli/archive_test.go @@ -2,6 +2,8 @@ package cli import ( "archive/tar" + "archive/zip" + "bytes" "os" "path/filepath" "testing" @@ -31,6 +33,25 @@ func TestArchiveScan(t *testing.T) { assertPackageCount(1), }, }, + { + // a Java resource adapter archive is a zip file with a .rar extension + name: "scan a java resource adapter archive", + args: []string{ + "scan", + "-o", + "json", + "--from", + "file", + createResourceAdapterArchive(t, t.TempDir()), + }, + assertions: []traitAssertion{ + assertSuccessfulReturnCode, + assertJsonReport, + assertInOutput("pkg:maven/org.example/example-ra@1.0.0"), + assertInOutput("pkg:maven/org.example/example-lib@1.0.0"), + assertPackageCount(2), + }, + }, } for _, test := range tests { @@ -58,3 +79,37 @@ func createArchive(t *testing.T, path string, destDir string) string { return archivePath } + +// createResourceAdapterArchive writes a minimal Java resource adapter archive (a zip file with a .rar extension that +// holds a deployment descriptor and a nested jar) into destDir and returns its path. +func createResourceAdapterArchive(t *testing.T, destDir string) string { + lib := zipBytes(t, [][2]string{ + {"META-INF/MANIFEST.MF", "Manifest-Version: 1.0\n"}, + {"META-INF/maven/org.example/example-lib/pom.properties", "groupId=org.example\nartifactId=example-lib\nversion=1.0.0\n"}, + }) + + ra := zipBytes(t, [][2]string{ + {"META-INF/MANIFEST.MF", "Manifest-Version: 1.0\n"}, + {"META-INF/ra.xml", "\n"}, + {"example-lib-1.0.0.jar", string(lib)}, + {"META-INF/maven/org.example/example-ra/pom.properties", "groupId=org.example\nartifactId=example-ra\nversion=1.0.0\n"}, + }) + + archivePath := filepath.Join(destDir, "example-ra-1.0.0.rar") + require.NoError(t, os.WriteFile(archivePath, ra, 0o600)) + return archivePath +} + +// zipBytes returns a zip archive holding the given (name, contents) entries, in order. +func zipBytes(t *testing.T, entries [][2]string) []byte { + var buf bytes.Buffer + w := zip.NewWriter(&buf) + for _, e := range entries { + f, err := w.Create(e[0]) + require.NoError(t, err) + _, err = f.Write([]byte(e[1])) + require.NoError(t, err) + } + require.NoError(t, w.Close()) + return buf.Bytes() +} diff --git a/test/cli/malformed_input_test.go b/test/cli/malformed_input_test.go new file mode 100644 index 000000000..d831ab363 --- /dev/null +++ b/test/cli/malformed_input_test.go @@ -0,0 +1,78 @@ +package cli + +import ( + "encoding/json" + "os" + "path/filepath" + "runtime" + "syscall" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// maxMalformedScanRSS is far above what scanning a handful of tiny files needs. RSS only sees memory that was actually +// touched, so a large allocation that is never filled can slip under it; the fuzz harness in syft/pkg/cataloger +// measures allocations directly. +const maxMalformedScanRSS = 1 << 30 + +// each directory under testdata/malformed holds a few tiny files shaped to trip parsers up (deep nesting, huge declared +// counts, truncation, type mismatches, non-finite floats). None of them should cost the rest of the SBOM, so each is +// scanned next to a well-formed requirements.txt whose package must still be reported. +func TestMalformedInputsStillProduceSBOM(t *testing.T) { + dirs, err := filepath.Glob("testdata/malformed/*") + require.NoError(t, err) + require.NotEmpty(t, dirs) + + for _, dir := range dirs { + t.Run(filepath.Base(dir), func(t *testing.T) { + root := t.TempDir() + require.NoError(t, os.CopyFS(root, os.DirFS(dir))) + require.NoError(t, os.WriteFile(filepath.Join(root, "requirements.txt"), []byte("well-formed==1.0.0\n"), 0o600)) + + out := filepath.Join(t.TempDir(), "sbom.json") + env := map[string]string{"GOMEMLIMIT": "512MiB"} + // runSyft aborts the process after 60s + cmd, stdout, stderr := runSyft(t, env, "scan", "dir:"+root, "--override-default-catalogers", "all", "-o", "syft-json="+out) + defer logOutputOnFailure(t, cmd, stdout, stderr) + + assertSuccessfulReturnCode(t, stdout, stderr, cmd.ProcessState.ExitCode()) + + if ru, ok := cmd.ProcessState.SysUsage().(*syscall.Rusage); ok { + rss := int64(ru.Maxrss) // int32 on 32-bit linux + if runtime.GOOS == "linux" { + rss *= 1024 // kilobytes on linux, bytes on darwin + } + assert.Less(t, rss, int64(maxMalformedScanRSS), "max RSS %d MB", rss>>20) + } + + b, err := os.ReadFile(out) + require.NoError(t, err, "no SBOM written") + var doc struct { + Artifacts []struct { + Name string `json:"name"` + } `json:"artifacts"` + Files []struct { + Location struct { + Path string `json:"path"` + } `json:"location"` + Unknowns []string `json:"unknowns"` + } `json:"files"` + } + require.NoError(t, json.Unmarshal(b, &doc), "SBOM is not valid JSON") + var names []string + for _, a := range doc.Artifacts { + names = append(names, a.Name) + } + assert.Contains(t, names, "well-formed", "package from the well-formed file is missing") + + // a per-file panic is recovered into an unknown and the scan still exits 0, so look for it explicitly + for _, f := range doc.Files { + for _, u := range f.Unknowns { + assert.NotContains(t, u, "recovered from panic", "parser panicked on %s", f.Location.Path) + } + } + }) + } +} diff --git a/test/cli/testdata/.gitignore b/test/cli/testdata/.gitignore index 3e8e5ed17..a11d22644 100644 --- a/test/cli/testdata/.gitignore +++ b/test/cli/testdata/.gitignore @@ -1,3 +1,4 @@ # negate jar and zip exclusions for test fixtures that need these file types !image-unknowns/*.jar !image-unknowns/*.zip +!malformed/**/*.jar diff --git a/test/cli/testdata/malformed/gguf-declared-count/model.gguf b/test/cli/testdata/malformed/gguf-declared-count/model.gguf new file mode 100644 index 000000000..41fc711f4 Binary files /dev/null and b/test/cli/testdata/malformed/gguf-declared-count/model.gguf differ diff --git a/test/cli/testdata/malformed/gguf-non-finite/model.gguf b/test/cli/testdata/malformed/gguf-non-finite/model.gguf new file mode 100644 index 000000000..f6c5e051e Binary files /dev/null and b/test/cli/testdata/malformed/gguf-non-finite/model.gguf differ diff --git a/test/cli/testdata/malformed/gguf-truncated/model.gguf b/test/cli/testdata/malformed/gguf-truncated/model.gguf new file mode 100644 index 000000000..1d9fda7d4 Binary files /dev/null and b/test/cli/testdata/malformed/gguf-truncated/model.gguf differ diff --git a/test/cli/testdata/malformed/gguf-type-mismatch/model.gguf b/test/cli/testdata/malformed/gguf-type-mismatch/model.gguf new file mode 100644 index 000000000..327ca18b4 Binary files /dev/null and b/test/cli/testdata/malformed/gguf-type-mismatch/model.gguf differ diff --git a/test/cli/testdata/malformed/json-deep-nesting/package.json b/test/cli/testdata/malformed/json-deep-nesting/package.json new file mode 100644 index 000000000..b14bbbce8 --- /dev/null +++ b/test/cli/testdata/malformed/json-deep-nesting/package.json @@ -0,0 +1 @@ +{"name":"deep","version":"1.0.0","config":[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]} \ No newline at end of file diff --git a/test/cli/testdata/malformed/list-for-scalar/Cargo.lock b/test/cli/testdata/malformed/list-for-scalar/Cargo.lock new file mode 100644 index 000000000..2a40d3b59 --- /dev/null +++ b/test/cli/testdata/malformed/list-for-scalar/Cargo.lock @@ -0,0 +1,6 @@ +version = 3 + +[[package]] +name = ["a"] +version = 1 +dependencies = "b" diff --git a/test/cli/testdata/malformed/list-for-scalar/package.json b/test/cli/testdata/malformed/list-for-scalar/package.json new file mode 100644 index 000000000..565908124 --- /dev/null +++ b/test/cli/testdata/malformed/list-for-scalar/package.json @@ -0,0 +1 @@ +{"name": ["a"], "version": {"v": 1}, "license": [1, 2], "author": 7, "dependencies": []} \ No newline at end of file diff --git a/test/cli/testdata/malformed/safetensors-declared-length/model.safetensors b/test/cli/testdata/malformed/safetensors-declared-length/model.safetensors new file mode 100644 index 000000000..45f9a3a1f Binary files /dev/null and b/test/cli/testdata/malformed/safetensors-declared-length/model.safetensors differ diff --git a/test/cli/testdata/malformed/truncated-archive/a-1.0.jar b/test/cli/testdata/malformed/truncated-archive/a-1.0.jar new file mode 100644 index 000000000..e87a7ecfd Binary files /dev/null and b/test/cli/testdata/malformed/truncated-archive/a-1.0.jar differ diff --git a/test/cli/testdata/malformed/truncated-lockfile/package-lock.json b/test/cli/testdata/malformed/truncated-lockfile/package-lock.json new file mode 100644 index 000000000..a1d749177 --- /dev/null +++ b/test/cli/testdata/malformed/truncated-lockfile/package-lock.json @@ -0,0 +1,14 @@ +{ + "name": "t", + "version": "1.0.0", + "lockfileVersion": 3, + "packages": { + "": { + "name": "t", + "version": "1.0.0" + }, + "node_modules/a": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/a/-/a-1.0.0.tgz" + } + } diff --git a/test/cli/testdata/malformed/yaml-deep-nesting/pnpm-lock.yaml b/test/cli/testdata/malformed/yaml-deep-nesting/pnpm-lock.yaml new file mode 100644 index 000000000..c9429fe0f --- /dev/null +++ b/test/cli/testdata/malformed/yaml-deep-nesting/pnpm-lock.yaml @@ -0,0 +1,3 @@ +lockfileVersion: '9.0' +packages: + /a@1.0.0: [[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]] diff --git a/test/cli/testdata/malformed/yaml-merge-keys/pnpm-lock.yaml b/test/cli/testdata/malformed/yaml-merge-keys/pnpm-lock.yaml new file mode 100644 index 000000000..d7c514726 --- /dev/null +++ b/test/cli/testdata/malformed/yaml-merge-keys/pnpm-lock.yaml @@ -0,0 +1,8 @@ +lockfileVersion: '9.0' +packages: + /a@1.0.0: &p + resolution: {integrity: sha512-x} + /b@1.0.0: + <<: *p + /c@1.0.0: + <<: [*p, {<<: *p}]