From 3f4105a657f7d53a29fbf1ffe8010853dea08fdc Mon Sep 17 00:00:00 2001 From: somaz <112675579+somaz94@users.noreply.github.com> Date: Wed, 23 Sep 2026 23:02:12 +0900 Subject: [PATCH 01/40] fix(elf): avoid a panic on a truncated dynamic section (#5321) Signed-off-by: somaz --- syft/file/cataloger/executable/elf.go | 26 +------ syft/file/cataloger/executable/elf_test.go | 88 ++++++++++++++++++++++ 2 files changed, 91 insertions(+), 23 deletions(-) diff --git a/syft/file/cataloger/executable/elf.go b/syft/file/cataloger/executable/elf.go index e7f6d588d..b58b3eaa2 100644 --- a/syft/file/cataloger/executable/elf.go +++ b/syft/file/cataloger/executable/elf.go @@ -156,32 +156,12 @@ func hasElfDynFlag1(f *elf.File, flag elf.DynFlag1) bool { } func hasElfDynTag(f *elf.File, tag elf.DynTag) bool { - // source https://github.com/golang/go/blob/9b4b3e5acca2dabe107fa2c3ed963097d78a4562/src/cmd/cgo/internal/testshared/shared_test.go#L280 - - ds := f.SectionByType(elf.SHT_DYNAMIC) - if ds == nil { - return false - } - d, err := ds.Data() + vals, err := f.DynValue(tag) if err != nil { + log.WithFields("error", err).Trace("unable to read dynamic section from elf file") return false } - - for len(d) > 0 { - var t elf.DynTag - switch f.Class { - case elf.ELFCLASS32: - t = elf.DynTag(f.ByteOrder.Uint32(d[0:4])) - d = d[8:] - case elf.ELFCLASS64: - t = elf.DynTag(f.ByteOrder.Uint64(d[0:8])) - d = d[16:] - } - if t == tag { - return true - } - } - return false + return len(vals) > 0 } func isELFPIE(f *elf.File) bool { diff --git a/syft/file/cataloger/executable/elf_test.go b/syft/file/cataloger/executable/elf_test.go index a380a5cd6..23b19eced 100644 --- a/syft/file/cataloger/executable/elf_test.go +++ b/syft/file/cataloger/executable/elf_test.go @@ -1,7 +1,9 @@ package executable import ( + "bytes" "debug/elf" + "encoding/binary" "os" "path/filepath" "testing" @@ -226,3 +228,89 @@ func Test_elfHasExports(t *testing.T) { }) } } + +func Test_hasElfDynTag(t *testing.T) { + dynEntry := func(class elf.Class, tag elf.DynTag) []byte { + if class == elf.ELFCLASS32 { + return binary.LittleEndian.AppendUint32(binary.LittleEndian.AppendUint32(nil, uint32(tag)), 0) + } + return binary.LittleEndian.AppendUint64(binary.LittleEndian.AppendUint64(nil, uint64(tag)), 0) + } + + tests := []struct { + name string + class elf.Class + dynamic []byte + want bool + }{ + { + name: "32-bit tag present", + class: elf.ELFCLASS32, + dynamic: dynEntry(elf.ELFCLASS32, elf.DT_BIND_NOW), + want: true, + }, + { + name: "64-bit tag present", + class: elf.ELFCLASS64, + dynamic: dynEntry(elf.ELFCLASS64, elf.DT_BIND_NOW), + want: true, + }, + { + name: "64-bit tag absent", + class: elf.ELFCLASS64, + dynamic: dynEntry(elf.ELFCLASS64, elf.DT_NULL), + want: false, + }, + { + name: "32-bit truncated dynamic section", + class: elf.ELFCLASS32, + dynamic: dynEntry(elf.ELFCLASS32, elf.DT_BIND_NOW)[:4], + want: false, + }, + { + name: "64-bit truncated dynamic section", + class: elf.ELFCLASS64, + dynamic: dynEntry(elf.ELFCLASS64, elf.DT_BIND_NOW)[:12], + want: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + f, err := elf.NewFile(bytes.NewReader(elfWithDynamicSection(t, tt.class, tt.dynamic))) + require.NoError(t, err) + assert.Equal(t, tt.want, hasElfDynTag(f, elf.DT_BIND_NOW)) + }) + } +} + +// elfWithDynamicSection builds a little-endian ELF holding only a null section and a SHT_DYNAMIC section. +func elfWithDynamicSection(t *testing.T, class elf.Class, dynamic []byte) []byte { + t.Helper() + ident := [elf.EI_NIDENT]byte{0x7f, 'E', 'L', 'F', byte(class), byte(elf.ELFDATA2LSB), byte(elf.EV_CURRENT)} + var buf bytes.Buffer + write := func(v any) { + require.NoError(t, binary.Write(&buf, binary.LittleEndian, v)) + } + + if class == elf.ELFCLASS32 { + ehsize, shentsize := binary.Size(elf.Header32{}), binary.Size(elf.Section32{}) + dataOff := ehsize + 2*shentsize + write(elf.Header32{ + Ident: ident, Type: uint16(elf.ET_DYN), Machine: uint16(elf.EM_386), Version: uint32(elf.EV_CURRENT), + Shoff: uint32(ehsize), Ehsize: uint16(ehsize), Shentsize: uint16(shentsize), Shnum: 2, + }) + write(elf.Section32{}) + write(elf.Section32{Type: uint32(elf.SHT_DYNAMIC), Off: uint32(dataOff), Size: uint32(len(dynamic)), Addralign: 1}) + } else { + ehsize, shentsize := binary.Size(elf.Header64{}), binary.Size(elf.Section64{}) + dataOff := ehsize + 2*shentsize + write(elf.Header64{ + Ident: ident, Type: uint16(elf.ET_DYN), Machine: uint16(elf.EM_X86_64), Version: uint32(elf.EV_CURRENT), + Shoff: uint64(ehsize), Ehsize: uint16(ehsize), Shentsize: uint16(shentsize), Shnum: 2, + }) + write(elf.Section64{}) + write(elf.Section64{Type: uint32(elf.SHT_DYNAMIC), Off: uint64(dataOff), Size: uint64(len(dynamic)), Addralign: 1}) + } + buf.Write(dynamic) + return buf.Bytes() +} From 97fd9393552f9ca0f76217ac268198d058ebcbed Mon Sep 17 00:00:00 2001 From: Arpit Jain <3242828+arpitjain099@users.noreply.github.com> Date: Wed, 23 Sep 2026 10:07:57 -0400 Subject: [PATCH 02/40] fix(haskell): do not slice past the end of a stack.yaml rev pin (#5310) An extra-dep can be pinned as "@sha256:," or as "@rev:". The hash slice assumed the first form, so a rev pin left the end index at -1 and the parser panicked with slice bounds out of range. A rev pin carries no package hash, so only the sha256 form now yields one. Signed-off-by: Arpit Jain --- syft/pkg/cataloger/haskell/parse_stack_lock.go | 8 ++++++-- syft/pkg/cataloger/haskell/parse_stack_yaml_test.go | 11 +++++++++++ syft/pkg/cataloger/haskell/testdata/stack.yaml | 1 + 3 files changed, 18 insertions(+), 2 deletions(-) diff --git a/syft/pkg/cataloger/haskell/parse_stack_lock.go b/syft/pkg/cataloger/haskell/parse_stack_lock.go index 90c25d02e..1f36ea424 100644 --- a/syft/pkg/cataloger/haskell/parse_stack_lock.go +++ b/syft/pkg/cataloger/haskell/parse_stack_lock.go @@ -91,8 +91,12 @@ func parseStackPackageEncoding(pkgEncoding string) (name, version, hash string) encodingSplits := strings.Split(remainingEncoding, "@") version = encodingSplits[0] if len(encodingSplits) > 1 { - startHash, endHash := strings.Index(encodingSplits[1], ":")+1, strings.Index(encodingSplits[1], ",") - hash = encodingSplits[1][startHash:endHash] + // a pin is either "@sha256:," or "@rev:", and only the first carries a + // hash. without the comma the end index is -1 and the slice below goes out of range. + colonIdx, commaIdx := strings.Index(encodingSplits[1], ":"), strings.Index(encodingSplits[1], ",") + if colonIdx != -1 && commaIdx > colonIdx { + hash = encodingSplits[1][colonIdx+1 : commaIdx] + } } return } diff --git a/syft/pkg/cataloger/haskell/parse_stack_yaml_test.go b/syft/pkg/cataloger/haskell/parse_stack_yaml_test.go index 853381d07..449276af2 100644 --- a/syft/pkg/cataloger/haskell/parse_stack_yaml_test.go +++ b/syft/pkg/cataloger/haskell/parse_stack_yaml_test.go @@ -102,6 +102,17 @@ func TestParseStackYaml(t *testing.T) { PkgHash: "e4dc6473faaa75fbd7eccab4e3ee1d651d75bb0e49946ef0b8b751ccde771a55", }, }, + { + Name: "acme-missiles", + Version: "0.3", + PURL: "pkg:hackage/acme-missiles@0.3", + Locations: locationSet, + Language: pkg.Haskell, + Type: pkg.HackagePkg, + Metadata: pkg.HackageStackYamlEntry{ + PkgHash: "", + }, + }, } // TODO: relationships are not under test yet diff --git a/syft/pkg/cataloger/haskell/testdata/stack.yaml b/syft/pkg/cataloger/haskell/testdata/stack.yaml index 8c05906e1..b6a63c4bd 100644 --- a/syft/pkg/cataloger/haskell/testdata/stack.yaml +++ b/syft/pkg/cataloger/haskell/testdata/stack.yaml @@ -12,5 +12,6 @@ extra-deps: - hspec-core-2.9.4@sha256:a126e9087409fef8dcafcd2f8656456527ac7bb163ed4d9cb3a57589042a5fe8,6498 - hspec-discover-2.9.4@sha256:fbcf49ecfc3d4da53e797fd0275264cba776ffa324ee223e2a3f4ec2d2c9c4a6,2165 - stm-2.5.0.2@sha256:e4dc6473faaa75fbd7eccab4e3ee1d651d75bb0e49946ef0b8b751ccde771a55,2314 + - acme-missiles-0.3@rev:0 ghc-options: "$everything": -haddock From 41870fec7f11b5a6bb0fc8b25dfe4c27825a39b0 Mon Sep 17 00:00:00 2001 From: Yunare Maia Date: Wed, 23 Sep 2026 11:13:22 -0300 Subject: [PATCH 03/40] fix: skip docker:// references in github-actions PURL generation (#5302) closes #5299: docker:// references are OCI images, not GitHub repos. Returning empty string produces a package with no PURL, matching the treatment for local composite actions. Signed-off-by: Yunare Maia --- syft/pkg/cataloger/githubactions/package.go | 7 ++++++ .../githubactions/packageurl_test.go | 23 +++++++++++++++++++ .../workflow-with-docker-actions.yaml | 10 ++++++++ 3 files changed, 40 insertions(+) create mode 100644 syft/pkg/cataloger/githubactions/packageurl_test.go create mode 100644 syft/pkg/cataloger/githubactions/testdata/workflow-with-docker-actions.yaml diff --git a/syft/pkg/cataloger/githubactions/package.go b/syft/pkg/cataloger/githubactions/package.go index 179957510..49c5c83e4 100644 --- a/syft/pkg/cataloger/githubactions/package.go +++ b/syft/pkg/cataloger/githubactions/package.go @@ -88,6 +88,13 @@ func packageURL(name, version string) string { var subPath string var namespace string + // docker:// references are OCI images, not GitHub repos. + // They don't fit the pkg:github PURL shape, so skip them entirely + // (same treatment as local composite actions below). + if strings.HasPrefix(name, "docker:") { + return "" + } + fields := strings.SplitN(name, "/", 3) switch len(fields) { case 1: diff --git a/syft/pkg/cataloger/githubactions/packageurl_test.go b/syft/pkg/cataloger/githubactions/packageurl_test.go new file mode 100644 index 000000000..4174c2026 --- /dev/null +++ b/syft/pkg/cataloger/githubactions/packageurl_test.go @@ -0,0 +1,23 @@ +package githubactions + +import "testing" + +func Test_packageURL_skipsDockerPrefix(t *testing.T) { + // docker:// references should return empty PURL (not a valid pkg:github) + got := packageURL("docker://ghcr.io/sethvargo/ratchet:latest", "sha256:527e78e6d29a9ac306e843cf766afe0604a0b35633f16913d85dd522218e8ca1") + if got != "" { + t.Errorf("packageURL(digest-form) = %q, want empty string", got) + } + + got = packageURL("docker://ghcr.io/some-org/some-action", "v1") + if got != "" { + t.Errorf("packageURL(tag-form) = %q, want empty string", got) + } + + // Sanity: regular github actions still work + got = packageURL("actions/checkout", "v4") + want := "pkg:github/actions/checkout@v4" + if got != want { + t.Errorf("packageURL(regular) = %q, want %q", got, want) + } +} diff --git a/syft/pkg/cataloger/githubactions/testdata/workflow-with-docker-actions.yaml b/syft/pkg/cataloger/githubactions/testdata/workflow-with-docker-actions.yaml new file mode 100644 index 000000000..c33a3d54b --- /dev/null +++ b/syft/pkg/cataloger/githubactions/testdata/workflow-with-docker-actions.yaml @@ -0,0 +1,10 @@ +name: "Docker Actions Test" +on: [push] +jobs: + build: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: docker://ghcr.io/sethvargo/ratchet:latest@sha256:527e78e6d29a9ac306e843cf766afe0604a0b35633f16913d85dd522218e8ca1 + - uses: docker://ghcr.io/some-org/some-action@v1 + - uses: actions/setup-go@v5 From 409ac78f62a44de7cffa5edaef2f05e41708a727 Mon Sep 17 00:00:00 2001 From: texasich <101962694+texasich@users.noreply.github.com> Date: Wed, 23 Sep 2026 12:20:28 -0500 Subject: [PATCH 04/40] fix(java): resolve Groovy 4+ group IDs to org.apache.groovy (#5313) * fix(java): resolve Groovy 4+ group IDs to org.apache.groovy Groovy 4.0 moved from the Codehaus coordinates (org.codehaus.groovy) to the Apache Software Foundation (org.apache.groovy). JARs that ship no pom metadata fall through to the static artifact-to-group map, which carries the 3.x-era org.codehaus.groovy entries for the whole groovy family, so 4.x artifacts were attributed to an obsolete group even though their manifest identifies the Apache vendor. Where the static map attributes an artifact to the historical group, a major-version migration resolves groovy 4.x and newer to org.apache.groovy, while 3.x and earlier keep their historical group. The Eclipse-hosted groovy-eclipse-* artifacts, which share the prefix but kept their Codehaus coordinates, are excluded. Fixes #5311 Signed-off-by: texasich <101962694+texasich@users.noreply.github.com> * refactor(java): share version-aware groovy group ID lookup with CPE generation - move the artifact-to-group lookup behind `cpegenerate.ArtifactIDToGroupID(artifactID, version)` so the PURL and CPE paths agree; the exported mutable map is now unexported - version-dependent entries are expressed as a `groupIDMigration{before, after, atMajor}` (groovy moved to `org.apache.groovy` at 4.0); unconditional entries, including `groovy-eclipse-*` and apache-only 4.0+ modules, stay in the plain map - add apache-only modules (`groovy-ginq`, `groovy-contracts`, `groovy-callsite`, ...) that otherwise took the group from their per-module `Automatic-Module-Name` This changes CPEs for pom-less groovy 4+ jars (all real ones): vendors are now `apache` (e.g. `cpe:2.3:a:apache:groovy:4.0.33`) instead of `codehaus`, matching the PURL, jars with pom metadata, and NVD. Signed-off-by: Alex Goodman --------- Signed-off-by: texasich <101962694+texasich@users.noreply.github.com> Signed-off-by: Alex Goodman Co-authored-by: Alex Goodman --- .../internal/cpegenerate/generate_test.go | 18 ++ .../cataloger/internal/cpegenerate/java.go | 12 +- .../internal/cpegenerate/java_groupid_map.go | 262 +++++++++++------- .../cpegenerate/java_groupid_map_test.go | 49 ++++ .../internal/cpegenerate/java_test.go | 12 +- syft/pkg/cataloger/java/archive_parser.go | 8 +- syft/pkg/cataloger/java/package_url.go | 17 +- syft/pkg/cataloger/java/package_url_test.go | 55 +++- 8 files changed, 311 insertions(+), 122 deletions(-) create mode 100644 syft/pkg/cataloger/internal/cpegenerate/java_groupid_map_test.go diff --git a/syft/pkg/cataloger/internal/cpegenerate/generate_test.go b/syft/pkg/cataloger/internal/cpegenerate/generate_test.go index 704468780..d0ace066c 100644 --- a/syft/pkg/cataloger/internal/cpegenerate/generate_test.go +++ b/syft/pkg/cataloger/internal/cpegenerate/generate_test.go @@ -182,6 +182,24 @@ func TestGeneratePackageCPEs(t *testing.T) { "cpe:2.3:a:name:name:3.2:*:*:*:*:*:*:*", }, }, + { + // groovy moved to org.apache.groovy at 4.0 and real jars ship no pom metadata, so the known package + // list drives the vendor. See https://github.com/anchore/syft/issues/5311 + name: "groovy 4 module without pom metadata", + p: pkg.Package{ + Name: "groovy-json", + Version: "4.0.33", + FoundBy: "some-analyzer", + Language: pkg.Java, + Type: pkg.JavaPkg, + Metadata: pkg.JavaArchive{Manifest: &pkg.JavaManifest{}}, + }, + expected: []string{ + "cpe:2.3:a:apache:groovy-json:4.0.33:*:*:*:*:*:*:*", + "cpe:2.3:a:apache:groovy_json:4.0.33:*:*:*:*:*:*:*", + "cpe:2.3:a:apache:groovy:4.0.33:*:*:*:*:*:*:*", + }, + }, { name: "java language with groupID", p: pkg.Package{ diff --git a/syft/pkg/cataloger/internal/cpegenerate/java.go b/syft/pkg/cataloger/internal/cpegenerate/java.go index 5f7129562..69d9f85c2 100644 --- a/syft/pkg/cataloger/internal/cpegenerate/java.go +++ b/syft/pkg/cataloger/internal/cpegenerate/java.go @@ -190,16 +190,16 @@ func GroupIDsFromJavaPackage(p pkg.Package) (groupIDs []string) { return nil } - return GroupIDsFromJavaMetadata(p.Name, metadata) + return GroupIDsFromJavaMetadata(p.Name, p.Version, metadata) } // GroupIDsFromJavaMetadata returns the possible group IDs for a Java package // This function is similar to GroupIDFromJavaPackage, but returns all possible group IDs and is less strict // It is used as a way to generate possible candidates for CPE matching. -func GroupIDsFromJavaMetadata(pkgName string, metadata pkg.JavaArchive) (groupIDs []string) { +func GroupIDsFromJavaMetadata(pkgName, version string, metadata pkg.JavaArchive) (groupIDs []string) { groupIDs = append(groupIDs, groupIDsFromPomProperties(metadata.PomProperties)...) groupIDs = append(groupIDs, groupIDsFromPomProject(metadata.PomProject)...) - groupIDs = append(groupIDs, groupIDsFromJavaManifest(pkgName, metadata.Manifest)...) + groupIDs = append(groupIDs, groupIDsFromJavaManifest(pkgName, version, metadata.Manifest)...) return groupIDs } @@ -253,8 +253,8 @@ func addGroupIDsFromGroupIDsAndArtifactID(groupID, artifactID string) (groupIDs return groupIDs } -func groupIDsFromJavaManifest(pkgName string, manifest *pkg.JavaManifest) []string { - if groupID, ok := DefaultArtifactIDToGroupID[pkgName]; ok { +func groupIDsFromJavaManifest(pkgName, version string, manifest *pkg.JavaManifest) []string { + if groupID, ok := ArtifactIDToGroupID(pkgName, version); ok { return []string{groupID} } @@ -303,7 +303,7 @@ func GetManifestFieldGroupIDs(manifest *pkg.JavaManifest, fields []string) (grou func cleanGroupID(groupID string) string { groupID = strings.TrimSpace(strings.Split(removeOSCIDirectives(groupID), "#")[0]) - if replacement, ok := GroupIDCorrections[groupID]; ok { + if replacement, ok := groupIDCorrections[groupID]; ok { return replacement } return groupID diff --git a/syft/pkg/cataloger/internal/cpegenerate/java_groupid_map.go b/syft/pkg/cataloger/internal/cpegenerate/java_groupid_map.go index 9dcb71567..9cb50ffd6 100644 --- a/syft/pkg/cataloger/internal/cpegenerate/java_groupid_map.go +++ b/syft/pkg/cataloger/internal/cpegenerate/java_groupid_map.go @@ -1,10 +1,17 @@ package cpegenerate -var GroupIDCorrections = map[string]string{ +import ( + "strconv" + "strings" +) + +var groupIDCorrections = map[string]string{ "org.lz4.java": "org.lz4", } -var DefaultArtifactIDToGroupID = map[string]string{ +// defaultArtifactIDToGroupID holds artifacts that always map to the same group ID. Look up group IDs with +// ArtifactIDToGroupID instead of reading this map directly, since migratedArtifactIDToGroupID takes precedence. +var defaultArtifactIDToGroupID = map[string]string{ "ant": "org.apache.ant", "ant-antlr": "org.apache.ant", "ant-antunit": "org.apache.ant", @@ -1832,104 +1839,83 @@ var DefaultArtifactIDToGroupID = map[string]string{ "commons-jelly-tags-velocity": "commons-jelly", "commons-jelly-tags-xml": "commons-jelly", "commons-jelly-tags-xmlunit": "commons-jelly", - "groovy": "org.codehaus.groovy", - "groovy-all": "org.codehaus.groovy", - "groovy-all-jdk14": "org.codehaus.groovy", - "groovy-all-minimal": "org.codehaus.groovy", - "groovy-all-tests": "org.codehaus.groovy", - "groovy-ant": "org.codehaus.groovy", - "groovy-astbuilder": "org.codehaus.groovy", - "groovy-backports-compat23": "org.codehaus.groovy", - "groovy-binary": "org.codehaus.groovy", - "groovy-bom": "org.codehaus.groovy", - "groovy-bsf": "org.codehaus.groovy", - "groovy-cli-commons": "org.codehaus.groovy", - "groovy-cli-picocli": "org.codehaus.groovy", - "groovy-console": "org.codehaus.groovy", - "groovy-datetime": "org.codehaus.groovy", - "groovy-dateutil": "org.codehaus.groovy", - "groovy-docgenerator": "org.codehaus.groovy", - "groovy-eclipse-batch": "org.codehaus.groovy", - "groovy-eclipse-compiler": "org.codehaus.groovy", - "groovy-groovydoc": "org.codehaus.groovy", - "groovy-groovysh": "org.codehaus.groovy", - "groovy-jaxb": "org.codehaus.groovy", - "groovy-jdk14": "org.codehaus.groovy", - "groovy-jmx": "org.codehaus.groovy", - "groovy-json": "org.codehaus.groovy", - "groovy-json-direct": "org.codehaus.groovy", - "groovy-jsr223": "org.codehaus.groovy", - "groovy-macro": "org.codehaus.groovy", - "groovy-nio": "org.codehaus.groovy", - "groovy-servlet": "org.codehaus.groovy", - "groovy-sql": "org.codehaus.groovy", - "groovy-swing": "org.codehaus.groovy", - "groovy-templates": "org.codehaus.groovy", - "groovy-test": "org.codehaus.groovy", - "groovy-test-junit5": "org.codehaus.groovy", - "groovy-testng": "org.codehaus.groovy", - "groovy-tests-vm8": "org.codehaus.groovy", - "groovy-xml": "org.codehaus.groovy", - "groovy-xmlrpc": "org.codehaus.groovy", - "groovy-yaml": "org.codehaus.groovy", - "kafka-clients": "org.apache.kafka", - "kafka-examples": "org.apache.kafka", - "kafka-group-coordinator": "org.apache.kafka", - "kafka-hadoop-consumer": "org.apache.kafka", - "kafka-hadoop-producer": "org.apache.kafka", - "kafka-java-examples": "org.apache.kafka", - "kafka-log4j-appender": "org.apache.kafka", - "kafka-metadata": "org.apache.kafka", - "kafka-perf_2.10": "org.apache.kafka", - "kafka-perf_2.8.0": "org.apache.kafka", - "kafka-perf_2.8.2": "org.apache.kafka", - "kafka-perf_2.9.1": "org.apache.kafka", - "kafka-perf_2.9.2": "org.apache.kafka", - "kafka-raft": "org.apache.kafka", - "kafka-server-common": "org.apache.kafka", - "kafka-shell": "org.apache.kafka", - "kafka-storage": "org.apache.kafka", - "kafka-storage-api": "org.apache.kafka", - "kafka-streams": "org.apache.kafka", - "kafka-streams-examples": "org.apache.kafka", - "kafka-streams-scala_2.11": "org.apache.kafka", - "kafka-streams-scala_2.12": "org.apache.kafka", - "kafka-streams-scala_2.13": "org.apache.kafka", - "kafka-streams-test-utils": "org.apache.kafka", - "kafka-streams-upgrade-system-tests-0100": "org.apache.kafka", - "kafka-streams-upgrade-system-tests-0101": "org.apache.kafka", - "kafka-streams-upgrade-system-tests-0102": "org.apache.kafka", - "kafka-streams-upgrade-system-tests-0110": "org.apache.kafka", - "kafka-streams-upgrade-system-tests-10": "org.apache.kafka", - "kafka-streams-upgrade-system-tests-11": "org.apache.kafka", - "kafka-streams-upgrade-system-tests-20": "org.apache.kafka", - "kafka-streams-upgrade-system-tests-21": "org.apache.kafka", - "kafka-streams-upgrade-system-tests-22": "org.apache.kafka", - "kafka-streams-upgrade-system-tests-23": "org.apache.kafka", - "kafka-streams-upgrade-system-tests-24": "org.apache.kafka", - "kafka-streams-upgrade-system-tests-25": "org.apache.kafka", - "kafka-streams-upgrade-system-tests-26": "org.apache.kafka", - "kafka-streams-upgrade-system-tests-27": "org.apache.kafka", - "kafka-streams-upgrade-system-tests-28": "org.apache.kafka", - "kafka-streams-upgrade-system-tests-30": "org.apache.kafka", - "kafka-streams-upgrade-system-tests-31": "org.apache.kafka", - "kafka-streams-upgrade-system-tests-32": "org.apache.kafka", - "kafka-streams-upgrade-system-tests-33": "org.apache.kafka", - "kafka-streams-upgrade-system-tests-34": "org.apache.kafka", - "kafka-streams-upgrade-system-tests-35": "org.apache.kafka", - "kafka-tools": "org.apache.kafka", - "kafka-tools-api": "org.apache.kafka", - "kafka_2.10": "org.apache.kafka", - "kafka_2.11": "org.apache.kafka", - "kafka_2.12": "org.apache.kafka", - "kafka_2.13": "org.apache.kafka", - "kafka_2.8.0": "org.apache.kafka", - "kafka_2.8.2": "org.apache.kafka", - "kafka_2.9.1": "org.apache.kafka", - "kafka_2.9.2": "org.apache.kafka", + // the Eclipse-hosted groovy-eclipse-* artifacts stayed on codehaus past 4.0, and the org.apache.groovy modules + // below were added in 4.0+ and only exist under apache. See https://github.com/anchore/syft/issues/5311 + "groovy-eclipse-batch": "org.codehaus.groovy", + "groovy-eclipse-compiler": "org.codehaus.groovy", + "groovy-callsite": "org.apache.groovy", + "groovy-concurrent-java": "org.apache.groovy", + "groovy-contracts": "org.apache.groovy", + "groovy-csv": "org.apache.groovy", + "groovy-ginq": "org.apache.groovy", + "groovy-grape-ivy": "org.apache.groovy", + "groovy-grape-maven": "org.apache.groovy", + "groovy-http-builder": "org.apache.groovy", + "groovy-macro-library": "org.apache.groovy", + "groovy-markdown": "org.apache.groovy", + "groovy-reactor": "org.apache.groovy", + "groovy-rxjava": "org.apache.groovy", + "groovy-test-junit6": "org.apache.groovy", + "groovy-toml": "org.apache.groovy", + "groovy-typecheckers": "org.apache.groovy", + "kafka-clients": "org.apache.kafka", + "kafka-examples": "org.apache.kafka", + "kafka-group-coordinator": "org.apache.kafka", + "kafka-hadoop-consumer": "org.apache.kafka", + "kafka-hadoop-producer": "org.apache.kafka", + "kafka-java-examples": "org.apache.kafka", + "kafka-log4j-appender": "org.apache.kafka", + "kafka-metadata": "org.apache.kafka", + "kafka-perf_2.10": "org.apache.kafka", + "kafka-perf_2.8.0": "org.apache.kafka", + "kafka-perf_2.8.2": "org.apache.kafka", + "kafka-perf_2.9.1": "org.apache.kafka", + "kafka-perf_2.9.2": "org.apache.kafka", + "kafka-raft": "org.apache.kafka", + "kafka-server-common": "org.apache.kafka", + "kafka-shell": "org.apache.kafka", + "kafka-storage": "org.apache.kafka", + "kafka-storage-api": "org.apache.kafka", + "kafka-streams": "org.apache.kafka", + "kafka-streams-examples": "org.apache.kafka", + "kafka-streams-scala_2.11": "org.apache.kafka", + "kafka-streams-scala_2.12": "org.apache.kafka", + "kafka-streams-scala_2.13": "org.apache.kafka", + "kafka-streams-test-utils": "org.apache.kafka", + "kafka-streams-upgrade-system-tests-0100": "org.apache.kafka", + "kafka-streams-upgrade-system-tests-0101": "org.apache.kafka", + "kafka-streams-upgrade-system-tests-0102": "org.apache.kafka", + "kafka-streams-upgrade-system-tests-0110": "org.apache.kafka", + "kafka-streams-upgrade-system-tests-10": "org.apache.kafka", + "kafka-streams-upgrade-system-tests-11": "org.apache.kafka", + "kafka-streams-upgrade-system-tests-20": "org.apache.kafka", + "kafka-streams-upgrade-system-tests-21": "org.apache.kafka", + "kafka-streams-upgrade-system-tests-22": "org.apache.kafka", + "kafka-streams-upgrade-system-tests-23": "org.apache.kafka", + "kafka-streams-upgrade-system-tests-24": "org.apache.kafka", + "kafka-streams-upgrade-system-tests-25": "org.apache.kafka", + "kafka-streams-upgrade-system-tests-26": "org.apache.kafka", + "kafka-streams-upgrade-system-tests-27": "org.apache.kafka", + "kafka-streams-upgrade-system-tests-28": "org.apache.kafka", + "kafka-streams-upgrade-system-tests-30": "org.apache.kafka", + "kafka-streams-upgrade-system-tests-31": "org.apache.kafka", + "kafka-streams-upgrade-system-tests-32": "org.apache.kafka", + "kafka-streams-upgrade-system-tests-33": "org.apache.kafka", + "kafka-streams-upgrade-system-tests-34": "org.apache.kafka", + "kafka-streams-upgrade-system-tests-35": "org.apache.kafka", + "kafka-tools": "org.apache.kafka", + "kafka-tools-api": "org.apache.kafka", + "kafka_2.10": "org.apache.kafka", + "kafka_2.11": "org.apache.kafka", + "kafka_2.12": "org.apache.kafka", + "kafka_2.13": "org.apache.kafka", + "kafka_2.8.0": "org.apache.kafka", + "kafka_2.8.2": "org.apache.kafka", + "kafka_2.9.1": "org.apache.kafka", + "kafka_2.9.2": "org.apache.kafka", // legacy "Jackson 1.x" (aka "jackson-asl") artifacts predate the convention of embedding - // META-INF/maven/.../pom.properties in the jar, so groupIDFromKnownPackageList is the only + // META-INF/maven/.../pom.properties in the jar, so ArtifactIDToGroupID is the only // way to recover the correct group ID for jars built without that metadata (e.g. Ant-built // jars from before ~2014). Without this, the group ID falls back to the artifact name itself, // producing purls that do not match the vulnerability database's namespace. @@ -1940,3 +1926,83 @@ var DefaultArtifactIDToGroupID = map[string]string{ "jackson-xc": "org.codehaus.jackson", "jackson-smile": "org.codehaus.jackson", } + +// groupIDMigration describes an artifact that moved from one group ID to another at a major version. +type groupIDMigration struct { + before, after string + atMajor int +} + +// groovy moved from org.codehaus.groovy to org.apache.groovy at 4.0 +var groovyMigration = groupIDMigration{before: "org.codehaus.groovy", after: "org.apache.groovy", atMajor: 4} + +// migratedArtifactIDToGroupID holds artifacts whose group ID depends on the version. It takes precedence over +// defaultArtifactIDToGroupID; use ArtifactIDToGroupID rather than reading it directly. +var migratedArtifactIDToGroupID = map[string]groupIDMigration{ + // groovy artifacts published under both coordinates. Without these, jars that ship no pom metadata get a + // stale codehaus group or one guessed from the per-module Automatic-Module-Name (e.g. org.apache.groovy.json). + // See https://github.com/anchore/syft/issues/5311 + "groovy": groovyMigration, + "groovy-all": groovyMigration, + "groovy-all-jdk14": groovyMigration, + "groovy-all-minimal": groovyMigration, + "groovy-all-tests": groovyMigration, + "groovy-ant": groovyMigration, + "groovy-astbuilder": groovyMigration, + "groovy-backports-compat23": groovyMigration, + "groovy-binary": groovyMigration, + "groovy-bom": groovyMigration, + "groovy-bsf": groovyMigration, + "groovy-cli-commons": groovyMigration, + "groovy-cli-picocli": groovyMigration, + "groovy-console": groovyMigration, + "groovy-datetime": groovyMigration, + "groovy-dateutil": groovyMigration, + "groovy-docgenerator": groovyMigration, + "groovy-groovydoc": groovyMigration, + "groovy-groovysh": groovyMigration, + "groovy-jaxb": groovyMigration, + "groovy-jdk14": groovyMigration, + "groovy-jmx": groovyMigration, + "groovy-json": groovyMigration, + "groovy-json-direct": groovyMigration, + "groovy-jsr223": groovyMigration, + "groovy-macro": groovyMigration, + "groovy-nio": groovyMigration, + "groovy-servlet": groovyMigration, + "groovy-sql": groovyMigration, + "groovy-swing": groovyMigration, + "groovy-templates": groovyMigration, + "groovy-test": groovyMigration, + "groovy-test-junit5": groovyMigration, + "groovy-testng": groovyMigration, + "groovy-tests-vm8": groovyMigration, + "groovy-xml": groovyMigration, + "groovy-xmlrpc": groovyMigration, + "groovy-yaml": groovyMigration, +} + +// ArtifactIDToGroupID returns the known group ID for the given artifact ID at the given version. For migrated +// artifacts an empty or unparseable version resolves to the pre-migration group ID. +func ArtifactIDToGroupID(artifactID, version string) (string, bool) { + if m, ok := migratedArtifactIDToGroupID[artifactID]; ok { + if majorVersion(version) >= m.atMajor { + return m.after, true + } + return m.before, true + } + groupID, ok := defaultArtifactIDToGroupID[artifactID] + return groupID, ok +} + +// majorVersion returns the leading numeric component of a version (e.g. "4.0.33" and "4-SNAPSHOT" +// resolve to 4), or 0 when the version does not start with a number. +func majorVersion(version string) int { + leading, _, _ := strings.Cut(version, ".") + leading, _, _ = strings.Cut(leading, "-") + n, err := strconv.Atoi(leading) + if err != nil { + return 0 + } + return n +} diff --git a/syft/pkg/cataloger/internal/cpegenerate/java_groupid_map_test.go b/syft/pkg/cataloger/internal/cpegenerate/java_groupid_map_test.go new file mode 100644 index 000000000..8232f0e93 --- /dev/null +++ b/syft/pkg/cataloger/internal/cpegenerate/java_groupid_map_test.go @@ -0,0 +1,49 @@ +package cpegenerate + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestArtifactIDToGroupID(t *testing.T) { + tests := []struct { + name string + artifactID string + version string + want string + wantOK bool + }{ + {name: "unconditional entry", artifactID: "ant-antlr", version: "1.10.0", want: "org.apache.ant", wantOK: true}, + {name: "unknown artifact", artifactID: "not-a-real-artifact", version: "1.0.0"}, + {name: "groovy 4 migrates to apache", artifactID: "groovy", version: "4.0.33", want: "org.apache.groovy", wantOK: true}, + {name: "groovy 3 stays on codehaus", artifactID: "groovy", version: "3.0.22", want: "org.codehaus.groovy", wantOK: true}, + {name: "groovy without a version falls back to codehaus", artifactID: "groovy-json", want: "org.codehaus.groovy", wantOK: true}, + {name: "groovy-eclipse stays on codehaus at 4.x", artifactID: "groovy-eclipse-batch", version: "4.0.33-02", want: "org.codehaus.groovy", wantOK: true}, + {name: "apache-only groovy module", artifactID: "groovy-ginq", version: "4.0.33", want: "org.apache.groovy", wantOK: true}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, ok := ArtifactIDToGroupID(tt.artifactID, tt.version) + assert.Equal(t, tt.wantOK, ok) + assert.Equal(t, tt.want, got) + }) + } +} + +func Test_majorVersion(t *testing.T) { + tests := map[string]int{ + "4.0.33": 4, + "4.0.0-alpha-1": 4, + "4-SNAPSHOT": 4, + "4": 4, + "3.0.22": 3, + "v4.0.33": 0, + "": 0, + } + for version, expect := range tests { + t.Run(version, func(t *testing.T) { + assert.Equal(t, expect, majorVersion(version)) + }) + } +} diff --git a/syft/pkg/cataloger/internal/cpegenerate/java_test.go b/syft/pkg/cataloger/internal/cpegenerate/java_test.go index 6529598c7..6b9211a41 100644 --- a/syft/pkg/cataloger/internal/cpegenerate/java_test.go +++ b/syft/pkg/cataloger/internal/cpegenerate/java_test.go @@ -400,6 +400,16 @@ func Test_groupIDsFromJavaPackage(t *testing.T) { }, expects: nil, }, + { + // pom-less groovy 4 jars (all real ones) must resolve to the apache group, same as the purl + name: "groovy 4 without pom metadata", + pkg: pkg.Package{ + Name: "groovy", + Version: "4.0.33", + Metadata: pkg.JavaArchive{Manifest: &pkg.JavaManifest{}}, + }, + expects: []string{"org.apache.groovy"}, + }, { name: "no manifest or pom info", pkg: pkg.Package{ @@ -562,7 +572,7 @@ func Test_groupIDsFromJavaManifest(t *testing.T) { for _, test := range tests { t.Run(test.name, func(t *testing.T) { - got := groupIDsFromJavaManifest(test.name, &test.manifest) + got := groupIDsFromJavaManifest(test.name, "", &test.manifest) require.Equal(t, test.expected, got) }) } diff --git a/syft/pkg/cataloger/java/archive_parser.go b/syft/pkg/cataloger/java/archive_parser.go index 0ec5b0054..d5608b89a 100644 --- a/syft/pkg/cataloger/java/archive_parser.go +++ b/syft/pkg/cataloger/java/archive_parser.go @@ -327,7 +327,7 @@ func (j *archiveParser) discoverNameVersionLicense(ctx context.Context, manifest // findLicenseFromJavaMetadata attempts to find license information from all available maven metadata properties and pom info func (j *archiveParser) findLicenseFromJavaMetadata(ctx context.Context, groupID, artifactID, version string, parsedPom *parsedPomProject, manifest *pkg.JavaManifest) []pkg.License { if groupID == "" { - if gID := groupIDFromJavaMetadata(artifactID, pkg.JavaArchive{Manifest: manifest}); gID != "" { + if gID := groupIDFromJavaMetadata(artifactID, version, pkg.JavaArchive{Manifest: manifest}); gID != "" { groupID = gID } } @@ -786,7 +786,7 @@ func newPackageFromMavenData(ctx context.Context, r *maven.Resolver, pomProperti vPathSuffix := "" groupID := "" if parentMetadata, ok := parentPkg.Metadata.(pkg.JavaArchive); ok { - groupID = groupIDFromJavaMetadata(parentPkg.Name, parentMetadata) + groupID = groupIDFromJavaMetadata(parentPkg.Name, parentPkg.Version, parentMetadata) } parentKey := fmt.Sprintf("%s:%s:%s", groupID, parentPkg.Name, parentPkg.Version) @@ -863,8 +863,8 @@ func packageIdentitiesMatch(p pkg.Package, parentPkg *pkg.Package) bool { } // try to determine identity with the metadata - groupID := groupIDFromJavaMetadata(p.Name, metadata) - parentGroupID := groupIDFromJavaMetadata(parentPkg.Name, parentMetadata) + groupID := groupIDFromJavaMetadata(p.Name, p.Version, metadata) + parentGroupID := groupIDFromJavaMetadata(parentPkg.Name, parentPkg.Version, parentMetadata) if uniquePkgKey(groupID, &p) == uniquePkgKey(parentGroupID, parentPkg) { return true } diff --git a/syft/pkg/cataloger/java/package_url.go b/syft/pkg/cataloger/java/package_url.go index e47e9a7e6..477b89a46 100644 --- a/syft/pkg/cataloger/java/package_url.go +++ b/syft/pkg/cataloger/java/package_url.go @@ -12,7 +12,7 @@ import ( func packageURL(name, version string, metadata pkg.JavaArchive) string { var groupID = name - if gID := groupIDFromJavaMetadata(name, metadata); gID != "" { + if gID := groupIDFromJavaMetadata(name, version, metadata); gID != "" { groupID = gID } @@ -30,9 +30,9 @@ func packageURL(name, version string, metadata pkg.JavaArchive) string { // The order of precedence is: // 1. The group ID from the POM properties // 2. The group ID from the POM project -// 3. The group ID from a select map of known group IDs +// 3. The group ID from a select map of known group IDs (see cpegenerate.ArtifactIDToGroupID) // 4. The group ID from the Java manifest -func groupIDFromJavaMetadata(pkgName string, metadata pkg.JavaArchive) (groupID string) { +func groupIDFromJavaMetadata(pkgName, version string, metadata pkg.JavaArchive) (groupID string) { if groupID = groupIDFromPomProperties(metadata.PomProperties); groupID != "" { return groupID } @@ -41,8 +41,8 @@ func groupIDFromJavaMetadata(pkgName string, metadata pkg.JavaArchive) (groupID return groupID } - if groupID = groupIDFromKnownPackageList(pkgName); groupID != "" { - return groupID + if gID, ok := cpegenerate.ArtifactIDToGroupID(pkgName, version); ok { + return gID } if groupID = groupIDFromJavaManifest(metadata.Manifest); groupID != "" { @@ -52,13 +52,6 @@ func groupIDFromJavaMetadata(pkgName string, metadata pkg.JavaArchive) (groupID return groupID } -func groupIDFromKnownPackageList(pkgName string) (groupID string) { - if groupID, ok := cpegenerate.DefaultArtifactIDToGroupID[pkgName]; ok { - return groupID - } - return groupID -} - func groupIDFromJavaManifest(manifest *pkg.JavaManifest) (groupID string) { if manifest == nil { return groupID diff --git a/syft/pkg/cataloger/java/package_url_test.go b/syft/pkg/cataloger/java/package_url_test.go index 5bb92fd00..a6fa7d973 100644 --- a/syft/pkg/cataloger/java/package_url_test.go +++ b/syft/pkg/cataloger/java/package_url_test.go @@ -132,6 +132,26 @@ func Test_packageURL(t *testing.T) { }, expect: "pkg:maven/parent/example-java-app-maven@0.1.0", }, + { + // regression for github.com/anchore/syft/issues/5311 + name: "groovy 4 without pom metadata", + pkg: pkg.Package{ + Name: "groovy", + Version: "4.0.33", + Language: pkg.Java, + Type: pkg.JavaPkg, + Metadata: pkg.JavaArchive{ + VirtualPath: "groovy-4.0.33.jar", + Manifest: &pkg.JavaManifest{ + Main: []pkg.KeyValue{ + {Key: "Bundle-SymbolicName", Value: "groovy"}, + {Key: "Automatic-Module-Name", Value: "org.apache.groovy.core"}, + }, + }, + }, + }, + expect: "pkg:maven/org.apache.groovy/groovy@4.0.33", + }, } for _, tt := range tests { t.Run(tt.expect, func(t *testing.T) { @@ -144,6 +164,7 @@ func Test_groupIDFromJavaMetadata(t *testing.T) { tests := []struct { name string pkgName string + version string metadata pkg.JavaArchive expect string }{ @@ -218,6 +239,38 @@ func Test_groupIDFromJavaMetadata(t *testing.T) { metadata: pkg.JavaArchive{}, expect: "org.apache.derby", }, + { + // regression for github.com/anchore/syft/issues/5311: groovy 4+ moved to org.apache.groovy, but + // jars without pom metadata fall through to the known package list, which keeps the codehaus-era + // coordinates. The map must also beat the per-module Automatic-Module-Name in the manifest. + name: "known package list groovy 4 beats the manifest", + pkgName: "groovy", + version: "4.0.33", + metadata: pkg.JavaArchive{ + Manifest: &pkg.JavaManifest{ + Main: []pkg.KeyValue{ + {Key: "Bundle-SymbolicName", Value: "groovy"}, + {Key: "Automatic-Module-Name", Value: "org.apache.groovy.core"}, + }, + }, + }, + expect: "org.apache.groovy", + }, + { + // modules only published under the apache coordinates would otherwise take the group ID + // from the per-module Automatic-Module-Name (org.apache.groovy.ginq) + name: "known package list apache-only groovy module", + pkgName: "groovy-ginq", + version: "4.0.33", + metadata: pkg.JavaArchive{ + Manifest: &pkg.JavaManifest{ + Main: []pkg.KeyValue{ + {Key: "Automatic-Module-Name", Value: "org.apache.groovy.ginq"}, + }, + }, + }, + expect: "org.apache.groovy", + }, { name: "java manifest", metadata: pkg.JavaArchive{ @@ -241,7 +294,7 @@ func Test_groupIDFromJavaMetadata(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - assert.Equal(t, tt.expect, groupIDFromJavaMetadata(tt.pkgName, tt.metadata)) + assert.Equal(t, tt.expect, groupIDFromJavaMetadata(tt.pkgName, tt.version, tt.metadata)) }) } } From af508550e6dc672920f0715c5fa827ac8b95ff06 Mon Sep 17 00:00:00 2001 From: Alex Goodman Date: Thu, 24 Sep 2026 10:05:59 -0400 Subject: [PATCH 05/40] test: pin multiplatform OCI fixture by digest (#5323) Docker Hub rebuilt and re-pushed `busybox:1.38.0`, which changed the per-platform config digests and broke `TestMultiPlatformOCIImageSelection` on uncached runs. - pin the fixture to the busybox 1.38.0 index digest so future re-pushes can't shift it - update the expected per-platform digests to match Signed-off-by: Alex Goodman --- .../integration/multiplatform_oci_test.go | 24 +++++++++---------- 1 file changed, 11 insertions(+), 13 deletions(-) diff --git a/cmd/syft/internal/test/integration/multiplatform_oci_test.go b/cmd/syft/internal/test/integration/multiplatform_oci_test.go index 3373d8f36..411095849 100644 --- a/cmd/syft/internal/test/integration/multiplatform_oci_test.go +++ b/cmd/syft/internal/test/integration/multiplatform_oci_test.go @@ -16,19 +16,21 @@ import ( "github.com/anchore/syft/syft/source" ) +// multiplatformImage is busybox:1.38.0 pinned by index digest, since docker hub periodically rebuilds and +// re-pushes official image tags, which changes the per-platform digests asserted below. +const multiplatformImage = "docker.io/library/busybox@sha256:fd7dc98638c8e305f4dc34e979f1c0fdfdcaeb0fbf8fcff77ae834b6da3d7e6e" + // TestMultiPlatformOCIImageSelection verifies that syft can load a single platform out of a // multi-platform OCI image on disk (both --from oci-dir and --from oci-archive) by selecting the // image that matches the requested platform. func TestMultiPlatformOCIImageSelection(t *testing.T) { - remoteImage := "docker.io/library/busybox:1.38.0" - // Per-platform image config digests within the multi-platform index, obtained from the OCI layout // (see anchore/stereoscope integration tests: TestPlatformSelectionWithOciLocalSources). expectedDigest := map[string]string{ - "arm64": "sha256:e0e8b3cbfed68a90084781e2962f9c0deead51c5a3f11a488eef0283a4284bc2", - "s390x": "sha256:0cf160e720a8e4f20883b72276a6eaded83b79539f3f1d39e35a3336154b9960", - "amd64": "sha256:c6348fa86ba0fb2108c9334f5fe913ddc6d853313e655891f133a0127c30099f", - "ppc64le": "sha256:b144fc0e06537d07956cde340b878a81a717e394edb736d3110858a12a6635cb", + "arm64": "sha256:d2482869a6b838d3b4c9cad0c8085c06277909482771eb525d298fc3a7d07927", + "s390x": "sha256:e55a0fa55032cceac2d983f4b4a25b28e42303b46a2f1915fc0fa45051e6ca70", + "amd64": "sha256:aaef90e065235eb0b2d9938be85d1b81add2902cf89affe2cdfa19be33476ac0", + "ppc64le": "sha256:8158829b015d1c5285d63d54d37e95998ef0af4177e3f416678decbcaff3170c", } // syft --from source tag -> stereoscope OCI source used to prepare the local fixture @@ -39,7 +41,7 @@ func TestMultiPlatformOCIImageSelection(t *testing.T) { for from, imageSource := range sources { t.Run(from, func(t *testing.T) { - localPath := imagetest.PrepareMultiplatformFixtureImage(t, imageSource, remoteImage) + localPath := imagetest.PrepareMultiplatformFixtureImage(t, imageSource, multiplatformImage) for _, arch := range []string{"amd64", "arm64", "s390x", "ppc64le"} { t.Run(fmt.Sprintf("linux/%s", arch), func(t *testing.T) { platform, err := image.NewPlatform("linux/" + arch) @@ -73,8 +75,6 @@ func TestMultiPlatformOCIImageSelection(t *testing.T) { // TestMultiPlatformOCIImageSelection_UnavailablePlatform verifies that requesting a platform not present // in the multi-platform OCI image results in an error rather than silently selecting the wrong image. func TestMultiPlatformOCIImageSelection_UnavailablePlatform(t *testing.T) { - remoteImage := "docker.io/library/busybox:1.38.0" - // windows/amd64 is not present in this linux-only multi-platform image platform, err := image.NewPlatform("windows/amd64") require.NoError(t, err) @@ -86,7 +86,7 @@ func TestMultiPlatformOCIImageSelection_UnavailablePlatform(t *testing.T) { for from, imageSource := range sources { t.Run(from, func(t *testing.T) { - localPath := imagetest.PrepareMultiplatformFixtureImage(t, imageSource, remoteImage) + localPath := imagetest.PrepareMultiplatformFixtureImage(t, imageSource, multiplatformImage) _, err := syft.GetSource( context.Background(), @@ -101,8 +101,6 @@ func TestMultiPlatformOCIImageSelection_UnavailablePlatform(t *testing.T) { // TestMultiPlatformOCIImageSelection_DefaultPlatform verifies that not specifying a platform results // in the current platform being selected. func TestMultiPlatformOCIImageSelection_DefaultPlatform(t *testing.T) { - remoteImage := "docker.io/library/busybox:1.38.0" - sources := map[string]image.Source{ "oci-dir": image.OciDirectorySource, "oci-archive": image.OciTarballSource, @@ -110,7 +108,7 @@ func TestMultiPlatformOCIImageSelection_DefaultPlatform(t *testing.T) { for from, imageSource := range sources { t.Run(from, func(t *testing.T) { - localPath := imagetest.PrepareMultiplatformFixtureImage(t, imageSource, remoteImage) + localPath := imagetest.PrepareMultiplatformFixtureImage(t, imageSource, multiplatformImage) src, err := syft.GetSource( context.Background(), From 14e96fcbf81e1db03c8b1ace62afa52c3fdc5865 Mon Sep 17 00:00:00 2001 From: Alex Goodman Date: Thu, 24 Sep 2026 10:40:52 -0400 Subject: [PATCH 06/40] fix(ui): render TUI log lines on their own rows, with colors and a small header (#5324) - log lines shown under the TUI were glued onto the last UI row, and the last row could get erased (and overwritten by the report) when the TUI exits. Fixed in `bubbly` v0.2.2 - log colors were lost because the logger latched its TTY check on the first entry, which went to the TUI footer buffer. `go-logger` v0.2.0 checks per entry and supports `TerminalWriter`, which the footer writer now implements (`NO_COLOR` is still honored) - log lines under the TUI get a dim `Logs (non-fatal, see -v for more):` title with a branch pointing at the first entry, so it's clear what the trailing lines are Signed-off-by: Alex Goodman --- cmd/syft/internal/ui/ui.go | 46 +++++++++++++++++++++++++++++++++++++- go.mod | 4 ++-- go.sum | 8 +++---- 3 files changed, 51 insertions(+), 7 deletions(-) diff --git a/cmd/syft/internal/ui/ui.go b/cmd/syft/internal/ui/ui.go index 193768c51..4673a2917 100644 --- a/cmd/syft/internal/ui/ui.go +++ b/cmd/syft/internal/ui/ui.go @@ -1,6 +1,7 @@ package ui import ( + "bytes" "fmt" "io" "os" @@ -8,6 +9,7 @@ import ( "time" tea "github.com/charmbracelet/bubbletea" + "github.com/charmbracelet/lipgloss" "github.com/wagoodman/go-partybus" "github.com/anchore/bubbly" @@ -52,7 +54,7 @@ func New(out io.Writer, quiet bool, handlers ...bubbly.EventHandler) *UI { func (m *UI) Setup(subscription partybus.Unsubscribable) error { // we still want to collect log messages, however, we also the logger shouldn't write to the screen directly if logWrapper, ok := log.Get().(logger.Controller); ok { - logWrapper.SetOutput(m.frame.(*frame.Frame).Footer()) + logWrapper.SetOutput(&logFooter{w: m.frame.(*frame.Frame).Footer()}) } m.subscription = subscription @@ -187,6 +189,48 @@ func (m UI) View() string { return m.frame.View() } +var ( + logTitleStyle = lipgloss.NewStyle().Foreground(lipgloss.Color("8")) // dark grey + // the first log line gets a branch pointing at it from the title, the rest are indented to match + logBranch = logTitleStyle.Render("└─▶") + " " + logIndent = " " +) + +// logFooter decorates log entries drawn below the TUI with a one-time title (written when the first entry arrives) +// and a branch from the title to the first log line. Writes are serialized by the logger. +type logFooter struct { + w io.Writer + started bool +} + +func (l *logFooter) Write(p []byte) (int, error) { + var buf bytes.Buffer + first := !l.started + if first { + buf.WriteString(logTitleStyle.Render("Logs (non-fatal, see -v for more):") + "\n") + l.started = true + } + for i, line := range bytes.SplitAfter(p, []byte("\n")) { + if len(line) == 0 { + continue + } + if first && i == 0 { + buf.WriteString(logBranch) + } else { + buf.WriteString(logIndent) + } + buf.Write(line) + } + if _, err := l.w.Write(buf.Bytes()); err != nil { + return 0, err + } + return len(p), nil +} + +// IsTerminal marks the footer as terminal-bound (it is drawn by the TUI) so the logger keeps colors. NO_COLOR is +// still honored by the formatter. +func (*logFooter) IsTerminal() bool { return true } + func runWithTimeout(timeout time.Duration, fn func() error) (err error) { c := make(chan struct{}, 1) go func() { diff --git a/go.mod b/go.mod index 8c49b84ff..327deee1d 100644 --- a/go.mod +++ b/go.mod @@ -11,12 +11,12 @@ require ( github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d github.com/acobaugh/osrelease v0.1.0 github.com/adrg/xdg v0.5.3 - github.com/anchore/bubbly v0.2.1 + github.com/anchore/bubbly v0.2.2 github.com/anchore/clio v0.1.1 github.com/anchore/fangs v0.1.1 github.com/anchore/go-collections v0.1.1 github.com/anchore/go-homedir v0.1.1 - github.com/anchore/go-logger v0.1.1 + github.com/anchore/go-logger v0.2.0 github.com/anchore/go-macholibre v0.1.1 github.com/anchore/go-rpmdb v0.2.0 github.com/anchore/go-sync v0.1.2 diff --git a/go.sum b/go.sum index 99c006c5d..6e9b0a686 100644 --- a/go.sum +++ b/go.sum @@ -120,8 +120,8 @@ github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuy github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= -github.com/anchore/bubbly v0.2.1 h1:b3ZNBefY8ks5DI4SQ7BXSbBIsN6eoaTD6s1su7NuCBo= -github.com/anchore/bubbly v0.2.1/go.mod h1:o6a9aH3/fCn+mb5g1T1tBbQZ4hGR0yO5/uD1d2Vtago= +github.com/anchore/bubbly v0.2.2 h1:53JHRlpaNZjMcUYdz7n1U7+kOhy0zsJv9gpGEXvTcz8= +github.com/anchore/bubbly v0.2.2/go.mod h1:M4twoFkczlfuNdU8QV7oQjVJwPbxdUWRMfKydcLba0U= github.com/anchore/clio v0.1.1 h1:9bkYnaFbpT0J3G0qSHyQqa/lpK5zV0C9Gtgh82H/jj0= github.com/anchore/clio v0.1.1/go.mod h1:CFPIopJlakJbBpKtAL/EPpj4W9hgBO7PfFc8N2FAtlE= github.com/anchore/fangs v0.1.1 h1:NBTHocmWM2/0gNhebpoE/wEB0Z0nxh2lESww2o/Uf5M= @@ -130,8 +130,8 @@ github.com/anchore/go-collections v0.1.1 h1:kGlmSehs+ctTBZr+98F2+taHeolHy+XDx1Nc github.com/anchore/go-collections v0.1.1/go.mod h1:zgDbPSNDpfU47tVo7aW+2suG7+vXB1Sq1qtDhRJknQ0= github.com/anchore/go-homedir v0.1.1 h1:KpaO5BPrVT7cVMcEZ9KfFfEi0iuo3DtagPGl0C744X0= github.com/anchore/go-homedir v0.1.1/go.mod h1:9B0DGhbmMAJVGGEbrlb+PkORM5eDFCEOtZ3xQ22qaLA= -github.com/anchore/go-logger v0.1.1 h1:HPBNmQVBdYliwaFVUL7oJ3UvSlaoXAMj/ZagAIYaCpA= -github.com/anchore/go-logger v0.1.1/go.mod h1:ekWuh5BkZVwyXnyEd4fwL29N4pNLssgRsqh2+kFN/b8= +github.com/anchore/go-logger v0.2.0 h1:V8Gazz+F8xwWEWz+whqt8/5163IkO4XrAie+1qbK10k= +github.com/anchore/go-logger v0.2.0/go.mod h1:omqLWBwT/B5/aTwIH6pl5WBylBzeH3BGHUTS6OFeFvY= github.com/anchore/go-lzo v0.1.1 h1:IwL/fvkdtlIrYIXck6WxZ3nb8WjjHziYYmGxlooyOnM= github.com/anchore/go-lzo v0.1.1/go.mod h1:3kLx0bve2oN1iDwgM1U5zGku1Tfbdb0No5qp1eL1fIk= github.com/anchore/go-macholibre v0.1.1 h1:sTBHi3S1WVIOCamc6NyT8Sn9mBmD30DCTfjfVm4k6/M= From cc1bb1f28cb39eab5163de1e2218542f188b51e4 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 25 Sep 2026 13:28:33 +0000 Subject: [PATCH 07/40] chore(deps): bump github.com/hashicorp/hcl/v2 from 2.24.0 to 2.25.0 (#5332) Bumps [github.com/hashicorp/hcl/v2](https://github.com/hashicorp/hcl) from 2.24.0 to 2.25.0. - [Release notes](https://github.com/hashicorp/hcl/releases) - [Changelog](https://github.com/hashicorp/hcl/blob/main/CHANGELOG.md) - [Commits](https://github.com/hashicorp/hcl/compare/v2.24.0...v2.25.0) --- updated-dependencies: - dependency-name: github.com/hashicorp/hcl/v2 dependency-version: 2.25.0 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 5 +++-- go.sum | 10 ++++++---- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/go.mod b/go.mod index 327deee1d..339e66330 100644 --- a/go.mod +++ b/go.mod @@ -56,7 +56,7 @@ require ( github.com/hashicorp/go-cleanhttp v0.5.2 github.com/hashicorp/go-getter v1.8.9 github.com/hashicorp/go-multierror v1.1.1 - github.com/hashicorp/hcl/v2 v2.24.0 + github.com/hashicorp/hcl/v2 v2.25.0 github.com/iancoleman/strcase v0.3.0 github.com/invopop/jsonschema v0.14.0 github.com/jedib0t/go-pretty/v6 v6.8.3 @@ -131,6 +131,7 @@ require ( github.com/anchore/go-struct-converter v0.2.0-rc2 // indirect github.com/andybalholm/brotli v1.2.0 // indirect github.com/apparentlymart/go-textseg/v15 v15.0.0 // indirect + github.com/apparentlymart/go-textseg/v17 v17.0.1 // indirect github.com/aquasecurity/go-version v0.0.1 // indirect github.com/atotto/clipboard v0.1.4 // indirect github.com/aws/aws-sdk-go-v2 v1.44.0 // indirect @@ -293,7 +294,7 @@ require ( github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 // indirect github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8 // indirect github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e // indirect - github.com/zclconf/go-cty v1.16.3 // indirect + github.com/zclconf/go-cty v1.19.0 // indirect go.opencensus.io v0.24.0 // indirect go.opentelemetry.io/auto/sdk v1.2.1 // indirect go.opentelemetry.io/contrib/detectors/gcp v1.44.0 // indirect diff --git a/go.sum b/go.sum index 6e9b0a686..b32684293 100644 --- a/go.sum +++ b/go.sum @@ -156,6 +156,8 @@ github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be/go.mod h1:ySMOLuW github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= github.com/apparentlymart/go-textseg/v15 v15.0.0 h1:uYvfpb3DyLSCGWnctWKGj857c6ew1u1fNQOlOtuGxQY= github.com/apparentlymart/go-textseg/v15 v15.0.0/go.mod h1:K8XmNZdhEBkdlyDdvbmmsvpAG721bKi0joRfFdHIWJ4= +github.com/apparentlymart/go-textseg/v17 v17.0.1 h1:bpMXRgQ5cEoRNuQke1a80/Nl6w3G5eoIbWo9f3gXkAs= +github.com/apparentlymart/go-textseg/v17 v17.0.1/go.mod h1:fa8X4jgGeevslICIY6LcdjkSecWnXmYd9Lk34z/VxZs= github.com/aquasecurity/go-pep440-version v0.0.1 h1:8VKKQtH2aV61+0hovZS3T//rUF+6GDn18paFTVS0h0M= github.com/aquasecurity/go-pep440-version v0.0.1/go.mod h1:3naPe+Bp6wi3n4l5iBFCZgS0JG8vY6FT0H4NGhFJ+i4= github.com/aquasecurity/go-version v0.0.1 h1:4cNl516agK0TCn5F7mmYN+xVs1E3S45LkgZk3cbaW2E= @@ -595,8 +597,8 @@ github.com/hashicorp/golang-lru v0.5.4/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uG github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k= github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM= github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= -github.com/hashicorp/hcl/v2 v2.24.0 h1:2QJdZ454DSsYGoaE6QheQZjtKZSUs9Nh2izTWiwQxvE= -github.com/hashicorp/hcl/v2 v2.24.0/go.mod h1:oGoO1FIQYfn/AgyOhlg9qLC6/nOJPX3qGbkZpYAcqfM= +github.com/hashicorp/hcl/v2 v2.25.0 h1:HmmQVYRny4MaBo4b20TjmL46wyuUxpnMWkPZ4+NTbWk= +github.com/hashicorp/hcl/v2 v2.25.0/go.mod h1:vR+FKETxoZAmRlHgFfKmuqivj+C4Izm/c66XkmZ3r7M= github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO+LraFDTW64= github.com/hashicorp/mdns v1.0.1/go.mod h1:4gW7WsVCke5TE7EPeYliwHlRUyBtfCwuFwuMg2DmyNY= github.com/hashicorp/mdns v1.0.4/go.mod h1:mtBihi+LeNXGtG8L9dX59gAEa12BDtBQSp4v/YAJqrc= @@ -976,8 +978,8 @@ github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9de github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= -github.com/zclconf/go-cty v1.16.3 h1:osr++gw2T61A8KVYHoQiFbFd1Lh3JOCXc/jFLJXKTxk= -github.com/zclconf/go-cty v1.16.3/go.mod h1:VvMs5i0vgZdhYawQNq5kePSpLAoz8u1xvZgrPIxfnZE= +github.com/zclconf/go-cty v1.19.0 h1:IV8WdqYZc2c5rLX9bEoLNXKojBAp0MZPBHMIrCoa/s4= +github.com/zclconf/go-cty v1.19.0/go.mod h1:12W89jGn3JCOIQi7infWr9m80rOkb5RNYJqXMZcN4c8= github.com/zclconf/go-cty-debug v0.0.0-20240509010212-0d6042c53940 h1:4r45xpDWB6ZMSMNJFMOjqrGHynW3DIBuR2H9j0ug+Mo= github.com/zclconf/go-cty-debug v0.0.0-20240509010212-0d6042c53940/go.mod h1:CmBdvvj3nqzfzJ6nTCIwDTPZ56aVGvDrmztiO5g3qrM= github.com/zyedidia/generic v1.2.2-0.20230320175451-4410d2372cb1 h1:V+UsotZpAVvfj3X/LMoEytoLzSiP6Lg0F7wdVyu9gGg= From 82718d36ca2f64a7c9427c8f4561e521071c9c8c Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 25 Sep 2026 13:29:03 +0000 Subject: [PATCH 08/40] chore(deps): bump github.com/dustin/go-humanize from 1.0.1 to 1.1.0 (#5334) Bumps [github.com/dustin/go-humanize](https://github.com/dustin/go-humanize) from 1.0.1 to 1.1.0. - [Commits](https://github.com/dustin/go-humanize/compare/v1.0.1...v1.1.0) --- updated-dependencies: - dependency-name: github.com/dustin/go-humanize dependency-version: 1.1.0 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 339e66330..d6606dfda 100644 --- a/go.mod +++ b/go.mod @@ -36,7 +36,7 @@ require ( github.com/deitch/magic v0.0.0-20230404182410-1ff89d7342da github.com/diskfs/go-diskfs v1.9.4 github.com/distribution/reference v0.6.0 - github.com/dustin/go-humanize v1.0.1 + github.com/dustin/go-humanize v1.1.0 github.com/elliotchance/phpserialize v1.4.0 github.com/facebookincubator/nvdtools v0.1.5 github.com/github/go-spdx/v2 v2.7.0 diff --git a/go.sum b/go.sum index b32684293..5c08de104 100644 --- a/go.sum +++ b/go.sum @@ -350,8 +350,8 @@ github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDD github.com/dsnet/compress v0.0.2-0.20230904184137-39efe44ab707 h1:2tV76y6Q9BB+NEBasnqvs7e49aEBFI8ejC89PSnWH+4= github.com/dsnet/compress v0.0.2-0.20230904184137-39efe44ab707/go.mod h1:qssHWj60/X5sZFNxpG4HBPDHVqxNm4DfnCKgrbZOT+s= github.com/dsnet/golib v0.0.0-20171103203638-1ea166775780/go.mod h1:Lj+Z9rebOhdfkVLjJ8T6VcRQv3SXugXy999NBtR9aFY= -github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= -github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= +github.com/dustin/go-humanize v1.1.0 h1:dbKTrvD0klcbBV/h4AWJdMuZogJACoMlvWIWZ5b2xWg= +github.com/dustin/go-humanize v1.1.0/go.mod h1:hc1CvRkJMsgxqjmjMQF3QNRAZBwY8AXBAzKYoSX9sFI= github.com/elazarl/goproxy v1.7.2 h1:Y2o6urb7Eule09PjlhQRGNsqRfPmYI3KKQLFpCAV3+o= github.com/elazarl/goproxy v1.7.2/go.mod h1:82vkLNir0ALaW14Rc399OTTjyNREgmdL2cVoIbS6XaE= github.com/elliotchance/phpserialize v1.4.0 h1:cAp/9+KSnEbUC8oYCE32n2n84BeW8HOY3HMDI8hG2OY= From 9d39701d50b767e137eb47e685fd42d46db93d91 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 25 Sep 2026 13:29:27 +0000 Subject: [PATCH 09/40] chore(deps): bump github.com/magiconair/properties (#5333) Bumps [github.com/magiconair/properties](https://github.com/magiconair/properties) from 1.18.11 to 1.18.12. - [Release notes](https://github.com/magiconair/properties/releases) - [Commits](https://github.com/magiconair/properties/compare/v1.18.11...v1.18.12) --- updated-dependencies: - dependency-name: github.com/magiconair/properties dependency-version: 1.18.12 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index d6606dfda..e9c52ca98 100644 --- a/go.mod +++ b/go.mod @@ -63,7 +63,7 @@ require ( github.com/jinzhu/copier v0.4.0 github.com/kastenhq/goversion v0.0.0-20230811215019-93b2f8823953 github.com/klauspost/compress v1.20.0 - github.com/magiconair/properties v1.18.11 + github.com/magiconair/properties v1.18.12 github.com/mholt/archives v0.1.5 github.com/moby/sys/mountinfo v0.7.2 github.com/nix-community/go-nix v0.0.0-20250101154619-4bdde671e0a1 diff --git a/go.sum b/go.sum index 5c08de104..7be686d69 100644 --- a/go.sum +++ b/go.sum @@ -670,8 +670,8 @@ github.com/lucasb-eyer/go-colorful v1.3.0 h1:2/yBRLdWBZKrf7gB40FoiKfAWYQ0lqNcbuQ github.com/lucasb-eyer/go-colorful v1.3.0/go.mod h1:R4dSotOR9KMtayYi1e77YzuveK+i7ruzyGqttikkLy0= github.com/lyft/protoc-gen-star v0.5.3/go.mod h1:V0xaHgaf5oCCqmcxYcWiDfTiKsZsRc87/1qhoTACD8w= github.com/magiconair/properties v1.8.5/go.mod h1:y3VJvCyxH9uVvJTWEGAELF3aiYNyPKd5NZ3oSwXrF60= -github.com/magiconair/properties v1.18.11 h1:j5ozYZl0zCjG7ahMDH0GWIobOvvUzT0BdAguG0ViKy0= -github.com/magiconair/properties v1.18.11/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3vdS329zhj2hYo0= +github.com/magiconair/properties v1.18.12 h1:sT9zQpvTB3B4gzrX0tmZNTEaGyg8Zw55MFYRE32Mr9I= +github.com/magiconair/properties v1.18.12/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3vdS329zhj2hYo0= github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= github.com/maruel/natural v1.3.0 h1:VsmCsBmEyrR46RomtgHs5hbKADGRVtliHTyCOLFBpsg= github.com/maruel/natural v1.3.0/go.mod h1:v+Rfd79xlw1AgVBjbO0BEQmptqb5HvL/k9GRHB7ZKEg= From ec8f0355849c91b2d8671aea7d6fa973ab1128a8 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 25 Sep 2026 13:29:46 +0000 Subject: [PATCH 10/40] chore(deps): bump docker/setup-buildx-action from 4.3.0 to 4.4.1 (#5329) Bumps [docker/setup-buildx-action](https://github.com/docker/setup-buildx-action) from 4.3.0 to 4.4.1. - [Release notes](https://github.com/docker/setup-buildx-action/releases) - [Commits](https://github.com/docker/setup-buildx-action/compare/37fe631027851001ddb9b187196cc803df7f5f0e...f87e5991a6d7451dcb8d9637bfbc97413f497069) --- updated-dependencies: - dependency-name: docker/setup-buildx-action dependency-version: 4.4.1 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/release.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index edc571c35..72ec17297 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -86,7 +86,7 @@ jobs: # dockers_v2 builds multi-platform manifests in a single buildx invocation, which requires a # docker-container driver builder (the default builder on the runner cannot build multi-platform). - name: Set up Docker Buildx - uses: docker/setup-buildx-action@37fe631027851001ddb9b187196cc803df7f5f0e # v4.3.0 + uses: docker/setup-buildx-action@f87e5991a6d7451dcb8d9637bfbc97413f497069 # v4.4.1 - name: Login to Docker Hub uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f #v4.6.0 From 19341fd0a279b5cff0b125b4aa0404d097ef70e4 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 25 Sep 2026 13:29:56 +0000 Subject: [PATCH 11/40] chore(deps): bump github.com/olekukonko/tablewriter from 1.1.4 to 1.1.5 (#5331) Bumps [github.com/olekukonko/tablewriter](https://github.com/olekukonko/tablewriter) from 1.1.4 to 1.1.5. - [Release notes](https://github.com/olekukonko/tablewriter/releases) - [Commits](https://github.com/olekukonko/tablewriter/compare/v1.1.4...v1.1.5) --- updated-dependencies: - dependency-name: github.com/olekukonko/tablewriter dependency-version: 1.1.5 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index e9c52ca98..f1316cc03 100644 --- a/go.mod +++ b/go.mod @@ -67,7 +67,7 @@ require ( github.com/mholt/archives v0.1.5 github.com/moby/sys/mountinfo v0.7.2 github.com/nix-community/go-nix v0.0.0-20250101154619-4bdde671e0a1 - github.com/olekukonko/tablewriter v1.1.4 + github.com/olekukonko/tablewriter v1.1.5 github.com/opencontainers/go-digest v1.0.0 github.com/pelletier/go-toml v1.9.5 github.com/quasilyte/go-ruleguard/dsl v0.3.23 diff --git a/go.sum b/go.sum index 7be686d69..4882cfc2d 100644 --- a/go.sum +++ b/go.sum @@ -766,8 +766,8 @@ github.com/olekukonko/errors v1.2.0 h1:10Zcn4GeV59t/EGqJc8fUjtFT/FuUh5bTMzZ1XwmC github.com/olekukonko/errors v1.2.0/go.mod h1:ppzxA5jBKcO1vIpCXQ9ZqgDh8iwODz6OXIGKU8r5m4Y= github.com/olekukonko/ll v0.1.6 h1:lGVTHO+Qc4Qm+fce/2h2m5y9LvqaW+DCN7xW9hsU3uA= github.com/olekukonko/ll v0.1.6/go.mod h1:NVUmjBb/aCtUpjKk75BhWrOlARz3dqsM+OtszpY4o88= -github.com/olekukonko/tablewriter v1.1.4 h1:ORUMI3dXbMnRlRggJX3+q7OzQFDdvgbN9nVWj1drm6I= -github.com/olekukonko/tablewriter v1.1.4/go.mod h1:+kedxuyTtgoZLwif3P1Em4hARJs+mVnzKxmsCL/C5RY= +github.com/olekukonko/tablewriter v1.1.5 h1:4LoZSfMySpMQY3PT8RWJsJeuEuMIoo9xGRgvmqjg6IQ= +github.com/olekukonko/tablewriter v1.1.5/go.mod h1:+kedxuyTtgoZLwif3P1Em4hARJs+mVnzKxmsCL/C5RY= github.com/onsi/gomega v1.34.1 h1:EUMJIKUjM8sKjYbtxQI9A4z2o+rruxnzNvpknOXie6k= github.com/onsi/gomega v1.34.1/go.mod h1:kU1QgUvBDLXBJq618Xvm2LUX6rSAfRaFRTcdOeDLwwY= github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= From c5baac0e947b348cd8b63dd15bfdcb8544ca5c7b Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 25 Sep 2026 13:35:26 +0000 Subject: [PATCH 12/40] chore(deps): bump modernc.org/sqlite from 1.58.0 to 1.59.0 (#5330) Bumps [modernc.org/sqlite](https://gitlab.com/cznic/sqlite) from 1.58.0 to 1.59.0. - [Changelog](https://gitlab.com/cznic/sqlite/blob/master/CHANGELOG.md) - [Commits](https://gitlab.com/cznic/sqlite/compare/v1.58.0...v1.59.0) --- updated-dependencies: - dependency-name: modernc.org/sqlite dependency-version: 1.59.0 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 4 ++-- go.sum | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/go.mod b/go.mod index f1316cc03..961ade39d 100644 --- a/go.mod +++ b/go.mod @@ -98,7 +98,7 @@ require ( golang.org/x/time v0.16.0 golang.org/x/tools v0.50.0 gopkg.in/yaml.v3 v3.0.1 - modernc.org/sqlite v1.58.0 + modernc.org/sqlite v1.59.0 ) require ( @@ -322,7 +322,7 @@ require ( google.golang.org/grpc v1.83.2 // indirect google.golang.org/protobuf v1.36.12 // indirect gopkg.in/warnings.v0 v0.1.2 // indirect - modernc.org/libc v1.75.6 // indirect + modernc.org/libc v1.75.7 // indirect modernc.org/mathutil v1.7.1 // indirect modernc.org/memory v1.12.1 // indirect ) diff --git a/go.sum b/go.sum index 4882cfc2d..2bb9a7c2f 100644 --- a/go.sum +++ b/go.sum @@ -1528,8 +1528,8 @@ modernc.org/gc/v3 v3.1.5 h1:21ldfPfRYE31Tb7B3mwAK8gy1AxP4+dKjrOQPfqakoc= modernc.org/gc/v3 v3.1.5/go.mod h1:HFK/6AGESC7Ex+EZJhJ2Gni6cTaYpSMmU/cT9RmlfYY= modernc.org/goabi0 v0.2.0 h1:HvEowk7LxcPd0eq6mVOAEMai46V+i7Jrj13t4AzuNks= modernc.org/goabi0 v0.2.0/go.mod h1:CEFRnnJhKvWT1c1JTI3Avm+tgOWbkOu5oPA8eH8LnMI= -modernc.org/libc v1.75.6 h1:yKk8qo+Di4gkmvRboK8ocCqH22FiUCR6jRy2OwtCRus= -modernc.org/libc v1.75.6/go.mod h1:bO5o2ztHxBb2rjz0PgdHN0sSMw57CgxGFLZ3Qd/QpVQ= +modernc.org/libc v1.75.7 h1:o3DTP9/0p9pKmY2WCKQaySW6wIiZhNM7wc2lUoyhfew= +modernc.org/libc v1.75.7/go.mod h1:bO5o2ztHxBb2rjz0PgdHN0sSMw57CgxGFLZ3Qd/QpVQ= modernc.org/mathutil v1.7.1 h1:GCZVGXdaN8gTqB1Mf/usp1Y/hSqgI2vAGGP4jZMCxOU= modernc.org/mathutil v1.7.1/go.mod h1:4p5IwJITfppl0G4sUEDtCr4DthTaT47/N3aT6MhfgJg= modernc.org/memory v1.12.1 h1:nFMiWrpStgZczNl6XI9GnIk/rWhYIyHGUaR04pGbp9g= @@ -1538,8 +1538,8 @@ modernc.org/opt v0.2.0 h1:tGyef5ApycA7FSEOMraay9SaTk5zmbx7Tu+cJs4QKZg= modernc.org/opt v0.2.0/go.mod h1:03fq9lsNfvkYSfxrfUhZCWPk1lm4cq4N+Bh//bEtgns= modernc.org/sortutil v1.2.1 h1:+xyoGf15mM3NMlPDnFqrteY07klSFxLElE2PVuWIJ7w= modernc.org/sortutil v1.2.1/go.mod h1:7ZI3a3REbai7gzCLcotuw9AC4VZVpYMjDzETGsSMqJE= -modernc.org/sqlite v1.58.0 h1:38u40/bwkfM7f0Myhosl+SEMltSDxnGdQf8o6Kjmys0= -modernc.org/sqlite v1.58.0/go.mod h1:rsD2CckafgObKC4DhBlGBf+RiHxkc3hINGt1Xw32tVY= +modernc.org/sqlite v1.59.0 h1:X1es1GpqBlS/5T+vbM4HLUdaa8OtQx468DF2vrx+38A= +modernc.org/sqlite v1.59.0/go.mod h1:+paeT2A3iPRHkQDwG7oA6Tk0zQd5woMEI8q7orfry8k= modernc.org/strutil v1.2.1 h1:UneZBkQA+DX2Rp35KcM69cSsNES9ly8mQWD71HKlOA0= modernc.org/strutil v1.2.1/go.mod h1:EHkiggD70koQxjVdSBM3JKM7k6L0FbGE5eymy9i3B9A= modernc.org/token v1.1.0 h1:Xl7Ap9dKaEs5kLoOQeQmPWevfnk/DM5qcLcYlA8ys6Y= From 643bf2c24d5acf9e898e1f9d9db8bb39d3ab3528 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 25 Sep 2026 19:53:30 +0000 Subject: [PATCH 13/40] chore(deps): bump github.com/containerd/containerd/v2 (#5335) Bumps [github.com/containerd/containerd/v2](https://github.com/containerd/containerd) from 2.3.5 to 2.3.6. - [Release notes](https://github.com/containerd/containerd/releases) - [Changelog](https://github.com/containerd/containerd/blob/main/RELEASES.md) - [Commits](https://github.com/containerd/containerd/compare/v2.3.5...v2.3.6) --- updated-dependencies: - dependency-name: github.com/containerd/containerd/v2 dependency-version: 2.3.6 dependency-type: indirect ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 4 ++-- go.sum | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/go.mod b/go.mod index 961ade39d..eadba32c8 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module github.com/anchore/syft -go 1.26.3 +go 1.26.8 require ( github.com/BurntSushi/toml v1.6.0 @@ -171,7 +171,7 @@ require ( github.com/cncf/xds/go v0.0.0-20260202195803-dba9d589def2 // indirect github.com/containerd/cgroups/v3 v3.1.3 // indirect github.com/containerd/containerd/api v1.11.1 // indirect - github.com/containerd/containerd/v2 v2.3.5 // indirect + github.com/containerd/containerd/v2 v2.3.6 // indirect github.com/containerd/continuity v0.5.0 // indirect github.com/containerd/errdefs v1.0.0 // indirect github.com/containerd/errdefs/pkg v0.3.0 // indirect diff --git a/go.sum b/go.sum index 2bb9a7c2f..f8bbcca93 100644 --- a/go.sum +++ b/go.sum @@ -296,8 +296,8 @@ github.com/containerd/cgroups/v3 v3.1.3 h1:eUNflyMddm18+yrDmZPn3jI7C5hJ9ahABE5q6 github.com/containerd/cgroups/v3 v3.1.3/go.mod h1:PKZ2AcWmSBsY/tJUVhtS/rluX0b1uq1GmPO1ElCmbOw= github.com/containerd/containerd/api v1.11.1 h1:h8nfoDW9+fNsC/9TwiAHj8B1GzXKtR4eFtkhi/X5RLU= github.com/containerd/containerd/api v1.11.1/go.mod h1:CaQFRu+N1MtbgL6JDOJLUB1hCKESU1lD6MuTJhgtdlw= -github.com/containerd/containerd/v2 v2.3.5 h1:9MYlI81gUcOZ0WsCkSMtvOU7rTR3hqAoa2eCzhoLlkA= -github.com/containerd/containerd/v2 v2.3.5/go.mod h1:RXDyLPaI3zoO7dFdAW9/54W4cix+z3A6larieufC9mg= +github.com/containerd/containerd/v2 v2.3.6 h1:huoqZXaDW1x1kgPvFC4rMwxv92vUJnqgq7I5b1rSddM= +github.com/containerd/containerd/v2 v2.3.6/go.mod h1:qaUSWKk9EchqbudSbhv1MxI8dXKGYHLpZtMQdkEkVZ4= github.com/containerd/continuity v0.5.0 h1:7a85HZpCSs+1Zps0Ee3DPSuAWY+0SJM1JNM51nlEVDg= github.com/containerd/continuity v0.5.0/go.mod h1:/lNJvtJKUQStBzpVQ1+rasXO1LAWtUQssk28EZvJ3nE= github.com/containerd/errdefs v1.0.0 h1:tg5yIfIlQIrxYtu9ajqY42W3lpS19XqdxRQeEwYG8PI= From a76b5fa36b47527cd2ccfa898ec2c2bbf92c8006 Mon Sep 17 00:00:00 2001 From: anchore-oss-update-bot Date: Sun, 27 Sep 2026 06:26:41 -0400 Subject: [PATCH 14/40] chore(deps): update tool versions (#5337) Signed-off-by: anchore-oss-update-bot Co-authored-by: anchore-oss-update-bot --- .binny.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.binny.yaml b/.binny.yaml index 197d52ede..561c3a792 100644 --- a/.binny.yaml +++ b/.binny.yaml @@ -16,7 +16,7 @@ tools: # used to upload test fixture cache - name: oras version: - want: v1.3.3 + want: v1.3.4 method: github-release with: repo: oras-project/oras From fb55856e337fd23e1d4c14a9f7aef8dce8f7f501 Mon Sep 17 00:00:00 2001 From: anchore-oss-update-bot Date: Sun, 27 Sep 2026 21:50:42 -0400 Subject: [PATCH 15/40] chore(deps): update CPE dictionary index (#5312) Signed-off-by: anchore-oss-update-bot Co-authored-by: anchore-oss-update-bot --- .../dictionary/data/cpe-index.json | 146 +++++++++++++++++- 1 file changed, 141 insertions(+), 5 deletions(-) diff --git a/syft/pkg/cataloger/internal/cpegenerate/dictionary/data/cpe-index.json b/syft/pkg/cataloger/internal/cpegenerate/dictionary/data/cpe-index.json index 30ca6c481..baa74273d 100644 --- a/syft/pkg/cataloger/internal/cpegenerate/dictionary/data/cpe-index.json +++ b/syft/pkg/cataloger/internal/cpegenerate/dictionary/data/cpe-index.json @@ -4,6 +4,9 @@ "aahframe.work": [ "cpe:2.3:a:aahframework:aah:*:*:*:*:*:go:*:*" ], + "cel.dev/cel-go": [ + "cpe:2.3:a:google:common_expression_language:*:*:*:*:*:go:*:*" + ], "chainguard.dev/apko": [ "cpe:2.3:a:chainguard:apko:*:*:*:*:*:go:*:*" ], @@ -288,6 +291,9 @@ "golang.org/x/text": [ "cpe:2.3:a:golang:text:*:*:*:*:*:*:*:*" ], + "golang.org/x/tools/gopls": [ + "cpe:2.3:a:golang:gopls:*:*:*:*:*:go:*:*" + ], "gopkg.in/yaml.v3": [ "cpe:2.3:a:yaml_project:yaml:*:*:*:*:*:go:*:*" ], @@ -2338,6 +2344,9 @@ "626": [ "cpe:2.3:a:626_project:626:*:*:*:*:*:node.js:*:*" ], + "@a2ui/web_core": [ + "cpe:2.3:a:google:a2ui\\/web_core:*:*:*:*:*:node.js:*:*" + ], "@aborruso/ckan-mcp-server": [ "cpe:2.3:a:ondata:ckan_mcp_server:*:*:*:*:*:node.js:*:*" ], @@ -2524,7 +2533,8 @@ "cpe:2.3:a:matheuspergoli:draftlab\\/db:*:*:*:*:*:node.js:*:*" ], "@electron/packager": [ - "cpe:2.3:a:electron:packager:*:*:*:*:*:node.js:*:*" + "cpe:2.3:a:electron:packager:*:*:*:*:*:node.js:*:*", + "cpe:2.3:a:openjsf:packager:*:*:*:*:*:node.js:*:*" ], "@ensdomains/ens-contracts": [ "cpe:2.3:a:ens.domains:ethereum_name_service:*:*:*:*:*:node.js:*:*" @@ -2538,6 +2548,9 @@ "@fastify/aws-lambda": [ "cpe:2.3:a:fastify:fastify\\/aws-lambda:*:*:*:*:*:node.js:*:*" ], + "@fastify/busboy": [ + "cpe:2.3:a:fastify:fastify\\/busyboy:*:*:*:*:*:node.js:*:*" + ], "@fastify/express": [ "cpe:2.3:a:fastify:fastify\\/express:*:*:*:*:*:node.js:*:*" ], @@ -2547,6 +2560,9 @@ "@fastify/http-proxy": [ "cpe:2.3:a:fastify:fastify\\/http-proxy:*:*:*:*:*:node.js:*:*" ], + "@fastify/jwt": [ + "cpe:2.3:a:fastify:fastify\\/jwt:*:*:*:*:*:node.js:*:*" + ], "@fastify/middie": [ "cpe:2.3:a:fastify:fastify\\/middie:*:*:*:*:*:node.js:*:*" ], @@ -2650,6 +2666,9 @@ "@nestjs/core": [ "cpe:2.3:a:nestjs:nest:*:*:*:*:*:node.js:*:*" ], + "@nestjs/devtools-integration": [ + "cpe:2.3:a:nestjs:devtools-integration:*:*:*:*:*:node.js:*:*" + ], "@nextcloud/dialogs": [ "cpe:2.3:a:nextcloud\\/dialogs_project:nextcloud\\/dialogs:*:*:*:*:*:node.js:*:*" ], @@ -2747,6 +2766,9 @@ "@shopify/hydrogen": [ "cpe:2.3:a:shopify:hydrogen:*:*:*:*:*:node.js:*:*" ], + "@shtse8/filesystem-mcp": [ + "cpe:2.3:a:sylphx:filesystem_mcp:*:*:*:*:*:node.js:*:*" + ], "@simonsmith/cypress-image-snapshot": [ "cpe:2.3:a:simonsmith:cypress_image_snapshot:*:*:*:*:*:node.js:*:*" ], @@ -2847,6 +2869,9 @@ "adb-driver": [ "cpe:2.3:a:adb-driver_project:adb-driver:*:*:*:*:*:node.js:*:*" ], + "adb-mcp": [ + "cpe:2.3:a:srmorete:adb_mcp_server:*:*:*:*:*:node.js:*:*" + ], "adm-zip": [ "cpe:2.3:a:adm-zip_project:adm-zip:*:*:*:*:*:node.js:*:*" ], @@ -3080,6 +3105,9 @@ "bson-objectid": [ "cpe:2.3:a:bson-objectid_project:bson-objectid:*:*:*:*:*:node.js:*:*" ], + "builder-util-runtime": [ + "cpe:2.3:a:electron:builder-util-runtime:*:*:*:*:*:node.js:*:*" + ], "buns": [ "cpe:2.3:a:buns_project:buns:*:*:*:*:*:node.js:*:*" ], @@ -3279,6 +3307,9 @@ "cordova-plugin-file-transfer": [ "cpe:2.3:a:apache:cordova_file_transfer:*:*:*:*:*:android:*:*" ], + "cordova-plugin-x-socialsharing": [ + "cpe:2.3:a:eddyverbruggen:cordova-plugin-x-socialsharing:*:*:*:*:*:node.js:*:*" + ], "corenlp-js-interface": [ "cpe:2.3:a:corenlp-js-interface_project:corenlp-js-interface:*:*:*:*:*:node.js:*:*" ], @@ -3313,6 +3344,9 @@ "csv-parse": [ "cpe:2.3:a:csv-parse_project:csv-parse:*:*:*:*:*:node.js:*:*" ], + "csvjson": [ + "cpe:2.3:a:pradeep-mishra:csvjson:*:*:*:*:*:node.js:*:*" + ], "cuciuci": [ "cpe:2.3:a:cuciuci_project:cuciuci:*:*:*:*:*:node.js:*:*" ], @@ -4758,7 +4792,8 @@ "cpe:2.3:a:mongo-express_project:mongo-express:*:*:*:*:*:node.js:*:*" ], "mongodb-client-encryption": [ - "cpe:2.3:a:mongodb:libmongocrypt:*:*:*:*:*:node.js:*:*" + "cpe:2.3:a:mongodb:libmongocrypt:*:*:*:*:*:node.js:*:*", + "cpe:2.3:a:mongodb:mongodb_client_encryption:*:*:*:*:*:node.js:*:*" ], "mongodb-instance": [ "cpe:2.3:a:mongodb-instance_project:mongodb-instance:*:*:*:*:*:node.js:*:*" @@ -4793,6 +4828,9 @@ "mppx": [ "cpe:2.3:a:wevm:mppx:*:*:*:*:*:node.js:*:*" ], + "mpregular": [ + "cpe:2.3:a:regularjs:mpregular:*:*:*:*:*:node.js:*:*" + ], "mqtt-packet": [ "cpe:2.3:a:mqtt-packet_project:mqtt-packet:*:*:*:*:*:node.js:*:*" ], @@ -5459,6 +5497,9 @@ "quickserver": [ "cpe:2.3:a:quickserver_project:quickserver:*:*:*:*:*:node.js:*:*" ], + "quill": [ + "cpe:2.3:a:slab:quill:*:*:*:*:*:node.js:*:*" + ], "quill-mention": [ "cpe:2.3:a:quill-mention:quill_mention:*:*:*:*:*:node.js:*:*" ], @@ -6074,6 +6115,9 @@ "trailing-slash": [ "cpe:2.3:a:trailing-slash_project:trailing-slash:*:*:*:*:*:node.js:*:*" ], + "translate": [ + "cpe:2.3:a:francisco:translate:*:*:*:*:*:node.js:*:*" + ], "transpile": [ "cpe:2.3:a:transpile_project:transpile:*:*:*:*:*:node.js:*:*" ], @@ -6098,6 +6142,9 @@ "ts-process-promises": [ "cpe:2.3:a:ts-process-promises_project:ts-process-promises:*:*:*:*:*:node.js:*:*" ], + "ts-tree": [ + "cpe:2.3:a:mbovel:ts-tree:*:*:*:*:*:wordpress:*:*" + ], "tsup": [ "cpe:2.3:a:egoist:tsup:*:*:*:*:*:node.js:*:*" ], @@ -6557,6 +6604,9 @@ "apache-airflow-providers-cncf-kubernetes": [ "cpe:2.3:a:apache:apache-airflow-providers-cncf-kubernetes:*:*:*:*:*:*:*:*" ], + "apache-airflow-providers-fab": [ + "cpe:2.3:a:apache:apache-airflow-providers-fab:*:*:*:*:*:*:*:*" + ], "api-res-py": [ "cpe:2.3:a:api-res-py_project:api-res-py:*:*:*:*:*:python:*:*" ], @@ -6880,6 +6930,9 @@ "mltable": [ "cpe:2.3:a:microsoft:azure_machine_learning_software_development_kit:*:*:*:*:*:*:*:*" ], + "mopidy": [ + "cpe:2.3:a:mopidy:mopidy:*:*:*:*:*:python:*:*" + ], "mpxj": [ "cpe:2.3:a:mpxj:mpxj:*:*:*:*:*:python:*:*" ], @@ -6946,6 +6999,9 @@ "passeo": [ "cpe:2.3:a:passeo_project:passeo:*:*:*:*:*:python:*:*" ], + "pdf-oxide": [ + "cpe:2.3:a:oxide:pdf_oxide:*:*:*:*:*:python:*:*" + ], "pdm": [ "cpe:2.3:a:frostming:pdm:*:*:*:*:*:python:*:*" ], @@ -7042,6 +7098,9 @@ "pymetasploit3": [ "cpe:2.3:a:danmcinerney:pymetasploit3:*:*:*:*:*:*:*:*" ], + "pymongo": [ + "cpe:2.3:a:mongodb:python_driver:*:*:*:*:*:mongodb:*:*" + ], "pyo": [ "cpe:2.3:a:pyo_project:pyo:*:*:*:*:*:*:*:*" ], @@ -7198,6 +7257,9 @@ "tuf": [ "cpe:2.3:a:linuxfoundation:the_update_framework:*:*:*:*:*:python:*:*" ], + "tweepy": [ + "cpe:2.3:a:tweepy:tweepy:*:*:*:*:*:python:*:*" + ], "uamqp": [ "cpe:2.3:a:microsoft:azure_uamqp:*:*:*:*:*:python:*:*" ], @@ -9000,6 +9062,9 @@ "affiliate-ads-builder-for-clickbank-products": [ "cpe:2.3:a:clickbank:affiliate_ads_for_clickbank_products:*:*:*:*:*:wordpress:*:*" ], + "affiliate-links": [ + "cpe:2.3:a:wecantrack:affiliate_links:*:*:*:*:*:wordpress:*:*" + ], "affiliate-toolkit-starter": [ "cpe:2.3:a:servit:affiliate-toolkit:*:*:*:*:*:wordpress:*:*" ], @@ -9629,6 +9694,9 @@ "cpe:2.3:a:awesomesupport:awesome_support_wordpress_helpdesk_\\\u0026_support:*:*:*:*:*:wordpress:*:*", "cpe:2.3:a:getawesomesupport:awesome_support:*:*:*:*:*:wordpress:*:*" ], + "awesome-surveys": [ + "cpe:2.3:a:willbrubaker:awesome_surveys:*:*:*:*:*:wordpress:*:*" + ], "awesome-weather": [ "cpe:2.3:a:awesome_weather_widget_project:awesome_weather_widget:*:*:*:*:*:wordpress:*:*" ], @@ -9994,7 +10062,8 @@ "cpe:2.3:a:wordpress:blogger_importer:*:*:*:*:*:wordpress:*:*" ], "blogintroduction-wordpress-plugin": [ - "cpe:2.3:a:kimhuebel:blog_introduction:*:*:*:*:*:wordpress:*:*" + "cpe:2.3:a:kimhuebel:blog_introduction:*:*:*:*:*:wordpress:*:*", + "cpe:2.3:a:kimhuebel:blogintroduction-wordpress-plugin:*:*:*:*:*:wordpress:*:*" ], "blogmentor": [ "cpe:2.3:a:auburnforest:blogmentor:*:*:*:*:*:wordpress:*:*" @@ -10850,6 +10919,9 @@ "cpe:2.3:a:code_snippets:code_snippets:*:*:*:*:*:wordpress:*:*", "cpe:2.3:a:codesnippets:code_snippets:*:*:*:*:*:wordpress:*:*" ], + "code-snippets-cpt": [ + "cpe:2.3:a:jtsternberg:code_snippets_cpt:*:*:*:*:*:wordpress:*:*" + ], "code-snippets-extended": [ "cpe:2.3:a:code_snippets_extended_project:code_snippets_extended:*:*:*:*:*:wordpress:*:*", "cpe:2.3:a:codesnippets:code_snippets:*:*:*:*:*:wordpress:*:*" @@ -11993,7 +12065,7 @@ "cpe:2.3:a:download_zip_attachments_project:download_zip_attachments:*:*:*:*:*:wordpress:*:*" ], "drag-and-drop-multiple-file-upload-contact-form-7": [ - "cpe:2.3:a:codedropz:drag_and_drop_multiple_file_upload_-_contact_form_7:*:*:*:*:*:wordpress:*:*", + "cpe:2.3:a:codedropz:drag_and_drop_multiple_file_upload_-_contact_form_7:*:*:*:*:free:wordpress:*:*", "cpe:2.3:a:codedropz:drag_and_drop_multiple_file_upload_-_contact_form_7:*:*:*:*:pro:wordpress:*:*", "cpe:2.3:a:codedropz:drag_and_drop_multiple_file_upload_-_contact_form_7:*:*:*:*:standard:wordpress:*:*", "cpe:2.3:a:codedropz:drag_and_drop_multiple_file_upload_for_contact_form_7:*:*:*:*:free:wordpress:*:*" @@ -12007,6 +12079,9 @@ "drive-folder-embeder": [ "cpe:2.3:a:azumbro:drive_folder_embedder:*:*:*:*:*:wordpress:*:*" ], + "drm-protected-video-streaming": [ + "cpe:2.3:a:s3bubble:s3player:*:*:*:*:*:wordpress:*:*" + ], "droit-dark-mode": [ "cpe:2.3:a:droitthemes:droit_dark_mode:*:*:*:*:*:wordpress:*:*" ], @@ -12345,7 +12420,8 @@ "cpe:2.3:a:nicheaddons:education_addon:*:*:*:*:*:wordpress:*:*" ], "edwiser-bridge": [ - "cpe:2.3:a:edwiser:bridge:*:*:*:*:*:wordpress:*:*" + "cpe:2.3:a:edwiser:bridge:*:*:*:*:*:wordpress:*:*", + "cpe:2.3:a:wisdmlabs:edwiser_bridge:*:*:*:*:*:wordpress:*:*" ], "eelv-newsletter": [ "cpe:2.3:a:eelv_newsletter_project:eelv_newsletter:*:*:*:*:*:wordpress:*:*" @@ -13696,6 +13772,9 @@ "google-cse": [ "cpe:2.3:a:erikeng:google_cse:*:*:*:*:*:wordpress:*:*" ], + "google-distance-calculator": [ + "cpe:2.3:a:webtamarin:mk_google_directions:*:*:*:*:*:wordpress:*:*" + ], "google-document-embedder": [ "cpe:2.3:a:google_doc_embedder:google_doc_embedder:*:*:*:*:*:wordpress:*:*", "cpe:2.3:a:google_doc_embedder_project:google_doc_embedder:*:*:*:*:*:wordpress:*:*" @@ -13971,6 +14050,9 @@ "header-enhancement": [ "cpe:2.3:a:catchplugins:header_enhancement:*:*:*:*:*:wordpress:*:*" ], + "header-footer": [ + "cpe:2.3:a:satollo:head\\,_footer\\,_and_post_injections:*:*:*:*:*:wordpress:*:*" + ], "header-footer-code": [ "cpe:2.3:a:ninjateam:header_footer_custom_code:*:*:*:*:*:wordpress:*:*" ], @@ -15046,6 +15128,9 @@ "lh-password-changer": [ "cpe:2.3:a:shawfactor:lh-password-changer:*:*:*:*:*:wordpress:*:*" ], + "library-bookshelves": [ + "cpe:2.3:a:photonicgnostic:library_bookshelves:*:*:*:*:*:wordpress:*:*" + ], "library-viewer": [ "cpe:2.3:a:pexlechris:library_viewer:*:*:*:*:*:wordpress:*:*" ], @@ -15124,6 +15209,9 @@ "linker": [ "cpe:2.3:a:pojo:linker:*:*:*:*:*:wordpress:*:*" ], + "linkmyposts": [ + "cpe:2.3:a:gunnettmd:linkmyposts:*:*:*:*:*:wordpress:*:*" + ], "linkworth-wp-plugin": [ "cpe:2.3:a:linkworth:linkworth:*:*:*:*:*:wordpress:*:*" ], @@ -15501,6 +15589,9 @@ "market-exporter": [ "cpe:2.3:a:vanyukov:market_exporter:*:*:*:*:*:wordpress:*:*" ], + "marketing-and-seo-booster": [ + "cpe:2.3:a:secretlab:marketing_and_seo_booster:*:*:*:*:*:wordpress:*:*" + ], "marketo-forms-and-tracking": [ "cpe:2.3:a:hutchhouse:marketo_forms_and_tracking:*:*:*:*:*:wordpress:*:*" ], @@ -17028,6 +17119,9 @@ "points-and-rewards-for-woocommerce": [ "cpe:2.3:a:wpswings:points_and_rewards_for_woocommerce:*:*:*:*:*:wordpress:*:*" ], + "policy-genius": [ + "cpe:2.3:a:waelhassan:privacy_policy_genius:*:*:*:*:*:wordpress:*:*" + ], "poll-maker": [ "cpe:2.3:a:ays-pro:poll_maker:*:*:*:*:free:wordpress:*:*" ], @@ -17713,6 +17807,9 @@ "rays-grid": [ "cpe:2.3:a:rays_grid_project:rays_grid:*:*:*:*:*:wordpress:*:*" ], + "razorpay-payment-button": [ + "cpe:2.3:a:razorpay:razorpay_payment_button:*:*:*:*:*:wordpress:*:*" + ], "rb-internal-links": [ "cpe:2.3:a:rb_internal_links_project:rb_internal_links:*:*:*:*:*:wordpress:*:*" ], @@ -18215,6 +18312,9 @@ "schreikasten": [ "cpe:2.3:a:schreikasten_project:schreikasten:*:*:*:*:*:wordpress:*:*" ], + "scottcart": [ + "cpe:2.3:a:wpplugin:scottcart:*:*:*:*:*:wordpress:*:*" + ], "scoutnet-kalender": [ "cpe:2.3:a:scoutnet:kalender:*:*:*:*:*:wordpress:*:*" ], @@ -18337,6 +18437,9 @@ "sender": [ "cpe:2.3:a:bestwebsoft:sender:*:*:*:*:*:wordpress:*:*" ], + "sender-net-automated-emails": [ + "cpe:2.3:a:sender:sender:*:*:*:*:*:wordpress:*:*" + ], "sendgrid-email-delivery-simplified": [ "cpe:2.3:a:sendgrid:sendgrid:*:*:*:*:*:wordpress:*:*" ], @@ -18398,6 +18501,9 @@ "sermon-browser": [ "cpe:2.3:a:sermon_browser_project:sermon_browser:*:*:*:*:*:wordpress:*:*" ], + "sermonaudio-widgets": [ + "cpe:2.3:a:sermonaudio:sermonaudio_widgets:*:*:*:*:*:wordpress:*:*" + ], "sermone-online-sermons-management": [ "cpe:2.3:a:bearthemes:sermon\\'e_-_sermons_online:*:*:*:*:*:wordpress:*:*" ], @@ -18445,6 +18551,9 @@ "shared-files": [ "cpe:2.3:a:tammersoft:shared_files:*:*:*:*:*:wordpress:*:*" ], + "sharethis-share-buttons": [ + "cpe:2.3:a:sharethis:sharethis_share_buttons:*:*:*:*:*:wordpress:*:*" + ], "shariff": [ "cpe:2.3:a:datenverwurstungszentrale:shariff_wrapper:*:*:*:*:*:wordpress:*:*" ], @@ -18747,6 +18856,9 @@ "simple-mobile-url-redirect": [ "cpe:2.3:a:ozette:simple_mobile_url_redirect:*:*:*:*:*:wordpress:*:*" ], + "simple-nav-archives": [ + "cpe:2.3:a:philipwalton:simple_nav_archives:*:*:*:*:*:wordpress:*:*" + ], "simple-page-access-restriction": [ "cpe:2.3:a:pluginsandsnippets:simple_page_access_restriction:*:*:*:*:*:wordpress:*:*" ], @@ -19412,6 +19524,9 @@ "stars-rating": [ "cpe:2.3:a:stars_rating_project:stars_rating:*:*:*:*:*:wordpress:*:*" ], + "stars-smtp-mailer": [ + "cpe:2.3:a:myriadsolutionz:stars_smtp_mailer:*:*:*:*:*:wordpress:*:*" + ], "starter-templates": [ "cpe:2.3:a:fancywp:starter_templates:*:*:*:*:*:wordpress:*:*" ], @@ -19445,6 +19560,9 @@ "sticky-popup": [ "cpe:2.3:a:sticky_popup_project:sticky_popup:*:*:*:*:*:wordpress:*:*" ], + "sticky-social-icons": [ + "cpe:2.3:a:sanil:sticky_social_icons:*:*:*:*:*:wordpress:*:*" + ], "stklcode-liveticker": [ "cpe:2.3:a:stklcode:liveticker:*:*:*:*:*:wordpress:*:*" ], @@ -19703,6 +19821,9 @@ "tabs-responsive": [ "cpe:2.3:a:wpshopmart:tabs_responsive:*:*:*:*:*:wordpress:*:*" ], + "tabulate": [ + "cpe:2.3:a:samwilson:tabulate:*:*:*:*:*:wordpress:*:*" + ], "tagregator": [ "cpe:2.3:a:tagregator_project:tagregator:*:*:*:*:*:wordpress:*:*" ], @@ -21678,6 +21799,9 @@ "wp-cleanfix": [ "cpe:2.3:a:undolog:wp_cleanfix:*:*:*:*:*:wordpress:*:*" ], + "wp-click-info": [ + "cpe:2.3:a:s-a:wp_click_info:*:*:*:*:*:wordpress:*:*" + ], "wp-client-logo-carousel": [ "cpe:2.3:a:client_logo_carousel_project:client_logo_carousel:*:*:*:*:*:wordpress:*:*" ], @@ -22303,6 +22427,9 @@ "wp-media-library-categories": [ "cpe:2.3:a:jeffrey-wp:media_library_categories:*:*:*:*:*:wordpress:*:*" ], + "wp-mediatagger": [ + "cpe:2.3:a:phd38:wp_mediatagger:*:*:*:*:*:wordpress:*:*" + ], "wp-members": [ "cpe:2.3:a:butlerblog:wp-members:*:*:*:*:*:wordpress:*:*" ], @@ -23639,6 +23766,9 @@ "chic-lite": [ "cpe:2.3:a:rarathemes:chic:*:*:*:*:lite:wordpress:*:*" ], + "ciyashop": [ + "cpe:2.3:a:potenzaglobalsolutions:ciyashop:*:*:*:*:*:wordpress:*:*" + ], "clean-retina": [ "cpe:2.3:a:themehorse:clean_retina:*:*:*:*:free:wordpress:*:*" ], @@ -23687,6 +23817,9 @@ "drop": [ "cpe:2.3:a:competethemes:drop:*:*:*:*:*:wordpress:*:*" ], + "dwt-listing-2": [ + "cpe:2.3:a:scriptsbundle:dwt_listing:*:*:*:*:*:wordpress:*:*" + ], "education-zone": [ "cpe:2.3:a:rarathemes:education_zone:*:*:*:*:*:wordpress:*:*" ], @@ -23961,6 +24094,9 @@ "travel-agency": [ "cpe:2.3:a:rarathemes:travel_agency:*:*:*:*:*:wordpress:*:*" ], + "traveler": [ + "cpe:2.3:a:shinecommerce:traveler:*:*:*:*:*:wordpress:*:*" + ], "tweaker5": [ "cpe:2.3:a:arnoldgoodway:tweaker5:*:*:*:*:*:wordpress:*:*" ], From ff45b9ea6391b4df47f97ba4fac62e85ddd8fad0 Mon Sep 17 00:00:00 2001 From: Sverrir Sigurdsson <43657200+sverrirsig@users.noreply.github.com> Date: Mon, 28 Sep 2026 17:10:55 +0000 Subject: [PATCH 16/40] fix: release package collection lock in Ubuntu ESM detection (#5336) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Sverrir Sigurðsson --- internal/os/feature_detection.go | 10 ++++------ internal/os/feature_detection_test.go | 28 +++++++++++++++++++++++++++ 2 files changed, 32 insertions(+), 6 deletions(-) diff --git a/internal/os/feature_detection.go b/internal/os/feature_detection.go index 1aaf3f23c..a836a2191 100644 --- a/internal/os/feature_detection.go +++ b/internal/os/feature_detection.go @@ -6,6 +6,7 @@ import ( "errors" "fmt" "regexp" + "slices" "github.com/anchore/syft/internal" "github.com/anchore/syft/internal/log" @@ -191,12 +192,9 @@ func hasInstalledESMPackage(packages *pkg.Collection) bool { if packages == nil { return false } - for p := range packages.Enumerate(pkg.DebPkg) { - if esmVersionPattern.MatchString(p.Version) { - return true - } - } - return false + return slices.ContainsFunc(packages.Sorted(pkg.DebPkg), func(p pkg.Package) bool { + return esmVersionPattern.MatchString(p.Version) + }) } func fileMatchesAny(resolver file.Resolver, location file.Location, patterns ...*regexp.Regexp) (bool, error) { diff --git a/internal/os/feature_detection_test.go b/internal/os/feature_detection_test.go index bb8a30de1..b4f2ed38d 100644 --- a/internal/os/feature_detection_test.go +++ b/internal/os/feature_detection_test.go @@ -8,6 +8,7 @@ import ( "path/filepath" "slices" "testing" + "time" "github.com/stretchr/testify/require" @@ -232,6 +233,33 @@ func Test_findUbuntuFeatures(t *testing.T) { } } +// returning early from Collection.Enumerate leaks its goroutine and read lock, so any later write blocks forever. +func Test_findUbuntuFeatures_releasesPackageCollectionLock(t *testing.T) { + resolver := fixtureResolverForDir(t, "testdata/ubuntu_plain") + + s := sbom.SBOM{} + s.Artifacts.LinuxDistribution = &linux.Release{ID: "ubuntu"} + s.Artifacts.Packages = pkg.NewCollection( + pkg.Package{Name: "openssl", Version: "1.1.1f-1ubuntu2.19+esm1", Type: pkg.DebPkg}, + pkg.Package{Name: "libcap2", Version: "1:2.32-1ubuntu0.1~esm1", Type: pkg.DebPkg}, + ) + + require.NoError(t, os.DetectFeatures(context.Background(), resolver, sbomsync.NewBuilder(&s))) + require.True(t, s.Artifacts.LinuxDistribution.ExtendedSupport) + + added := make(chan struct{}) + go func() { + s.Artifacts.Packages.Add(pkg.Package{Name: "zlib1g", Version: "1:1.2.11.dfsg-2ubuntu1", Type: pkg.DebPkg}) + close(added) + }() + + select { + case <-added: + case <-time.After(5 * time.Second): + t.Fatal("package collection is still read-locked after feature detection") + } +} + // a read error in the apt-source signal must not suppress the ubuntu-advantage status signal; the three // signals are meant to be independent fallbacks. func Test_findUbuntuFeatures_signalErrorDoesNotSuppressOthers(t *testing.T) { From a8d4fb845f6c9165668f5601781cdacb7164292b Mon Sep 17 00:00:00 2001 From: "Antoni (Tony) Jagodka" Date: Mon, 28 Sep 2026 14:01:13 -0500 Subject: [PATCH 17/40] fix(spdx): don't panic on a root file without a supplier or null entries (#5326) The package supplier is optional in SPDX, but fileSource dereferenced it without the nil check containerSource already has, so a valid SPDX 2.3 document whose described package has primaryPackagePurpose FILE and no supplier crashed syft convert and grype sbom: input. A JSON null in files, a package's files or externalRefs (or an SPDX 2.1 package the version converter cannot map) decodes to a nil pointer that the conversion then dereferences. Drop nil entries once at the start of ToSyftModel instead of guarding each loop. Signed-off-by: Tony --- .../common/spdxhelpers/to_syft_model.go | 26 +++++++++++++- syft/format/spdxjson/decoder_test.go | 35 +++++++++++++++++++ .../spdx/file-root-without-supplier.spdx.json | 29 +++++++++++++++ 3 files changed, 89 insertions(+), 1 deletion(-) create mode 100644 syft/format/spdxjson/testdata/spdx/file-root-without-supplier.spdx.json diff --git a/syft/format/common/spdxhelpers/to_syft_model.go b/syft/format/common/spdxhelpers/to_syft_model.go index 59a40957d..e8b3e2bfb 100644 --- a/syft/format/common/spdxhelpers/to_syft_model.go +++ b/syft/format/common/spdxhelpers/to_syft_model.go @@ -34,6 +34,8 @@ func ToSyftModel(doc *spdx.Document) (*sbom.SBOM, error) { return nil, errors.New("cannot convert SPDX document to Syft model because document is nil") } + removeNilElements(doc) + spdxIDMap := make(map[string]any) s := &sbom.SBOM{ @@ -55,6 +57,28 @@ func ToSyftModel(doc *spdx.Document) (*sbom.SBOM, error) { return s, nil } +// removeNilElements drops null entries from the document's lists. A JSON null (or an element the SPDX version +// converter cannot map) decodes to a nil pointer, and the rest of the conversion dereferences these elements. +func removeNilElements(doc *spdx.Document) { + doc.Packages = withoutNil(doc.Packages) + for _, p := range doc.Packages { + p.Files = withoutNil(p.Files) + p.PackageExternalReferences = withoutNil(p.PackageExternalReferences) + } + doc.Files = withoutNil(doc.Files) + doc.Relationships = withoutNil(doc.Relationships) +} + +func withoutNil[T any](items []*T) []*T { + var out []*T + for _, item := range items { + if item != nil { + out = append(out, item) + } + } + return out +} + func isDirectory(name string) bool { if name == "." || name == ".." || strings.HasSuffix(name, "/") || !strings.Contains(path.Base(name), ".") { return true @@ -201,7 +225,7 @@ func fileSource(p *spdx.Package) source.Description { } supplier := "" - if p.PackageSupplier.Supplier != helpers.NOASSERTION { + if p.PackageSupplier != nil && p.PackageSupplier.Supplier != helpers.NOASSERTION { supplier = p.PackageSupplier.Supplier } diff --git a/syft/format/spdxjson/decoder_test.go b/syft/format/spdxjson/decoder_test.go index 9fb0d05dd..aca7fba6e 100644 --- a/syft/format/spdxjson/decoder_test.go +++ b/syft/format/spdxjson/decoder_test.go @@ -5,6 +5,7 @@ import ( "io" "os" "path/filepath" + "strings" "testing" "github.com/stretchr/testify/assert" @@ -183,3 +184,37 @@ func TestDecoder_Identify(t *testing.T) { }) } } + +func TestDecoder_DecodeNullElements(t *testing.T) { + // a JSON null in these lists (or an SPDX 2.1 package the version converter cannot map) decodes to a nil pointer. + // a null entry directly in "packages" is not covered: for SPDX 2.2 and 2.3 it panics inside spdx/tools-golang + for _, doc := range []string{ + `{"spdxVersion":"SPDX-2.1","packages":[{}]}`, + `{"spdxVersion":"SPDX-2.3","files":[null]}`, + `{"spdxVersion":"SPDX-2.3","packages":[{"SPDXID":"SPDXRef-a","name":"a","files":[null]}]}`, + `{"spdxVersion":"SPDX-2.3","packages":[{"SPDXID":"SPDXRef-a","name":"a","externalRefs":[null]}]}`, + } { + t.Run(doc, func(t *testing.T) { + assert.NotPanics(t, func() { + _, _, _, _ = NewFormatDecoder().Decode(strings.NewReader(doc)) + }) + }) + } +} + +func TestDecoder_DecodeFileRootWithoutSupplier(t *testing.T) { + // the package supplier is optional in SPDX, including on the package the document describes + f, err := os.Open("testdata/spdx/file-root-without-supplier.spdx.json") + require.NoError(t, err) + defer f.Close() + + s, _, _, err := NewFormatDecoder().Decode(f) + require.NoError(t, err) + assert.Equal(t, "app.tar.gz", s.Source.Name) + + var names []string + for _, p := range s.Artifacts.Packages.Sorted() { + names = append(names, p.Name) + } + assert.Equal(t, []string{"requests"}, names) +} diff --git a/syft/format/spdxjson/testdata/spdx/file-root-without-supplier.spdx.json b/syft/format/spdxjson/testdata/spdx/file-root-without-supplier.spdx.json new file mode 100644 index 000000000..25630796b --- /dev/null +++ b/syft/format/spdxjson/testdata/spdx/file-root-without-supplier.spdx.json @@ -0,0 +1,29 @@ +{ + "spdxVersion": "SPDX-2.3", + "dataLicense": "CC0-1.0", + "SPDXID": "SPDXRef-DOCUMENT", + "name": "app.tar.gz", + "documentNamespace": "https://example.com/spdx/app-1.0", + "creationInfo": { "created": "2026-09-24T00:00:00Z", "creators": ["Tool: example-1.0"] }, + "packages": [ + { + "SPDXID": "SPDXRef-app", + "name": "app.tar.gz", + "downloadLocation": "NOASSERTION", + "filesAnalyzed": false, + "primaryPackagePurpose": "FILE" + }, + { + "SPDXID": "SPDXRef-requests", + "name": "requests", + "versionInfo": "2.31.0", + "downloadLocation": "NOASSERTION", + "filesAnalyzed": false, + "externalRefs": [{ "referenceCategory": "PACKAGE-MANAGER", "referenceType": "purl", "referenceLocator": "pkg:pypi/requests@2.31.0" }] + } + ], + "relationships": [ + { "spdxElementId": "SPDXRef-DOCUMENT", "relationshipType": "DESCRIBES", "relatedSpdxElement": "SPDXRef-app" }, + { "spdxElementId": "SPDXRef-app", "relationshipType": "CONTAINS", "relatedSpdxElement": "SPDXRef-requests" } + ] +} From f7f3c6efc3f3ca4a62db667e0d0d8471fb75356e Mon Sep 17 00:00:00 2001 From: Ahmed Hassan Date: Tue, 29 Sep 2026 00:22:45 +0500 Subject: [PATCH 18/40] fix(ai): use go.yaml.in/yaml/v3 to parse model card frontmatter (#5328) * fix(ai): use go.yaml.in/yaml/v3 to parse model card frontmatter gopkg.in/yaml.v3 panics on some malformed merge-key payloads (for example "<<:\r? 0:"), which crashed the gguf and safetensors catalogers when a model README contained such frontmatter. Switch the Hugging Face frontmatter parser to the maintained go.yaml.in/yaml/v3 fork, which is already used by the other catalogers and returns an error instead of panicking. The existing error path logs and skips the frontmatter. Fixes #5327 Signed-off-by: tk1475 <118763009+tk1475@users.noreply.github.com> Co-Authored-By: Claude Opus 5.5 (1M context) * chore: finish moving off gopkg.in/yaml.v3 - move the snap catalogers and `internal/capabilities` to `go.yaml.in/yaml/v3`. The snap parsers decode yaml from scanned sources and hit the same merge-key panic as #5327 - enable `depguard` with a deny rule for `gopkg.in/yaml.v3` so it can't be reintroduced Signed-off-by: Alex Goodman --------- Signed-off-by: tk1475 <118763009+tk1475@users.noreply.github.com> Signed-off-by: Alex Goodman Co-authored-by: Claude Opus 5.5 (1M context) Co-authored-by: Alex Goodman --- .golangci.yaml | 9 ++++++++- go.mod | 1 - internal/capabilities/capabilities.go | 2 +- internal/capabilities/generate/io.go | 4 ++-- internal/capabilities/generate/io_test.go | 2 +- internal/capabilities/internal/load_capabilities.go | 2 +- internal/capabilities/model_test.go | 2 +- syft/pkg/cataloger/ai/huggingface.go | 2 +- syft/pkg/cataloger/ai/parse_safetensors_test.go | 12 ++++++++++++ syft/pkg/cataloger/snap/parse_base_dpkg.go | 2 +- syft/pkg/cataloger/snap/parse_snapd_snapcraft.go | 2 +- syft/pkg/cataloger/snap/parse_system_manifest.go | 2 +- 12 files changed, 30 insertions(+), 12 deletions(-) diff --git a/.golangci.yaml b/.golangci.yaml index 6b9bf74b0..5aa2a4905 100644 --- a/.golangci.yaml +++ b/.golangci.yaml @@ -7,6 +7,7 @@ linters: - asciicheck - bodyclose - copyloopvar + - depguard - dogsled - dupl - errcheck @@ -29,6 +30,13 @@ linters: - unused - whitespace settings: + depguard: + rules: + yaml: + deny: + # gopkg.in/yaml.v3 is unmaintained and panics on some malformed input (see #5327) + - pkg: gopkg.in/yaml.v3 + desc: use go.yaml.in/yaml/v3 instead funlen: lines: 70 statements: 50 @@ -71,7 +79,6 @@ linters: # do not enable... # - deadcode # The owner seems to have abandoned the linter. Replaced by "unused". -# - depguard # We don't have a configuration for this yet # - goprintffuncname # does not catch all cases and there are exceptions # - nakedret # does not catch all cases and should not fail a build # - gochecknoglobals diff --git a/go.mod b/go.mod index eadba32c8..b7573e132 100644 --- a/go.mod +++ b/go.mod @@ -97,7 +97,6 @@ require ( golang.org/x/net v0.59.0 golang.org/x/time v0.16.0 golang.org/x/tools v0.50.0 - gopkg.in/yaml.v3 v3.0.1 modernc.org/sqlite v1.59.0 ) diff --git a/internal/capabilities/capabilities.go b/internal/capabilities/capabilities.go index 492cdd2c4..2844f7136 100644 --- a/internal/capabilities/capabilities.go +++ b/internal/capabilities/capabilities.go @@ -9,7 +9,7 @@ import ( "sort" "github.com/scylladb/go-set/strset" - "gopkg.in/yaml.v3" + "go.yaml.in/yaml/v3" "github.com/anchore/syft/internal/task" ) diff --git a/internal/capabilities/generate/io.go b/internal/capabilities/generate/io.go index 3d660cf32..2eb839a6e 100644 --- a/internal/capabilities/generate/io.go +++ b/internal/capabilities/generate/io.go @@ -1,4 +1,4 @@ -// this file handles YAML file reading and writing with comment preservation, using gopkg.in/yaml.v3's node tree to maintain all existing comments during regeneration. +// this file handles YAML file reading and writing with comment preservation, using go.yaml.in/yaml/v3's node tree to maintain all existing comments during regeneration. package main import ( @@ -6,7 +6,7 @@ import ( "os" "regexp" - "gopkg.in/yaml.v3" + "go.yaml.in/yaml/v3" "github.com/anchore/syft/internal/capabilities" "github.com/anchore/syft/internal/capabilities/internal" diff --git a/internal/capabilities/generate/io_test.go b/internal/capabilities/generate/io_test.go index 0d6e790b4..65189c75f 100644 --- a/internal/capabilities/generate/io_test.go +++ b/internal/capabilities/generate/io_test.go @@ -4,7 +4,7 @@ import ( "testing" "github.com/stretchr/testify/require" - "gopkg.in/yaml.v3" + "go.yaml.in/yaml/v3" ) func TestFindFieldValue(t *testing.T) { diff --git a/internal/capabilities/internal/load_capabilities.go b/internal/capabilities/internal/load_capabilities.go index 43c12be27..b07230def 100644 --- a/internal/capabilities/internal/load_capabilities.go +++ b/internal/capabilities/internal/load_capabilities.go @@ -5,7 +5,7 @@ import ( "os" "path/filepath" - "gopkg.in/yaml.v3" + "go.yaml.in/yaml/v3" "github.com/anchore/syft/internal/capabilities" ) diff --git a/internal/capabilities/model_test.go b/internal/capabilities/model_test.go index 12ddb8994..f658e5c27 100644 --- a/internal/capabilities/model_test.go +++ b/internal/capabilities/model_test.go @@ -5,7 +5,7 @@ import ( "github.com/google/go-cmp/cmp" "github.com/stretchr/testify/require" - "gopkg.in/yaml.v3" + "go.yaml.in/yaml/v3" ) func TestConfigSchemaExtensions(t *testing.T) { diff --git a/syft/pkg/cataloger/ai/huggingface.go b/syft/pkg/cataloger/ai/huggingface.go index 6c333835b..dc8333e70 100644 --- a/syft/pkg/cataloger/ai/huggingface.go +++ b/syft/pkg/cataloger/ai/huggingface.go @@ -3,7 +3,7 @@ package ai import ( "bytes" - "gopkg.in/yaml.v3" + "go.yaml.in/yaml/v3" "github.com/anchore/syft/internal/log" "github.com/anchore/syft/syft/pkg" diff --git a/syft/pkg/cataloger/ai/parse_safetensors_test.go b/syft/pkg/cataloger/ai/parse_safetensors_test.go index c210bfbe0..943892c40 100644 --- a/syft/pkg/cataloger/ai/parse_safetensors_test.go +++ b/syft/pkg/cataloger/ai/parse_safetensors_test.go @@ -1066,6 +1066,12 @@ func TestParseFrontmatter(t *testing.T) { input: "---\nlicense: mit\n", wantNil: true, }, + { + // https://github.com/anchore/syft/issues/5327 + name: "malformed frontmatter that panics the yaml decoder", + input: "---\n<<:\r? 0:\n---\nbody", + wantNil: true, + }, } for _, tt := range tests { @@ -1157,6 +1163,12 @@ limitations: input: "---\nspdx-id: MIT\n(never closes)\n", want: "", }, + { + // https://github.com/anchore/syft/issues/5327 + name: "malformed frontmatter that panics the yaml decoder", + input: "---\n<<:\r? 0:\n---\nbody\n", + want: "", + }, } for _, tt := range tests { diff --git a/syft/pkg/cataloger/snap/parse_base_dpkg.go b/syft/pkg/cataloger/snap/parse_base_dpkg.go index b73f82555..72bb20482 100644 --- a/syft/pkg/cataloger/snap/parse_base_dpkg.go +++ b/syft/pkg/cataloger/snap/parse_base_dpkg.go @@ -5,7 +5,7 @@ import ( "fmt" "strings" - "gopkg.in/yaml.v3" + "go.yaml.in/yaml/v3" "github.com/anchore/syft/syft/artifact" "github.com/anchore/syft/syft/file" diff --git a/syft/pkg/cataloger/snap/parse_snapd_snapcraft.go b/syft/pkg/cataloger/snap/parse_snapd_snapcraft.go index 8a085f9ab..90dccead7 100644 --- a/syft/pkg/cataloger/snap/parse_snapd_snapcraft.go +++ b/syft/pkg/cataloger/snap/parse_snapd_snapcraft.go @@ -5,7 +5,7 @@ import ( "fmt" "strings" - "gopkg.in/yaml.v3" + "go.yaml.in/yaml/v3" "github.com/anchore/syft/syft/artifact" "github.com/anchore/syft/syft/file" diff --git a/syft/pkg/cataloger/snap/parse_system_manifest.go b/syft/pkg/cataloger/snap/parse_system_manifest.go index ee7159c83..8a84e3233 100644 --- a/syft/pkg/cataloger/snap/parse_system_manifest.go +++ b/syft/pkg/cataloger/snap/parse_system_manifest.go @@ -5,7 +5,7 @@ import ( "fmt" "strings" - "gopkg.in/yaml.v3" + "go.yaml.in/yaml/v3" "github.com/anchore/syft/syft/artifact" "github.com/anchore/syft/syft/file" From 10af8342c03816dd100a17df0f6d9de5ab4d0346 Mon Sep 17 00:00:00 2001 From: Alex Goodman Date: Tue, 29 Sep 2026 09:21:03 -0400 Subject: [PATCH 19/40] fix: don't fail the whole SBOM when a cataloger panics on one file (#5342) * fix(generic): recover from parser and processor panics - a panic while parsing one file becomes an unknown for that location instead of failing the whole SBOM - a panic in a package processor is logged as a warning and the unprocessed results are kept - stacks go to the trace log, not into the unknown text Signed-off-by: Alex Goodman * fix(unknown): keep errors that unwrap to nil `sync.PanicError` with a non-error panic value (e.g. `panic("boom")`) unwraps to nil, and was being dropped entirely from the error graph instead of being kept as a leaf. Signed-off-by: Alex Goodman * fix(task): don't put the stack trace in task panic errors Task panics are still fatal, but the error now reads `panic in task "": ` and the stack goes to the debug log. Signed-off-by: Alex Goodman * test(generic): cover chained processors when a later one panics Signed-off-by: Alex Goodman * fix(file): recover from panics in the executable and file digest catalogers A panic while reading one file is now recorded as an unknown for that file instead of failing the whole SBOM, the same as generic catalogers. Signed-off-by: Alex Goodman --------- Signed-off-by: Alex Goodman --- internal/task/executor.go | 3 +- internal/task/executor_test.go | 22 +++- internal/unknown/coordinate_error.go | 4 + internal/unknown/coordinate_error_test.go | 9 ++ syft/file/cataloger/executable/cataloger.go | 12 +- .../cataloger/executable/cataloger_test.go | 40 ++++++ syft/file/cataloger/filedigest/cataloger.go | 14 ++- .../cataloger/filedigest/cataloger_test.go | 25 ++++ syft/pkg/cataloger/generic/cataloger.go | 27 +++- syft/pkg/cataloger/generic/cataloger_test.go | 117 +++++++++++++++++- 10 files changed, 261 insertions(+), 12 deletions(-) create mode 100644 syft/file/cataloger/executable/cataloger_test.go diff --git a/internal/task/executor.go b/internal/task/executor.go index 6cee1242f..610f0a963 100644 --- a/internal/task/executor.go +++ b/internal/task/executor.go @@ -51,7 +51,8 @@ func runTaskSafely(ctx context.Context, t Task, resolver file.Resolver, s sbomsy // handle individual cataloger panics defer func() { if e := recover(); e != nil { - err = fmt.Errorf("%v at:\n%s", e, string(debug.Stack())) + log.WithFields("task", t.Name()).Debugf("task panic stack:\n%s", debug.Stack()) + err = fmt.Errorf("panic in task %q: %v", t.Name(), e) } }() diff --git a/internal/task/executor_test.go b/internal/task/executor_test.go index 527066cd0..af165b4be 100644 --- a/internal/task/executor_test.go +++ b/internal/task/executor_test.go @@ -8,8 +8,12 @@ import ( "github.com/wagoodman/go-progress" "github.com/anchore/syft/internal/sbomsync" + "github.com/anchore/syft/syft/artifact" "github.com/anchore/syft/syft/event/monitor" "github.com/anchore/syft/syft/file" + "github.com/anchore/syft/syft/pkg" + "github.com/anchore/syft/syft/pkg/cataloger/generic" + "github.com/anchore/syft/syft/sbom" ) func Test_TaskExecutor_PanicHandling(t *testing.T) { @@ -21,6 +25,20 @@ func Test_TaskExecutor_PanicHandling(t *testing.T) { Manual: progress.NewManual(-1), }) - require.Error(t, err) - require.Contains(t, err.Error(), "executor_test.go") + require.EqualError(t, err, `panic in task "panicking-cataloger": something bad happened`) +} + +func Test_RunTask_GenericParserPanicBecomesUnknown(t *testing.T) { + c := generic.NewCataloger("panicking-cataloger"). + WithParserByPath(func(context.Context, file.Resolver, *generic.Environment, file.LocationReadCloser) ([]pkg.Package, []artifact.Relationship, error) { + panic("boom") + }, "executor_test.go") + s := &sbom.SBOM{} + + err := RunTask(context.Background(), NewPackageTask(DefaultCatalogingFactoryConfig(), c), file.NewMockResolverForPaths("executor_test.go"), sbomsync.NewBuilder(s), &monitor.TaskProgress{ + Manual: progress.NewManual(-1), + }) + + require.NoError(t, err) + require.Len(t, s.Artifacts.Unknowns, 1) } diff --git a/internal/unknown/coordinate_error.go b/internal/unknown/coordinate_error.go index caa0a160e..2db6caf7e 100644 --- a/internal/unknown/coordinate_error.go +++ b/internal/unknown/coordinate_error.go @@ -183,6 +183,10 @@ func visitErrors(err error, fn func(error) error) error { // unwrap singly wrapped errors if e, ok := err.(interface{ Unwrap() error }); ok { wrapped := e.Unwrap() + if wrapped == nil { + // nothing is wrapped (e.g. a sync.PanicError with a non-error value), so this is a leaf + return fn(err) + } got := visitErrors(wrapped, fn) if got == nil { return nil diff --git a/internal/unknown/coordinate_error_test.go b/internal/unknown/coordinate_error_test.go index 27c53a5ff..e863e6359 100644 --- a/internal/unknown/coordinate_error_test.go +++ b/internal/unknown/coordinate_error_test.go @@ -8,6 +8,7 @@ import ( "github.com/stretchr/testify/require" + "github.com/anchore/go-sync" "github.com/anchore/syft/syft/file" ) @@ -234,3 +235,11 @@ func Test_Append(t *testing.T) { e3 = Appendf(e2, file.NewLocation("l3"), "%s", "e3") require.Equal(t, "l1: e1\nl3: e3", e3.Error()) } + +func Test_ExtractCoordinateErrors_keepsNilUnwrap(t *testing.T) { + // sync.PanicError unwraps to nil when the panic value is not an error + err := sync.PanicError{Value: "x"} + unknowns, remaining := ExtractCoordinateErrors(err) + require.Empty(t, unknowns) + require.Equal(t, err, remaining) +} diff --git a/syft/file/cataloger/executable/cataloger.go b/syft/file/cataloger/executable/cataloger.go index f6cdb5e65..552698f48 100644 --- a/syft/file/cataloger/executable/cataloger.go +++ b/syft/file/cataloger/executable/cataloger.go @@ -7,6 +7,7 @@ import ( "debug/macho" "encoding/binary" "fmt" + "runtime/debug" "sort" "github.com/bmatcuk/doublestar/v4" @@ -89,7 +90,7 @@ func (i *Cataloger) CatalogCtx(ctx context.Context, resolver file.Resolver) (map return results, errs } -func processExecutableLocation(loc file.Location, resolver file.Resolver) (*file.Executable, error) { +func processExecutableLocation(loc file.Location, resolver file.Resolver) (exec *file.Executable, err error) { reader, err := resolver.FileContentsByLocation(loc) if err != nil { log.WithFields("error", err, "path", loc.RealPath).Debug("unable to get file contents") @@ -97,6 +98,15 @@ func processExecutableLocation(loc file.Location, resolver file.Resolver) (*file } defer internal.CloseAndLogError(reader, loc.RealPath) + defer func() { + if r := recover(); r != nil { + // a malformed binary must not take down the whole SBOM; the caller records this as an unknown + log.WithFields("path", loc.RealPath, "panic", r).Debug("recovered from panic while reading executable") + log.Tracef("executable panic stack:\n%s", debug.Stack()) + exec, err = nil, fmt.Errorf("recovered from panic while reading executable: %v", r) + } + }() + uReader, err := unionreader.GetUnionReader(reader) if err != nil { log.WithFields("error", err, "path", loc.RealPath).Debug("unable to get union reader") diff --git a/syft/file/cataloger/executable/cataloger_test.go b/syft/file/cataloger/executable/cataloger_test.go new file mode 100644 index 000000000..7b670170c --- /dev/null +++ b/syft/file/cataloger/executable/cataloger_test.go @@ -0,0 +1,40 @@ +package executable + +import ( + "context" + "io" + "testing" + + "github.com/stretchr/testify/require" + + "github.com/anchore/syft/internal/unknown" + "github.com/anchore/syft/syft/file" +) + +type panicReader struct{} + +func (panicReader) Read([]byte) (int, error) { panic("boom") } + +type panickingContentsResolver struct { + *file.MockResolver +} + +func (r panickingContentsResolver) FilesByMIMEType(...string) ([]file.Location, error) { + return r.FilesByPath("cataloger_test.go") +} + +func (r panickingContentsResolver) FileContentsByLocation(file.Location) (io.ReadCloser, error) { + return io.NopCloser(panicReader{}), nil +} + +func TestCatalogCtx_panicBecomesUnknown(t *testing.T) { + resolver := panickingContentsResolver{file.NewMockResolverForPaths("cataloger_test.go")} + + results, err := NewCataloger(DefaultConfig()).CatalogCtx(context.Background(), resolver) + + require.Empty(t, results) + unknowns, remaining := unknown.ExtractCoordinateErrors(err) + require.NoError(t, remaining) + require.Len(t, unknowns, 1) + require.ErrorContains(t, unknowns[0].Reason, "recovered from panic while reading executable: boom") +} diff --git a/syft/file/cataloger/filedigest/cataloger.go b/syft/file/cataloger/filedigest/cataloger.go index 9ca6e7867..3340cc5cb 100644 --- a/syft/file/cataloger/filedigest/cataloger.go +++ b/syft/file/cataloger/filedigest/cataloger.go @@ -5,6 +5,7 @@ import ( "crypto" "errors" "fmt" + "runtime/debug" "github.com/dustin/go-humanize" @@ -87,7 +88,7 @@ func (i *Cataloger) Catalog(ctx context.Context, resolver file.Resolver, coordin return results, err } -func (i *Cataloger) catalogLocation(ctx context.Context, resolver file.Resolver, location file.Location) ([]file.Digest, error) { +func (i *Cataloger) catalogLocation(ctx context.Context, resolver file.Resolver, location file.Location) (digests []file.Digest, err error) { meta, err := resolver.FileMetadataByLocation(location) if err != nil { return nil, err @@ -104,7 +105,16 @@ func (i *Cataloger) catalogLocation(ctx context.Context, resolver file.Resolver, } defer internal.CloseAndLogError(contentReader, location.AccessPath) - digests, err := intFile.NewDigestsFromFile(ctx, contentReader, i.hashes) + defer func() { + if r := recover(); r != nil { + // the caller records this as an unknown for the location instead of failing the whole SBOM + log.WithFields("path", location.RealPath, "panic", r).Debug("recovered from panic while digesting file") + log.Tracef("file digest panic stack:\n%s", debug.Stack()) + digests, err = nil, fmt.Errorf("recovered from panic while digesting file: %v", r) + } + }() + + digests, err = intFile.NewDigestsFromFile(ctx, contentReader, i.hashes) if err != nil { return nil, internal.ErrPath{Context: "digests-cataloger", Path: location.RealPath, Err: err} } diff --git a/syft/file/cataloger/filedigest/cataloger_test.go b/syft/file/cataloger/filedigest/cataloger_test.go index 00e72a7d1..6188bd066 100644 --- a/syft/file/cataloger/filedigest/cataloger_test.go +++ b/syft/file/cataloger/filedigest/cataloger_test.go @@ -15,6 +15,7 @@ import ( stereoscopeFile "github.com/anchore/stereoscope/pkg/file" "github.com/anchore/stereoscope/pkg/imagetest" intFile "github.com/anchore/syft/internal/file" + "github.com/anchore/syft/internal/unknown" "github.com/anchore/syft/syft/file" "github.com/anchore/syft/syft/source" "github.com/anchore/syft/syft/source/directorysource" @@ -202,3 +203,27 @@ func TestFileDigestCataloger_GivenCoordinates(t *testing.T) { } } + +type panicReader struct{} + +func (panicReader) Read([]byte) (int, error) { panic("boom") } + +type panickingContentsResolver struct { + *file.MockResolver +} + +func (r panickingContentsResolver) FileContentsByLocation(file.Location) (io.ReadCloser, error) { + return io.NopCloser(panicReader{}), nil +} + +func TestDigestsCataloger_panicBecomesUnknown(t *testing.T) { + resolver := panickingContentsResolver{file.NewMockResolverForPaths("cataloger_test.go")} + + results, err := NewCataloger([]crypto.Hash{crypto.SHA256}).Catalog(context.Background(), resolver, file.NewLocation("cataloger_test.go").Coordinates) + + require.Empty(t, results) + unknowns, remaining := unknown.ExtractCoordinateErrors(err) + require.NoError(t, remaining) + require.Len(t, unknowns, 1) + require.ErrorContains(t, unknowns[0].Reason, "recovered from panic while digesting file: boom") +} diff --git a/syft/pkg/cataloger/generic/cataloger.go b/syft/pkg/cataloger/generic/cataloger.go index d357d0e07..ec4a5384a 100644 --- a/syft/pkg/cataloger/generic/cataloger.go +++ b/syft/pkg/cataloger/generic/cataloger.go @@ -2,6 +2,8 @@ package generic import ( "context" + "fmt" + "runtime/debug" "github.com/anchore/go-logger" "github.com/anchore/go-sync" @@ -221,12 +223,24 @@ func (c *Cataloger) Catalog(ctx context.Context, resolver file.Resolver) ([]pkg. func (c *Cataloger) process(ctx context.Context, resolver file.Resolver, pkgs []pkg.Package, rels []artifact.Relationship, err error) ([]pkg.Package, []artifact.Relationship, error) { for _, p := range c.processors { - pkgs, rels, err = p.process(ctx, resolver, pkgs, rels, err) + pkgs, rels, err = c.processSafely(ctx, p, resolver, pkgs, rels, err) } return pkgs, rels, err } -func invokeParser(ctx context.Context, resolver file.Resolver, location file.Location, logger logger.Logger, parser Parser, env *Environment) ([]pkg.Package, []artifact.Relationship, error) { +func (c *Cataloger) processSafely(ctx context.Context, p processExecutor, resolver file.Resolver, pkgs []pkg.Package, rels []artifact.Relationship, err error) (outPkgs []pkg.Package, outRels []artifact.Relationship, outErr error) { + defer func() { + if r := recover(); r != nil { + // keep the unprocessed results; there is no single location to attach an unknown to + log.WithFields("cataloger", c.upstreamCataloger, "panic", r).Warn("recovered from panic in package processor") + log.Tracef("processor panic stack:\n%s", debug.Stack()) + outPkgs, outRels, outErr = pkgs, rels, err + } + }() + return p.process(ctx, resolver, pkgs, rels, err) +} + +func invokeParser(ctx context.Context, resolver file.Resolver, location file.Location, logger logger.Logger, parser Parser, env *Environment) (pkgs []pkg.Package, rels []artifact.Relationship, err error) { contentReader, err := resolver.FileContentsByLocation(location) if err != nil { logger.WithFields("location", location.RealPath, "error", err).Debug("unable to fetch contents") @@ -234,6 +248,15 @@ func invokeParser(ctx context.Context, resolver file.Resolver, location file.Loc } defer internal.CloseAndLogError(contentReader, location.AccessPath) + defer func() { + if r := recover(); r != nil { + // a malformed file must not take down the whole SBOM; the caller records this as an unknown + logger.WithFields("location", location.RealPath, "panic", r).Debug("recovered from panic while parsing file") + logger.Tracef("parser panic stack:\n%s", debug.Stack()) + pkgs, rels, err = nil, nil, fmt.Errorf("recovered from panic while parsing file: %v", r) + } + }() + discoveredPackages, discoveredRelationships, err := parser(ctx, resolver, env, file.NewLocationReadCloser(location, contentReader)) if err != nil { // these errors are propagated up, and are likely to be coordinate errors diff --git a/syft/pkg/cataloger/generic/cataloger_test.go b/syft/pkg/cataloger/generic/cataloger_test.go index 2209a3fc9..93f7813d0 100644 --- a/syft/pkg/cataloger/generic/cataloger_test.go +++ b/syft/pkg/cataloger/generic/cataloger_test.go @@ -10,7 +10,6 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - "github.com/anchore/go-sync" "github.com/anchore/syft/internal/unknown" "github.com/anchore/syft/syft/artifact" "github.com/anchore/syft/syft/file" @@ -190,12 +189,122 @@ func TestClosesFileOnParserPanic(t *testing.T) { _, _, err := c.Catalog(ctx, resolver) require.Error(t, err) - var panicErr sync.PanicError - require.ErrorAs(t, err, &panicErr) - assert.Equal(t, "panic!", panicErr.Value) + var coordErr *unknown.CoordinateError + require.ErrorAs(t, err, &coordErr) + assert.ErrorContains(t, err, "recovered from panic while parsing file: panic!") require.True(t, spy.closed) } +func TestParserPanicBecomesUnknown(t *testing.T) { + tests := []struct { + name string + doPanic func() + wantErr string + }{ + { + name: "string panic", + doPanic: func() { panic("boom") }, + wantErr: "boom", + }, + { + name: "runtime panic", + doPanic: func() { + var m map[string]int + m["x"] = 1 + }, + wantErr: "assignment to entry in nil map", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + parser := func(_ context.Context, _ file.Resolver, _ *Environment, reader file.LocationReadCloser) ([]pkg.Package, []artifact.Relationship, error) { + if reader.Path() == "testdata/another-path.txt" { + tt.doPanic() + } + return []pkg.Package{{Name: "good", Locations: file.NewLocationSet(reader.Location)}}, nil, nil + } + + resolver := file.NewMockResolverForPaths("testdata/a-path.txt", "testdata/another-path.txt") + c := NewCataloger("unit-test-cataloger"). + WithParserByPath(parser, "testdata/a-path.txt", "testdata/another-path.txt") + + pkgs, _, err := c.Catalog(context.Background(), resolver) + + require.Len(t, pkgs, 1) + assert.Equal(t, "good", pkgs[0].Name) + + unknowns, remaining := unknown.ExtractCoordinateErrors(err) + require.NoError(t, remaining) + require.Len(t, unknowns, 1) + assert.Equal(t, "testdata/another-path.txt", unknowns[0].Coordinates.RealPath) + assert.ErrorContains(t, unknowns[0].Reason, tt.wantErr) + }) + } +} + +func TestProcessorPanicKeepsResults(t *testing.T) { + parser := func(_ context.Context, _ file.Resolver, _ *Environment, reader file.LocationReadCloser) ([]pkg.Package, []artifact.Relationship, error) { + return []pkg.Package{{Name: "good", Locations: file.NewLocationSet(reader.Location)}}, nil, nil + } + + rename := func(pkgs []pkg.Package, rels []artifact.Relationship, err error) ([]pkg.Package, []artifact.Relationship, error) { + for i := range pkgs { + pkgs[i].Name = "processed" + } + return pkgs, rels, err + } + + tests := []struct { + name string + cataloger *Cataloger + want string + }{ + { + name: "processor", + cataloger: NewCataloger("unit-test-cataloger"). + WithParserByPath(parser, "testdata/a-path.txt"). + WithProcessors(func([]pkg.Package, []artifact.Relationship, error) ([]pkg.Package, []artifact.Relationship, error) { + panic("boom") + }), + want: "good", + }, + { + // output from earlier processors survives a later processor panicking + name: "chained processors", + cataloger: NewCataloger("unit-test-cataloger"). + WithParserByPath(parser, "testdata/a-path.txt"). + WithProcessors(rename, func([]pkg.Package, []artifact.Relationship, error) ([]pkg.Package, []artifact.Relationship, error) { + panic("boom") + }), + want: "processed", + }, + { + name: "resolving processor", + cataloger: NewCataloger("unit-test-cataloger"). + WithParserByPath(parser, "testdata/a-path.txt"). + WithResolvingProcessors(func(context.Context, file.Resolver, []pkg.Package, []artifact.Relationship, error) ([]pkg.Package, []artifact.Relationship, error) { + var m map[string]int + m["x"] = 1 + return nil, nil, nil + }), + want: "good", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + resolver := file.NewMockResolverForPaths("testdata/a-path.txt") + + pkgs, _, err := tt.cataloger.Catalog(context.Background(), resolver) + + require.NoError(t, err) + require.Len(t, pkgs, 1) + assert.Equal(t, tt.want, pkgs[0].Name) + }) + } +} + func Test_CatalogerWithParserByMediaType(t *testing.T) { allParsedPaths := make(map[string]bool) parser := func(_ context.Context, resolver file.Resolver, env *Environment, reader file.LocationReadCloser) ([]pkg.Package, []artifact.Relationship, error) { From 50dbb33e662a21e14ffb954ec46350500b43d546 Mon Sep 17 00:00:00 2001 From: Alex Goodman Date: Tue, 29 Sep 2026 09:33:57 -0400 Subject: [PATCH 20/40] fix(ai): bound model companion file reads and validate their contents (#5344) Companion files next to AI models (`config.json`, `README.md`, and the OCI model.file, license and model config layers) are now read with named size limits, and are only used when they look like what they claim to be. - JSON companions are read whole up to a limit and skipped when larger, rather than being fully buffered or parsed truncated. OCI companion layers are fetched one byte past the cap so oversize is detectable. - README and license layers read a prefix. The frontmatter block is capped at 256 KB and list values at 32. - dir and OCI scans share one 8 MB safetensors header cap. Directory scans used to allow 100 MB. - `config.json` is only used when it has `architectures` or `_name_or_path`, and is looked up beside the model or in its parent directory, no longer all the way up to the scan root - OCI model.file layers are processed in a stable order, so the chosen name no longer changes between runs - README frontmatter: - a list-form `license:` is accepted and no longer drops `base_model` - `null` values are ignored - delimiters must be exact `---` lines - model names from `_name_or_path` / `base_model`: - Windows paths are handled - `.`, `..` and `/` fall back to the next name source - names are capped at 256 characters Behavior changes worth noting: - a `config.json` two or more directories above the model no longer names it - JSON `config.json` files without `architectures` or `_name_or_path` (e.g. `generation_config.json`, app configs) are ignored - safetensors headers over 8 MB in directory scans are now rejected, matching OCI scans (real shard headers are well under 1 MB) Signed-off-by: Alex Goodman --- internal/safetensors.go | 7 + syft/pkg/cataloger/ai/huggingface.go | 78 ++++-- syft/pkg/cataloger/ai/identity_dir.go | 48 ++-- syft/pkg/cataloger/ai/identity_oci.go | 52 ++-- syft/pkg/cataloger/ai/limits.go | 46 ++++ syft/pkg/cataloger/ai/naming.go | 35 ++- syft/pkg/cataloger/ai/parse_safetensors.go | 4 +- .../pkg/cataloger/ai/parse_safetensors_oci.go | 2 +- .../cataloger/ai/parse_safetensors_test.go | 239 +++++++++++++++++- syft/pkg/cataloger/ai/processor.go | 4 +- .../ai/testdata/safetensors/extract_header.go | 6 +- .../source/ocimodelsource/oci_model_source.go | 10 +- syft/source/ocimodelsource/registry_client.go | 25 +- 13 files changed, 456 insertions(+), 100 deletions(-) create mode 100644 internal/safetensors.go create mode 100644 syft/pkg/cataloger/ai/limits.go diff --git a/internal/safetensors.go b/internal/safetensors.go new file mode 100644 index 000000000..deae046f5 --- /dev/null +++ b/internal/safetensors.go @@ -0,0 +1,7 @@ +package internal + +// MaxSafeTensorsHeaderSize caps a safetensors header's JSON (excluding its +// 8-byte length prefix). Shared by the ai cataloger and the OCI model source so +// dir and OCI scans accept the same headers. Real-world shard headers are well +// under 1 MB. +const MaxSafeTensorsHeaderSize = 8 * 1024 * 1024 diff --git a/syft/pkg/cataloger/ai/huggingface.go b/syft/pkg/cataloger/ai/huggingface.go index dc8333e70..9b9d644b1 100644 --- a/syft/pkg/cataloger/ai/huggingface.go +++ b/syft/pkg/cataloger/ai/huggingface.go @@ -15,6 +15,12 @@ type hfConfig struct { NameOrPath string `json:"_name_or_path"` } +// looksLikeHF reports whether the config carries any field we use. Other JSON +// (generation_config.json, unrelated app configs) is ignored. +func (c hfConfig) looksLikeHF() bool { + return len(c.Architectures) > 0 || c.NameOrPath != "" +} + func applyHFConfig(md *pkg.SafeTensorsModelInfo, cfg *hfConfig) { if md.Architecture == "" && len(cfg.Architectures) > 0 { md.Architecture = cfg.Architectures[0] @@ -23,8 +29,8 @@ func applyHFConfig(md *pkg.SafeTensorsModelInfo, cfg *hfConfig) { // readmeFrontmatter holds the subset of YAML frontmatter fields we extract. type readmeFrontmatter struct { - License string `yaml:"license"` - BaseModel []string `yaml:"base_model"` + Licenses []string + BaseModel []string } type licenseFrontmatter struct { @@ -32,21 +38,33 @@ type licenseFrontmatter struct { } // extractFrontmatterBlock returns the YAML bytes between the first and second -// "---" delimiters of a file +// "---" delimiter lines of a file func extractFrontmatterBlock(buf []byte) []byte { trimmed := bytes.TrimLeft(buf, "\xef\xbb\xbf \t\r\n") - if !bytes.HasPrefix(trimmed, []byte("---")) { + first, rest, _ := bytes.Cut(trimmed, []byte("\n")) + if !isFrontmatterDelimiter(first) { return nil } - rest := trimmed[3:] - if i := bytes.IndexByte(rest, '\n'); i >= 0 { - rest = rest[i+1:] + for off := 0; off < len(rest); { + line, _, found := bytes.Cut(rest[off:], []byte("\n")) + if isFrontmatterDelimiter(line) { + if off == 0 || off > maxFrontmatterSize { + return nil + } + return rest[:off] + } + if !found { + break + } + off += len(line) + 1 } - block, _, found := bytes.Cut(rest, []byte("\n---")) - if !found { - return nil - } - return block + return nil +} + +// isFrontmatterDelimiter reports whether a line is exactly "---", ignoring +// trailing whitespace and \r. +func isFrontmatterDelimiter(line []byte) bool { + return bytes.Equal(bytes.TrimRight(line, " \t\r"), []byte("---")) } // parseFrontmatter decodes a Hugging Face model card YAML frontmatter block @@ -57,8 +75,9 @@ func parseFrontmatter(buf []byte) *readmeFrontmatter { return nil } + // both fields may be a scalar or a list, so decode them as nodes var raw struct { - License string `yaml:"license"` + License yaml.Node `yaml:"license"` BaseModel yaml.Node `yaml:"base_model"` } if err := yaml.Unmarshal(block, &raw); err != nil { @@ -66,16 +85,37 @@ func parseFrontmatter(buf []byte) *readmeFrontmatter { return nil } - fm := readmeFrontmatter{License: raw.License} - switch raw.BaseModel.Kind { + return &readmeFrontmatter{ + Licenses: yamlStrings(raw.License), + BaseModel: yamlStrings(raw.BaseModel), + } +} + +// yamlStrings returns the non-empty, non-null values of a scalar or a sequence +// of scalars, keeping at most maxFrontmatterValues. Anything else yields nil. +func yamlStrings(n yaml.Node) []string { + switch n.Kind { case yaml.ScalarNode: - if raw.BaseModel.Value != "" { - fm.BaseModel = []string{raw.BaseModel.Value} + if isYAMLString(&n) { + return []string{n.Value} } case yaml.SequenceNode: - _ = raw.BaseModel.Decode(&fm.BaseModel) + var out []string + for _, c := range n.Content { + if len(out) == maxFrontmatterValues { + break + } + if c.Kind == yaml.ScalarNode && isYAMLString(c) { + out = append(out, c.Value) + } + } + return out } - return &fm + return nil +} + +func isYAMLString(n *yaml.Node) bool { + return n.Value != "" && n.ShortTag() != "!!null" } // parseLicenseFrontmatter returns the producer-declared SPDX identifier diff --git a/syft/pkg/cataloger/ai/identity_dir.go b/syft/pkg/cataloger/ai/identity_dir.go index eaa0d1efa..dd2460092 100644 --- a/syft/pkg/cataloger/ai/identity_dir.go +++ b/syft/pkg/cataloger/ai/identity_dir.go @@ -3,7 +3,6 @@ package ai import ( "context" "encoding/json" - "io" "path" "github.com/anchore/syft/internal" @@ -13,42 +12,42 @@ import ( ) // resolveSafeTensorsDirIdentity handles the directory-scan case for safe tensors -// find config.json beside the model files (walking up parent directories to the -// scanned source root if no sibling exists) and a sibling README.md. It returns +// find config.json beside the model files (or one directory up, for the shard +// subdirectory layout) and a sibling README.md. It returns // the group's name candidates, resolved licenses, and supporting evidence. func resolveSafeTensorsDirIdentity(ctx context.Context, resolver file.Resolver, dir string, md *pkg.SafeTensorsModelInfo) safeTensorsIdentity { id := safeTensorsIdentity{fallbackName: safeTensorsDirName(dir)} if loc, cfg := findDirHFConfig(resolver, dir); cfg != nil { applyHFConfig(md, cfg) - id.nameOrPath = cfg.NameOrPath + id.nameOrPath = sanitizeModelName(cfg.NameOrPath) id.supporting = append(id.supporting, *loc) } if loc, fm := readDirReadmeFrontmatter(resolver, path.Join(dir, "README.md")); fm != nil { - if fm.License != "" { - id.licenses = pkg.NewLicensesFromValuesWithContext(ctx, fm.License) + if len(fm.Licenses) > 0 { + id.licenses = pkg.NewLicensesFromValuesWithContext(ctx, fm.Licenses...) } - if id.nameOrPath == "" && len(fm.BaseModel) > 0 { - id.nameOrPath = fm.BaseModel[0] + if id.nameOrPath == "" { + id.nameOrPath = firstModelName(fm.BaseModel) } id.supporting = append(id.supporting, *loc) } return id } -// findDirHFConfig looks for a config.json beside the model files +// findDirHFConfig looks for an HF config.json beside the model files, then in +// the parent directory (covers weights kept in a subdirectory of the model). It +// does not walk further up, so an unrelated config.json higher in the tree can't +// name every model below it. func findDirHFConfig(resolver file.Resolver, dir string) (*file.Location, *hfConfig) { - for { - if loc, cfg := readDirHFConfig(resolver, path.Join(dir, "config.json")); cfg != nil { - return loc, cfg - } - parent := path.Dir(dir) - if parent == dir { - return nil, nil // reached the source root - } - dir = parent + if loc, cfg := readDirHFConfig(resolver, path.Join(dir, "config.json")); cfg != nil { + return loc, cfg } + if parent := path.Dir(dir); parent != dir { + return readDirHFConfig(resolver, path.Join(parent, "config.json")) + } + return nil, nil } func readDirHFConfig(resolver file.Resolver, p string) (*file.Location, *hfConfig) { @@ -62,11 +61,19 @@ func readDirHFConfig(resolver file.Resolver, p string) (*file.Location, *hfConfi } defer internal.CloseAndLogError(rc, p) + buf, err := readBounded(rc, maxHFConfigSize) + if err != nil { + log.Debugf("failed to read %s: %v", p, err) + return nil, nil + } var cfg hfConfig - if err := json.NewDecoder(rc).Decode(&cfg); err != nil { + if err := json.Unmarshal(buf, &cfg); err != nil { log.Debugf("failed to decode %s: %v", p, err) return nil, nil } + if !cfg.looksLikeHF() { + return nil, nil + } return &locations[0], &cfg } @@ -81,8 +88,9 @@ func readDirReadmeFrontmatter(resolver file.Resolver, p string) (*file.Location, } defer internal.CloseAndLogError(rc, p) - buf, err := io.ReadAll(io.LimitReader(rc, 1024*1024)) + buf, err := readPrefix(rc, maxReadmePrefixSize) if err != nil { + log.Debugf("failed to read %s: %v", p, err) return nil, nil } fm := parseFrontmatter(buf) diff --git a/syft/pkg/cataloger/ai/identity_oci.go b/syft/pkg/cataloger/ai/identity_oci.go index 6209a3f8b..97d58aa6b 100644 --- a/syft/pkg/cataloger/ai/identity_oci.go +++ b/syft/pkg/cataloger/ai/identity_oci.go @@ -4,8 +4,8 @@ import ( "bytes" "context" "encoding/json" - "io" "path" + "sort" gcrname "github.com/google/go-containerregistry/pkg/name" @@ -28,9 +28,18 @@ func resolveSafeTensorsOCIIdentity(ctx context.Context, resolver file.Resolver, log.Debugf("failed to list docker AI model-file layers: %v", err) } - // Collect config / readme candidates separately so the layer-iteration order - // returned by the resolver doesn't decide the precedence. - var configName, readmeName, readmeLicense string + // the resolver returns layers in map order, and the first config / README + // seen wins below, so sort for a stable result across runs. For OCI + // artifacts every RealPath is "/", so in practice this is digest order. + sort.Slice(modelFileLocs, func(i, j int) bool { + if modelFileLocs[i].RealPath != modelFileLocs[j].RealPath { + return modelFileLocs[i].RealPath < modelFileLocs[j].RealPath + } + return modelFileLocs[i].FileSystemID < modelFileLocs[j].FileSystemID + }) + + var configName, readmeName string + var readmeLicenses []string var supporting []file.Location for _, loc := range modelFileLocs { cfg, fm := classifyOCIModelFileLayer(resolver, loc) @@ -38,15 +47,15 @@ func resolveSafeTensorsOCIIdentity(ctx context.Context, resolver file.Resolver, case cfg != nil: applyHFConfig(md, cfg) if configName == "" { - configName = cfg.NameOrPath + configName = sanitizeModelName(cfg.NameOrPath) } supporting = append(supporting, loc) case fm != nil: - if readmeLicense == "" { - readmeLicense = fm.License + if len(readmeLicenses) == 0 { + readmeLicenses = fm.Licenses } - if readmeName == "" && len(fm.BaseModel) > 0 { - readmeName = fm.BaseModel[0] + if readmeName == "" { + readmeName = firstModelName(fm.BaseModel) } supporting = append(supporting, loc) } @@ -64,7 +73,7 @@ func resolveSafeTensorsOCIIdentity(ctx context.Context, resolver file.Resolver, supporting: supporting, } - // License precedence: a dedicated vnd.docker.ai.license layer is a + // License precedence: a dedicated vnd.docker.ai.license layer // outranks the free-text license field in a model card's README frontmatter. licLocs, err := ociResolver.FilesByMediaType(dockerAILicenseMediaType) if err != nil { @@ -74,8 +83,8 @@ func resolveSafeTensorsOCIIdentity(ctx context.Context, resolver file.Resolver, case len(licLocs) > 0: id.licenses = identifyLicenseLayers(ctx, resolver, licLocs) id.supporting = append(id.supporting, licLocs...) - case readmeLicense != "": - id.licenses = pkg.NewLicensesFromValuesWithContext(ctx, readmeLicense) + case len(readmeLicenses) > 0: + id.licenses = pkg.NewLicensesFromValuesWithContext(ctx, readmeLicenses...) } return id @@ -108,10 +117,11 @@ func ociImageRefBasename(resolver file.Resolver) string { return path.Base(parsed.Context().RepositoryStr()) } -// classifyOCIModelFileLayer reads up to 4 MiB of a model.file layer and decodes -// it as either an HF config.json or a README model card's YAML frontmatter, -// based on its leading bytes. It returns whichever it recognized; both are nil -// when the layer is neither (or fails to decode). The caller owns precedence and +// classifyOCIModelFileLayer reads a model.file layer (up to maxModelFileLayerSize) +// and decodes it as either an HF config.json or a README model card's YAML +// frontmatter, based on its leading bytes. It returns whichever it recognized; +// both are nil when the layer is neither, fails to decode, or is JSON that +// doesn't look like an HF config (e.g. generation_config.json). The caller owns precedence and // metadata enrichment. func classifyOCIModelFileLayer(resolver file.Resolver, loc file.Location) (*hfConfig, *readmeFrontmatter) { rc, err := resolver.FileContentsByLocation(loc) @@ -120,17 +130,18 @@ func classifyOCIModelFileLayer(resolver file.Resolver, loc file.Location) (*hfCo } defer internal.CloseAndLogError(rc, loc.RealPath) - buf, err := io.ReadAll(io.LimitReader(rc, 4*1024*1024)) + buf, err := readBounded(rc, maxModelFileLayerSize) if err != nil { + log.Debugf("failed to read docker AI model-file layer: %v", err) return nil, nil } trimmed := bytes.TrimLeft(buf, "\xef\xbb\xbf \t\r\n") switch { case bytes.HasPrefix(trimmed, []byte("---")): - return nil, parseFrontmatter(buf) + return nil, parseFrontmatter(trimmed) case bytes.HasPrefix(trimmed, []byte("{")): var cfg hfConfig - if err := json.Unmarshal(buf, &cfg); err != nil { + if err := json.Unmarshal(trimmed, &cfg); err != nil || !cfg.looksLikeHF() { return nil, nil } return &cfg, nil @@ -165,8 +176,9 @@ func readLicenseSPDXIDFromFrontmatter(resolver file.Resolver, loc file.Location) } defer internal.CloseAndLogError(rc, loc.RealPath) - buf, err := io.ReadAll(io.LimitReader(rc, 64*1024)) + buf, err := readPrefix(rc, maxLicensePrefixSize) if err != nil { + log.Debugf("failed to read docker AI license layer: %v", err) return "" } return parseLicenseFrontmatter(buf) diff --git a/syft/pkg/cataloger/ai/limits.go b/syft/pkg/cataloger/ai/limits.go new file mode 100644 index 000000000..9de7bfd58 --- /dev/null +++ b/syft/pkg/cataloger/ai/limits.go @@ -0,0 +1,46 @@ +package ai + +import ( + "fmt" + "io" + + "github.com/anchore/syft/internal" +) + +// size limits for model companion files and headers. JSON reads that hit a +// limit fail (see readBounded) instead of parsing a truncated document. +// Frontmatter-only reads (README, license) take a prefix instead, since the +// frontmatter sits at the top and is itself capped by maxFrontmatterSize. +const ( + maxHFConfigSize = 4 * 1024 * 1024 + maxReadmePrefixSize = 1 * 1024 * 1024 // dir scans; OCI README layers use maxModelFileLayerSize + maxModelFileLayerSize = 4 * 1024 * 1024 + maxLicensePrefixSize = 64 * 1024 + maxModelConfigBlobSize = 1 * 1024 * 1024 + maxFrontmatterSize = 256 * 1024 + maxFrontmatterValues = 32 + maxModelNameLength = 256 + + // maxSafeTensorsHeaderSize caps the header JSON (excluding its 8-byte length + // prefix). It is shared with the OCI source so dir and OCI scans accept the + // same headers. + maxSafeTensorsHeaderSize = internal.MaxSafeTensorsHeaderSize +) + +// readPrefix reads at most limit bytes. It is for frontmatter-only reads, where +// the rest of the file is never needed. +func readPrefix(r io.Reader, limit int64) ([]byte, error) { + return io.ReadAll(io.LimitReader(r, limit)) +} + +// readBounded reads at most limit bytes and reports overflow instead of silently truncating. +func readBounded(r io.Reader, limit int64) ([]byte, error) { + b, err := io.ReadAll(io.LimitReader(r, limit+1)) + if err != nil { + return nil, err + } + if int64(len(b)) > limit { + return nil, fmt.Errorf("content exceeds %d bytes", limit) + } + return b, nil +} diff --git a/syft/pkg/cataloger/ai/naming.go b/syft/pkg/cataloger/ai/naming.go index ed2571dcb..65de1d367 100644 --- a/syft/pkg/cataloger/ai/naming.go +++ b/syft/pkg/cataloger/ai/naming.go @@ -1,15 +1,44 @@ package ai -import "path" +import ( + "path" + "strings" + "unicode/utf8" +) // pickSafeTensorsName implements the documented naming precedence chain: func pickSafeTensorsName(nameOrPath, fallbackName string) string { - if nameOrPath != "" { - return path.Base(nameOrPath) + if name := sanitizeModelName(nameOrPath); name != "" { + return name } return fallbackName } +// firstModelName returns the first value that sanitizes to a usable name. +func firstModelName(values []string) string { + for _, v := range values { + if name := sanitizeModelName(v); name != "" { + return name + } + } + return "" +} + +// sanitizeModelName reduces a producer-supplied name or path (_name_or_path, +// base_model) to its last path element, capped at maxModelNameLength runes. It +// returns "" when nothing usable is left. +func sanitizeModelName(nameOrPath string) string { + base := path.Base(strings.ReplaceAll(nameOrPath, `\`, "/")) + switch base { + case "/", ".", "..", "": + return "" + } + if utf8.RuneCountInString(base) > maxModelNameLength { + base = string([]rune(base)[:maxModelNameLength]) + } + return base +} + // safeTensorsDirName returns the directory-scan naming fallback: the base name // of the group's parent directory (the group key is already that directory). func safeTensorsDirName(directory string) string { diff --git a/syft/pkg/cataloger/ai/parse_safetensors.go b/syft/pkg/cataloger/ai/parse_safetensors.go index bc1ff1402..119a182ff 100644 --- a/syft/pkg/cataloger/ai/parse_safetensors.go +++ b/syft/pkg/cataloger/ai/parse_safetensors.go @@ -15,9 +15,7 @@ import ( // SafeTensors file format: [8 bytes u64 LE header size] [N bytes JSON header] [tensor data]. // Reference: https://github.com/huggingface/safetensors#format -const ( - maxSafeTensorsHeaderSize = 100 * 1024 * 1024 // 100MB ceiling on header JSON to prevent OOM -) +// The header JSON is capped at maxSafeTensorsHeaderSize (see limits.go). // safeTensorsHeader is the decoded JSON header. Tensor entries live alongside a // reserved "__metadata__" key holding a string-to-string producer map. We decode diff --git a/syft/pkg/cataloger/ai/parse_safetensors_oci.go b/syft/pkg/cataloger/ai/parse_safetensors_oci.go index 7fd9f3b3e..5b517b846 100644 --- a/syft/pkg/cataloger/ai/parse_safetensors_oci.go +++ b/syft/pkg/cataloger/ai/parse_safetensors_oci.go @@ -50,7 +50,7 @@ type dockerAIModelConfig struct { func parseSafeTensorsOCIConfig(_ context.Context, _ file.Resolver, _ *generic.Environment, reader file.LocationReadCloser) ([]pkg.Package, []artifact.Relationship, error) { defer internal.CloseAndLogError(reader, reader.Path()) - body, err := io.ReadAll(io.LimitReader(reader, 1024*1024)) + body, err := readBounded(reader, maxModelConfigBlobSize) if err != nil { return nil, nil, fmt.Errorf("failed to read docker AI model config: %w", err) } diff --git a/syft/pkg/cataloger/ai/parse_safetensors_test.go b/syft/pkg/cataloger/ai/parse_safetensors_test.go index 943892c40..7b98cb915 100644 --- a/syft/pkg/cataloger/ai/parse_safetensors_test.go +++ b/syft/pkg/cataloger/ai/parse_safetensors_test.go @@ -8,6 +8,8 @@ import ( "io" "os" "path/filepath" + "slices" + "strings" "testing" "github.com/stretchr/testify/assert" @@ -48,7 +50,7 @@ func buildSafeTensorsFile(t *testing.T, metadata map[string]string, tensors map[ // // The naming precedence (owned by the merge processor's pickSafeTensorsName) is: // 1. config.json _name_or_path (path.Base applied), found beside the model or -// by walking up parent directories to the scan root +// in its parent directory // 2. otherwise the model's immediate parent directory base name // → drop (no package) when neither yields a usable name // @@ -231,9 +233,9 @@ func TestSafeTensorsCataloger(t *testing.T) { expectedPackages: nil, }, { - // rung 1 via parent-walk: findDirHFConfig walks up from the model - // directory, so a config.json in an ancestor names a nested model. - name: "config.json in an ancestor directory names a nested model", + // rung 1 via the parent: findDirHFConfig also checks the model + // directory's parent, so a config.json there names a nested model. + name: "config.json in the parent directory names a nested model", setup: func(t *testing.T) string { dir := t.TempDir() model(t, filepath.Join(dir, "dir", "someothertensor")) @@ -248,6 +250,87 @@ func TestSafeTensorsCataloger(t *testing.T) { }, }, }, + { + // the walk stops at the parent: a config.json two levels up is not used. + name: "config.json two levels up is not used", + setup: func(t *testing.T) string { + dir := t.TempDir() + model(t, filepath.Join(dir, "a", "b", "someothertensor")) + writeFile(t, filepath.Join(dir, "a", "config.json"), `{"_name_or_path":"org/TooFar"}`) + return dir + }, + expectedPackages: []pkg.Package{ + { + Name: "someothertensor", + Type: pkg.ModelPkg, + Metadata: wantMetadata(""), + }, + }, + }, + { + // JSON without any HF field is not treated as a model config, so the + // lookup keeps going to the parent. + name: "non-HF config.json is skipped", + setup: func(t *testing.T) string { + dir := t.TempDir() + modelDir := filepath.Join(dir, "app", "sometensor") + model(t, modelDir) + writeFile(t, filepath.Join(modelDir, "config.json"), `{"port":8080}`) + writeFile(t, filepath.Join(dir, "app", "config.json"), `{"_name_or_path":"org/Parent"}`) + return dir + }, + expectedPackages: []pkg.Package{ + { + Name: "Parent", + Type: pkg.ModelPkg, + Metadata: wantMetadata(""), + }, + }, + }, + { + // an oversized config.json is skipped entirely, while a large README + // still yields its frontmatter (only a prefix is read). + name: "oversized config.json is ignored and a large README still names the model", + setup: func(t *testing.T) string { + dir := t.TempDir() + modelDir := filepath.Join(dir, "sometensor") + model(t, modelDir) + writeFile(t, filepath.Join(modelDir, "config.json"), + `{"architectures":["X"],"_name_or_path":"`+strings.Repeat("A", maxHFConfigSize)+`"}`) + writeFile(t, filepath.Join(modelDir, "README.md"), + "---\nlicense: MIT\nbase_model: org/Readme\n---\n"+strings.Repeat("A", maxReadmePrefixSize)) + return dir + }, + expectedPackages: []pkg.Package{ + { + Name: "Readme", + Type: pkg.ModelPkg, + Licenses: pkg.NewLicenseSet( + pkg.NewLicenseFromFields("MIT", "", nil), + ), + Metadata: wantMetadata(""), + }, + }, + }, + { + // a _name_or_path that sanitizes to nothing doesn't block README base_model. + name: "degenerate _name_or_path falls through to README base_model", + setup: func(t *testing.T) string { + dir := t.TempDir() + modelDir := filepath.Join(dir, "sometensor") + model(t, modelDir) + writeFile(t, filepath.Join(modelDir, "config.json"), `{"_name_or_path":"."}`) + writeFile(t, filepath.Join(modelDir, "README.md"), "---\nbase_model: org/Readme\n---\n") + return dir + }, + expectedPackages: []pkg.Package{ + { + Name: "Readme", + Type: pkg.ModelPkg, + Metadata: wantMetadata(""), + }, + }, + }, { // grouping: independent models in one scan are grouped by their own // parent directory and each named from it. @@ -601,6 +684,60 @@ spdx-id: Apache-2.0 assertHasLicense(t, out[0], "Apache-2.0") }) + t.Run("OCI: config precedence is stable across layer order", func(t *testing.T) { + dir := t.TempDir() + a := filepath.Join(dir, "a.json") + b := filepath.Join(dir, "b.json") + gen := filepath.Join(dir, "generation_config.json") + require.NoError(t, os.WriteFile(a, []byte(`{"_name_or_path":"org/first"}`), 0o644)) + require.NoError(t, os.WriteFile(b, []byte(`{"_name_or_path":"org/second"}`), 0o644)) + require.NoError(t, os.WriteFile(gen, []byte(`{"max_new_tokens":10}`), 0o644)) + + configMd := pkg.SafeTensorsModelInfo{Format: "safetensors", TensorCount: 1} + for _, order := range [][]string{{a, b, gen}, {gen, b, a}} { + var locs []file.Location + for _, p := range order { + locs = append(locs, file.NewLocation(p)) + } + resolver := file.NewMockResolverForMediaTypes(map[string][]file.Location{ + dockerAIModelFileMediaType: locs, + }) + out, _, err := safeTensorsMergeProcessor(context.Background(), resolver, []pkg.Package{ociPkg(configMd)}, nil, nil) + require.NoError(t, err) + require.Len(t, out, 1) + assert.Equal(t, "first", out[0].Name) + // the non-HF generation config is not recorded as supporting evidence + assert.Len(t, out[0].Locations.ToSlice(), 3) + } + }) + + t.Run("dir scan: non-HF config.json in the parent is not supporting evidence", func(t *testing.T) { + dir := t.TempDir() + cfgPath := filepath.Join(dir, "config.json") + require.NoError(t, os.WriteFile(cfgPath, []byte(`{"port":8080}`), 0o644)) + loc, cfg := findDirHFConfig(file.NewMockResolverForPaths(cfgPath), filepath.Join(dir, "model")) + assert.Nil(t, loc) + assert.Nil(t, cfg) + }) + + t.Run("OCI: a large license layer still yields its frontmatter spdx-id", func(t *testing.T) { + dir := t.TempDir() + licensePath := filepath.Join(dir, "LICENSE") + require.NoError(t, os.WriteFile(licensePath, + []byte("---\nspdx-id: Apache-2.0\n---\n"+strings.Repeat("A", maxLicensePrefixSize)), 0o644)) + assert.Equal(t, "Apache-2.0", readLicenseSPDXIDFromFrontmatter(file.NewMockResolverForPaths(licensePath), file.NewLocation(licensePath))) + }) + + t.Run("OCI: oversized model.file layer is rejected", func(t *testing.T) { + dir := t.TempDir() + cfgPath := filepath.Join(dir, "config.json") + require.NoError(t, os.WriteFile(cfgPath, + []byte(`{"_name_or_path":"org/x","pad":"`+strings.Repeat("A", maxModelFileLayerSize)+`"}`), 0o644)) + cfg, fm := classifyOCIModelFileLayer(file.NewMockResolverForPaths(cfgPath), file.NewLocation(cfgPath)) + assert.Nil(t, cfg) + assert.Nil(t, fm) + }) + t.Run("names packages and still propagates an upstream error", func(t *testing.T) { // A non-nil upstream error (e.g. another file in the run failed to parse) // must NOT short-circuit naming. The package is still nameless at this @@ -622,7 +759,7 @@ spdx-id: Apache-2.0 // (or neither, asserting the drop path). // // Precedence (highest → lowest): -// 1. config.json _name_or_path (path.Base applied; both dir-scan and OCI) +// 1. config.json _name_or_path (sanitized to its last path element; both dir-scan and OCI) // 2. fallback name — OCI image-ref last segment, or dir-scan parent directory // base name (the merge processor computes the right one per group) // → drop (empty name) when nothing matches @@ -663,6 +800,31 @@ func TestSafeTensorsNamingPrecedence(t *testing.T) { want: "tiny-llama", }, + // sanitizing + { + name: "windows-style path uses the last element", + nameOrPath: `C:\models\win-model`, + fallbackName: "fallback-name", + want: "win-model", + }, + { + name: "degenerate path falls back", + nameOrPath: "..", + fallbackName: "fallback-name", + want: "fallback-name", + }, + { + name: "root path falls back", + nameOrPath: "/", + fallbackName: "fallback-name", + want: "fallback-name", + }, + { + name: "long names are truncated", + nameOrPath: "org/" + strings.Repeat("x", maxModelNameLength+10), + want: strings.Repeat("x", maxModelNameLength), + }, + // drops { name: "drop: both rungs empty", @@ -1036,25 +1198,78 @@ func TestParseFrontmatter(t *testing.T) { name string input string wantNil bool - wantLicense string + wantLicenses []string wantBaseModel []string }{ { name: "list base_model", input: "---\nlicense: mit\nbase_model:\n - org/Model\n---\nbody", - wantLicense: "mit", + wantLicenses: []string{"mit"}, wantBaseModel: []string{"org/Model"}, }, { name: "scalar base_model", input: "---\nlicense: apache-2.0\nbase_model: org/Model\n---\n", - wantLicense: "apache-2.0", + wantLicenses: []string{"apache-2.0"}, wantBaseModel: []string{"org/Model"}, }, { - name: "leading BOM", - input: "\xef\xbb\xbf---\nlicense: mit\n---\n", - wantLicense: "mit", + name: "list license keeps base_model", + input: "---\nlicense: [mit, apache-2.0]\nbase_model: org/Model\n---\n", + wantLicenses: []string{"mit", "apache-2.0"}, + wantBaseModel: []string{"org/Model"}, + }, + { + name: "mapping-valued fields are ignored, not fatal", + input: "---\nlicense: {name: mit}\nbase_model: org/Model\n---\n", + wantBaseModel: []string{"org/Model"}, + }, + { + name: "leading BOM", + input: "\xef\xbb\xbf---\nlicense: mit\n---\n", + wantLicenses: []string{"mit"}, + }, + { + name: "null license is ignored", + input: "---\nlicense: null\nbase_model: ~\n---\n", + wantLicenses: nil, + wantBaseModel: nil, + }, + { + name: "empty block", + input: "---\n---\n", + wantNil: true, + }, + { + name: "oversized block is rejected", + input: "---\nlicense: mit\n" + strings.Repeat("# pad\n", maxFrontmatterSize/6+1) + "---\n", + wantNil: true, + }, + { + name: "closing delimiter at EOF with trailing whitespace", + input: "---\nlicense: mit\n--- ", + wantLicenses: []string{"mit"}, + }, + { + name: "list values are capped", + input: "---\nlicense: [" + strings.Repeat("mit,", maxFrontmatterValues+5) + "mit]\n---\n", + wantLicenses: slices.Repeat([]string{"mit"}, maxFrontmatterValues), + }, + { + name: "CRLF delimiters", + input: "---\r\nlicense: mit\r\n---\r\nbody", + wantLicenses: []string{"mit"}, + }, + { + name: "opening delimiter must be exactly ---", + input: "----\nlicense: mit\n---\n", + wantNil: true, + }, + { + name: "a ---x line inside the block does not close it", + input: "---\nbase_model: org/Model\n---x: 1\nlicense: mit\n---\n", + wantLicenses: []string{"mit"}, + wantBaseModel: []string{"org/Model"}, }, { name: "no frontmatter", @@ -1082,7 +1297,7 @@ func TestParseFrontmatter(t *testing.T) { return } require.NotNil(t, fm) - assert.Equal(t, tt.wantLicense, fm.License) + assert.Equal(t, tt.wantLicenses, fm.Licenses) if tt.wantBaseModel != nil { assert.Equal(t, tt.wantBaseModel, fm.BaseModel) } diff --git a/syft/pkg/cataloger/ai/processor.go b/syft/pkg/cataloger/ai/processor.go index fd0e29292..5cb31c3e7 100644 --- a/syft/pkg/cataloger/ai/processor.go +++ b/syft/pkg/cataloger/ai/processor.go @@ -104,10 +104,10 @@ func mergeDirModels(ctx context.Context, resolver file.Resolver, pkgs []pkg.Pack // groupByParentDir buckets filesystem-scanned models by the directory their // primary-evidence file lives in. - +// // This encodes a deliberate assumption: a directory holds one logical // model, so every .safetensors file in a directory is treated as a shard of the -// same modeland merged into one package. +// same model and merged into one package. // The trade-off is that if a directory happens to contain several unrelated models, // they are merged into one package rather than reported separately. // We accept that because the conventional on-disk layout gives each model diff --git a/syft/pkg/cataloger/ai/testdata/safetensors/extract_header.go b/syft/pkg/cataloger/ai/testdata/safetensors/extract_header.go index 4f7d72a60..8445adcb7 100644 --- a/syft/pkg/cataloger/ai/testdata/safetensors/extract_header.go +++ b/syft/pkg/cataloger/ai/testdata/safetensors/extract_header.go @@ -28,9 +28,9 @@ import ( const ( safetensorsLayerMediaType = "application/vnd.docker.ai.safetensors" - // 8 MB matches maxHeaderBytes in the OCI model source. Real model headers - // are well under 1 MB; the extra slack covers outliers. - fetchBytes = 8 * 1024 * 1024 + // matches maxSafeTensorsHeaderBytes in the OCI model source (8 MB header + // cap plus the 8-byte length prefix). Real model headers are well under 1 MB. + fetchBytes = 8*1024*1024 + 8 ) func main() { diff --git a/syft/source/ocimodelsource/oci_model_source.go b/syft/source/ocimodelsource/oci_model_source.go index c93e9c3fa..77ab1b187 100644 --- a/syft/source/ocimodelsource/oci_model_source.go +++ b/syft/source/ocimodelsource/oci_model_source.go @@ -135,8 +135,8 @@ func fetchAndStoreModelHeaders(ctx context.Context, client *registryClient, arti } } - // SafeTensors weight-layer headers. We only pull the leading prefix (same - // budget as a GGUF header) + // SafeTensors weight-layer headers. We only pull the leading + // maxSafeTensorsHeaderBytes if artifact.Format == modelFormatSafeTensors { for _, layer := range artifact.SafeTensorsLayers { li, err := fetchSafeTensorsLayerHeader(ctx, client, artifact.Reference, layer, tempDir) @@ -174,7 +174,7 @@ func storeConfigBlobAsLayer(artifact *modelArtifact, tempDir string) (fileresolv // fetchCompanionLayer downloads a companion (non-weight) layer to a temp file func fetchCompanionLayer(ctx context.Context, client *registryClient, ref name.Reference, layer v1.Descriptor, tempDir string) (fileresolver.LayerInfo, error) { - data, err := client.fetchBlobRange(ctx, ref, layer.Digest, maxCompanionBytes) + data, err := client.fetchBlobRange(ctx, ref, layer.Digest, maxCompanionBytes+1) if err != nil { return fileresolver.LayerInfo{}, fmt.Errorf("failed to fetch companion layer: %w", err) } @@ -191,7 +191,7 @@ func fetchCompanionLayer(ctx context.Context, client *registryClient, ref name.R // fetchSingleGGUFHeader fetches a single GGUF layer header and writes it to a temp file func fetchSingleGGUFHeader(ctx context.Context, client *registryClient, ref name.Reference, layer v1.Descriptor, tempDir string) (fileresolver.LayerInfo, error) { - headerData, err := client.fetchBlobRange(ctx, ref, layer.Digest, maxWeightHeaderBytes) + headerData, err := client.fetchBlobRange(ctx, ref, layer.Digest, maxGGUFHeaderBytes) if err != nil { return fileresolver.LayerInfo{}, fmt.Errorf("failed to fetch GGUF layer header: %w", err) } @@ -212,7 +212,7 @@ func fetchSingleGGUFHeader(ctx context.Context, client *registryClient, ref name // fetchSafeTensorsLayerHeader fetches the leading bytes of a SafeTensors weight // layer (enough to cover the JSON header) and writes them to a temp file func fetchSafeTensorsLayerHeader(ctx context.Context, client *registryClient, ref name.Reference, layer v1.Descriptor, tempDir string) (fileresolver.LayerInfo, error) { - headerData, err := client.fetchBlobRange(ctx, ref, layer.Digest, maxWeightHeaderBytes) + headerData, err := client.fetchBlobRange(ctx, ref, layer.Digest, maxSafeTensorsHeaderBytes) if err != nil { return fileresolver.LayerInfo{}, fmt.Errorf("failed to fetch safetensors layer header: %w", err) } diff --git a/syft/source/ocimodelsource/registry_client.go b/syft/source/ocimodelsource/registry_client.go index 8e6c6b9c7..0d4be3a22 100644 --- a/syft/source/ocimodelsource/registry_client.go +++ b/syft/source/ocimodelsource/registry_client.go @@ -16,6 +16,7 @@ import ( "github.com/google/go-containerregistry/pkg/v1/remote" "github.com/anchore/stereoscope/pkg/image" + "github.com/anchore/syft/internal" ) // errNotModelArtifact is returned when a reference does not point to a model artifact. @@ -37,17 +38,20 @@ const ( modelFormatGGUF = "gguf" modelFormatSafeTensors = "safetensors" - // maxWeightHeaderBytes is the leading slice we range-GET from a (multi-GB) - // weight layer — enough to cover the GGUF/safetensors header. Note this is - // smaller than the ai cataloger's own maxSafeTensorsHeaderSize (100 MB) parse - // ceiling: a safetensors header between the two is parseable from a directory - // scan but truncated here, so its shard would go uncounted on an OCI scan. - // Keep this comfortably above real-world header sizes. - maxWeightHeaderBytes = 8 * 1024 * 1024 // 8 MB + // maxGGUFHeaderBytes is the leading slice we range-GET from a (multi-GB) + // GGUF weight layer, enough to cover its header. See + // https://github.com/ggml-org/ggml/blob/master/docs/gguf.md#file-structure + maxGGUFHeaderBytes = 8 * 1024 * 1024 // 8 MB + + // maxSafeTensorsHeaderBytes is the leading slice we range-GET from a + // safetensors weight layer: the 8-byte length prefix plus the same header + // ceiling the ai cataloger applies to directory scans. + maxSafeTensorsHeaderBytes = internal.MaxSafeTensorsHeaderSize + 8 // maxCompanionBytes caps a whole companion blob (README, config.json, - // license); these are small by convention. Matches the 4 MB read cap in - // classifyOCIModelFileLayer. + // license); these are small by convention. It is the upper bound of the ai + // cataloger's per-layer caps. One extra byte is fetched so the cataloger can + // tell an oversized blob from one that is exactly at the cap. maxCompanionBytes = 4 * 1024 * 1024 // 4 MB ) @@ -279,9 +283,6 @@ func (c *registryClient) fetchBlobRange(ctx context.Context, ref name.Reference, } defer reader.Close() - // Note: this is not some arbitrary number picked out of the blue. - // This is based on the specification of header data found here: - // https://github.com/ggml-org/ggml/blob/master/docs/gguf.md#file-structure data := make([]byte, maxBytes) n, err := io.ReadFull(reader, data) From a4d3aedf4e7527089934f257045a11cee184d134 Mon Sep 17 00:00:00 2001 From: Alex Goodman Date: Tue, 29 Sep 2026 09:40:17 -0400 Subject: [PATCH 21/40] fix(ai): GGUF layer handling and safetensors parameter counts (#5343) * fix(ai): keep merging GGUF layers when one fails to parse a single unparseable layer in an OCI GGUF artifact skipped the merge, so the named model lost its parts and nameless layers were dropped. The error is still returned. Signed-off-by: Alex Goodman * fix(ai): only send GGUF layers to the GGUF cataloger the media type glob `application/vnd.docker.ai*` also matched config, model.file, license and safetensors layers, so every safetensors OCI model reported spurious "invalid GGUF magic number" unknowns. Match `application/vnd.docker.ai.gguf.v3` exactly. Signed-off-by: Alex Goodman * fix(ai): don't wrap safetensors parameter counts - tensors whose shape product overflows uint64 are skipped instead of wrapping - per-file and per-shard parameter totals saturate instead of wrapping - parameter count and dominant dtype are computed in one pass - GGUF licenses are created with the scan context, and drop no-op unknown checks Signed-off-by: Alex Goodman * refactor(ai): derive safetensors model info from the header in one place the file and OCI layer parsers built identical metadata from a parsed header. Signed-off-by: Alex Goodman * fix(ai): keep the GGUF media type with the GGUF cataloger also corrects stale comments in the GGUF merge processor. Signed-off-by: Alex Goodman --------- Signed-off-by: Alex Goodman --- syft/pkg/cataloger/ai/cataloger.go | 8 +- syft/pkg/cataloger/ai/cataloger_test.go | 46 +++++++++++ syft/pkg/cataloger/ai/gguf_processor.go | 14 ++-- syft/pkg/cataloger/ai/merge.go | 2 +- syft/pkg/cataloger/ai/merge_test.go | 8 ++ syft/pkg/cataloger/ai/package.go | 6 +- syft/pkg/cataloger/ai/package_test.go | 2 + syft/pkg/cataloger/ai/parse_gguf.go | 2 +- syft/pkg/cataloger/ai/parse_gguf_model.go | 4 +- syft/pkg/cataloger/ai/parse_safetensors.go | 80 ++++++++++++------- .../cataloger/ai/parse_safetensors_model.go | 14 +--- .../pkg/cataloger/ai/parse_safetensors_oci.go | 12 +-- .../cataloger/ai/parse_safetensors_test.go | 33 +++++++- syft/pkg/cataloger/ai/processor_test.go | 18 +++++ 14 files changed, 179 insertions(+), 70 deletions(-) diff --git a/syft/pkg/cataloger/ai/cataloger.go b/syft/pkg/cataloger/ai/cataloger.go index 5b7a75882..9aa7ef4b3 100644 --- a/syft/pkg/cataloger/ai/cataloger.go +++ b/syft/pkg/cataloger/ai/cataloger.go @@ -11,8 +11,12 @@ import ( const ( catalogerName = "gguf-cataloger" - ggufLayerMediaType = "application/vnd.docker.ai*" safeTensorsCatalogerName = "safetensors-cataloger" + + // dockerAIGGUFMediaType mirrors ggufLayerMediaType in + // syft/source/ocimodelsource/registry_client.go. It is matched exactly: the + // other application/vnd.docker.ai.* layers are not GGUF. + dockerAIGGUFMediaType = "application/vnd.docker.ai.gguf.v3" ) // NewGGUFCataloger returns a new cataloger instance for GGUF model files. @@ -21,7 +25,7 @@ const ( func NewGGUFCataloger() pkg.Cataloger { return generic.NewCataloger(catalogerName). WithParserByGlobs(parseGGUFModel, "**/*.gguf"). - WithParserByMediaType(parseGGUFModel, ggufLayerMediaType). + WithParserByMediaType(parseGGUFModel, dockerAIGGUFMediaType). WithProcessors(ggufMergeProcessor) } diff --git a/syft/pkg/cataloger/ai/cataloger_test.go b/syft/pkg/cataloger/ai/cataloger_test.go index d8c28d6d0..1ab6b59dd 100644 --- a/syft/pkg/cataloger/ai/cataloger_test.go +++ b/syft/pkg/cataloger/ai/cataloger_test.go @@ -1,10 +1,17 @@ package ai import ( + "context" "os" "path/filepath" "testing" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/anchore/syft/internal/tmpdir" + "github.com/anchore/syft/syft/internal/fileresolver" + "github.com/anchore/syft/syft/artifact" "github.com/anchore/syft/syft/pkg" "github.com/anchore/syft/syft/pkg/cataloger/internal/pkgtest" @@ -138,3 +145,42 @@ func TestGGUFCataloger(t *testing.T) { }) } } + +func TestGGUFCataloger_ociLayers(t *testing.T) { + // a GGUF OCI artifact is merged into one named package, while the other + // application/vnd.docker.ai.* layers (config, model.file, license and + // safetensors weights) never reach the GGUF parser. + named := newTestGGUFBuilder(). + withStringKV("general.architecture", "llama"). + withStringKV("general.name", "llama3-8b"). + build() + part := newTestGGUFBuilder().withStringKV("general.architecture", "llama").build() + + dir := t.TempDir() + layers := map[string]fileresolver.LayerInfo{} + for digest, l := range map[string]struct { + mediaType string + content []byte + }{ + "sha256:named": {dockerAIGGUFMediaType, named}, + "sha256:part": {dockerAIGGUFMediaType, part}, + "sha256:config": {dockerAIModelConfigMediaTypes[0], []byte("not a gguf file")}, + "sha256:file": {dockerAIModelFileMediaType, []byte("not a gguf file")}, + "sha256:license": {dockerAILicenseMediaType, []byte("not a gguf file")}, + "sha256:weights": {dockerAISafeTensorsMediaType, []byte("not a gguf file")}, + } { + p := filepath.Join(dir, digest[len("sha256:"):]) + require.NoError(t, os.WriteFile(p, l.content, 0o600)) + layers[digest] = fileresolver.LayerInfo{TempPath: p, MediaType: l.mediaType} + } + resolver := fileresolver.NewContainerImageModel(dir, layers, "example/model:latest") + ctx := tmpdir.WithValue(context.Background(), tmpdir.FromPath(t.TempDir())) + + pkgs, _, err := NewGGUFCataloger().Catalog(ctx, resolver) + require.NoError(t, err) + require.Len(t, pkgs, 1) + assert.Equal(t, "llama3-8b", pkgs[0].Name) + header, ok := pkgs[0].Metadata.(pkg.GGUFFileHeader) + require.True(t, ok) + assert.Len(t, header.Parts, 1) +} diff --git a/syft/pkg/cataloger/ai/gguf_processor.go b/syft/pkg/cataloger/ai/gguf_processor.go index fe31e3f8b..34b93f006 100644 --- a/syft/pkg/cataloger/ai/gguf_processor.go +++ b/syft/pkg/cataloger/ai/gguf_processor.go @@ -8,17 +8,17 @@ import ( // ggufMergeProcessor consolidates multiple GGUF packages into a single package // representing the AI model. When scanning OCI images with multiple layers, // each layer may produce a separate package. This processor finds the package -// with a name and merges metadata from nameless packages into its GGUFFileParts field. +// with a name and merges metadata from nameless packages into its Parts field. // Only packages with a non-empty name are returned in the final result. func ggufMergeProcessor(pkgs []pkg.Package, rels []artifact.Relationship, err error) ([]pkg.Package, []artifact.Relationship, error) { - if err != nil { - return pkgs, rels, err - } + // don't early-return on err: it aggregates per-file failures, and skipping the + // merge would drop every nameless layer and the named package's Parts along + // with it (see safeTensorsMergeProcessor). if len(pkgs) == 0 { return pkgs, rels, err } - // Separate packages with names from those without + // separate packages with names from those without var namedPkgs []pkg.Package var namelessHeaders []pkg.GGUFFileHeader @@ -27,14 +27,14 @@ func ggufMergeProcessor(pkgs []pkg.Package, rels []artifact.Relationship, err er namedPkgs = append(namedPkgs, p) } else { if header, ok := p.Metadata.(pkg.GGUFFileHeader); ok { - // We do not want a kv hash for nameless headers + // we do not want a kv hash for nameless headers header.MetadataKeyValuesHash = "" namelessHeaders = append(namelessHeaders, header) } } } - // If there are no named packages, return nothing + // if there are no named packages, return nothing if len(namedPkgs) == 0 { return nil, rels, err } diff --git a/syft/pkg/cataloger/ai/merge.go b/syft/pkg/cataloger/ai/merge.go index a8d2b4e74..24e37e685 100644 --- a/syft/pkg/cataloger/ai/merge.go +++ b/syft/pkg/cataloger/ai/merge.go @@ -75,7 +75,7 @@ func mergeShardsInto(merged *pkg.SafeTensorsModelInfo, shards []pkg.SafeTensorsM seenKV := map[string]bool{} for _, s := range shards { shardTensorTotal += s.TensorCount - merged.Parameters += s.Parameters + merged.Parameters = saturatingAdd(merged.Parameters, s.Parameters) firstNonEmpty(&merged.Quantization, s.Quantization) for _, kv := range s.UserMetadata { if seenKV[kv.Key] { diff --git a/syft/pkg/cataloger/ai/merge_test.go b/syft/pkg/cataloger/ai/merge_test.go index 7276d8896..08f22939f 100644 --- a/syft/pkg/cataloger/ai/merge_test.go +++ b/syft/pkg/cataloger/ai/merge_test.go @@ -1,6 +1,7 @@ package ai import ( + "math" "testing" "github.com/stretchr/testify/assert" @@ -164,3 +165,10 @@ func TestRollupHash(t *testing.T) { assert.NotEqual(t, "a", ab) assert.NotEqual(t, "b", ab) } + +func TestMergeShardsInto_parametersSaturate(t *testing.T) { + var merged pkg.SafeTensorsModelInfo + shards := []pkg.SafeTensorsModelInfo{{Parameters: math.MaxUint64 - 1}, {Parameters: 2}} + mergeShardsInto(&merged, shards) + assert.Equal(t, uint64(math.MaxUint64), merged.Parameters) +} diff --git a/syft/pkg/cataloger/ai/package.go b/syft/pkg/cataloger/ai/package.go index 3abb42343..978ad2a95 100644 --- a/syft/pkg/cataloger/ai/package.go +++ b/syft/pkg/cataloger/ai/package.go @@ -1,17 +1,19 @@ package ai import ( + "context" + "github.com/anchore/syft/syft/file" "github.com/anchore/syft/syft/pkg" ) -func newGGUFPackage(metadata *pkg.GGUFFileHeader, modelName, version, license string, locations ...file.Location) pkg.Package { +func newGGUFPackage(ctx context.Context, metadata *pkg.GGUFFileHeader, modelName, version, license string, locations ...file.Location) pkg.Package { p := pkg.Package{ Name: modelName, Version: version, Locations: file.NewLocationSet(locations...), Type: pkg.ModelPkg, - Licenses: pkg.NewLicenseSet(pkg.NewLicensesFromValues(license)...), + Licenses: pkg.NewLicenseSet(pkg.NewLicensesFromValuesWithContext(ctx, license)...), Metadata: *metadata, // NOTE: PURL is intentionally not set as the package-url spec // has not yet finalized support for ML model packages diff --git a/syft/pkg/cataloger/ai/package_test.go b/syft/pkg/cataloger/ai/package_test.go index ea58304d6..3c8efbb7f 100644 --- a/syft/pkg/cataloger/ai/package_test.go +++ b/syft/pkg/cataloger/ai/package_test.go @@ -1,6 +1,7 @@ package ai import ( + "context" "testing" "github.com/stretchr/testify/require" @@ -103,6 +104,7 @@ func TestNewGGUFPackage(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { actual := newGGUFPackage( + context.Background(), tt.metadata, tt.input.modelName, tt.input.version, diff --git a/syft/pkg/cataloger/ai/parse_gguf.go b/syft/pkg/cataloger/ai/parse_gguf.go index 3150110a3..9437cafe2 100644 --- a/syft/pkg/cataloger/ai/parse_gguf.go +++ b/syft/pkg/cataloger/ai/parse_gguf.go @@ -45,7 +45,7 @@ func copyHeader(w io.Writer, r io.Reader) error { return nil } -// Helper to convert gguf_parser metadata to simpler types +// helper to convert gguf_parser metadata to simpler types func convertGGUFMetadataKVs(kvs gguf_parser.GGUFMetadataKVs) map[string]any { result := make(map[string]any) diff --git a/syft/pkg/cataloger/ai/parse_gguf_model.go b/syft/pkg/cataloger/ai/parse_gguf_model.go index f59fddfd5..d879a1165 100644 --- a/syft/pkg/cataloger/ai/parse_gguf_model.go +++ b/syft/pkg/cataloger/ai/parse_gguf_model.go @@ -15,7 +15,6 @@ import ( "github.com/anchore/syft/internal" "github.com/anchore/syft/internal/log" "github.com/anchore/syft/internal/tmpdir" - "github.com/anchore/syft/internal/unknown" "github.com/anchore/syft/syft/artifact" "github.com/anchore/syft/syft/file" "github.com/anchore/syft/syft/pkg" @@ -79,6 +78,7 @@ func parseGGUFModel(ctx context.Context, _ file.Resolver, _ *generic.Environment // Create package from metadata p := newGGUFPackage( + ctx, syftMetadata, metadata.Name, modelVersion, @@ -86,7 +86,7 @@ func parseGGUFModel(ctx context.Context, _ file.Resolver, _ *generic.Environment reader.WithAnnotation(pkg.EvidenceAnnotationKey, pkg.PrimaryEvidenceAnnotation), ) - return []pkg.Package{p}, nil, unknown.IfEmptyf([]pkg.Package{p}, "unable to parse GGUF file") + return []pkg.Package{p}, nil, nil } // computeKVMetadataHash computes a stable hash of the KV metadata for use as a global identifier diff --git a/syft/pkg/cataloger/ai/parse_safetensors.go b/syft/pkg/cataloger/ai/parse_safetensors.go index 119a182ff..2ee3f206b 100644 --- a/syft/pkg/cataloger/ai/parse_safetensors.go +++ b/syft/pkg/cataloger/ai/parse_safetensors.go @@ -5,6 +5,8 @@ import ( "encoding/json" "fmt" "io" + "math" + "math/bits" "sort" "strings" @@ -82,47 +84,67 @@ func readSafeTensorsHeader(r io.Reader) (*safeTensorsHeader, error) { return h, nil } -// parameterCount sums the element counts across all tensors in the header. -func (h *safeTensorsHeader) parameterCount() uint64 { - var total uint64 - for _, t := range h.tensors { - count := uint64(1) - for _, dim := range t.Shape { - if dim <= 0 { - count = 0 - break - } - count *= uint64(dim) - } - total += count +// modelInfo returns the metadata derivable from the header bytes alone. Naming, +// licenses and ShardCount are left to the merge processor. +func (h *safeTensorsHeader) modelInfo() pkg.SafeTensorsModelInfo { + params, dtype := h.parameterStats() + return pkg.SafeTensorsModelInfo{ + Format: "safetensors", + TensorCount: uint64(len(h.tensors)), + Parameters: params, + Quantization: normalizeDType(dtype), + UserMetadata: userMetadataKeyValues(h.metadata), + MetadataHash: h.metadataHash(), } - return total } -// dominantDType returns the dtype that accounts for the largest fraction of parameters. -// For mixed-precision models the "dominant" dtype is still a useful summary. -func (h *safeTensorsHeader) dominantDType() string { +// parameterStats sums the element counts across all tensors and returns the +// dtype that accounts for the largest share of them. For mixed-precision models +// the "dominant" dtype is still a useful summary. Totals saturate rather than wrap. +func (h *safeTensorsHeader) parameterStats() (total uint64, dominantDType string) { sizeByDType := make(map[string]uint64) for _, t := range h.tensors { - count := uint64(1) - for _, dim := range t.Shape { - if dim <= 0 { - count = 0 - break - } - count *= uint64(dim) + count, ok := tensorElements(t.Shape) + if !ok { + continue } - sizeByDType[t.DType] += count + total = saturatingAdd(total, count) + sizeByDType[t.DType] = saturatingAdd(sizeByDType[t.DType], count) } - var best string var bestSize uint64 for dtype, size := range sizeByDType { - if size > bestSize || (size == bestSize && dtype < best) { - best = dtype + if size > bestSize || (size == bestSize && dtype < dominantDType) { + dominantDType = dtype bestSize = size } } - return best + return total, dominantDType +} + +// tensorElements returns the element count for a shape, or ok=false on a +// non-positive dim or when the product overflows uint64. +func tensorElements(shape []int64) (uint64, bool) { + n := uint64(1) + for _, d := range shape { + if d <= 0 { + return 0, false + } + hi, lo := bits.Mul64(n, uint64(d)) + if hi != 0 { + return 0, false + } + n = lo + } + return n, true +} + +// saturatingAdd returns a+b, clamped to math.MaxUint64 instead of wrapping. +func saturatingAdd(a, b uint64) uint64 { + sum, carry := bits.Add64(a, b, 0) + if carry != 0 { + return math.MaxUint64 + } + return sum } // metadataHash returns a stable xxhash64 over the logical tensor content diff --git a/syft/pkg/cataloger/ai/parse_safetensors_model.go b/syft/pkg/cataloger/ai/parse_safetensors_model.go index a9a8f3c66..da3234240 100644 --- a/syft/pkg/cataloger/ai/parse_safetensors_model.go +++ b/syft/pkg/cataloger/ai/parse_safetensors_model.go @@ -6,7 +6,6 @@ import ( "io" "github.com/anchore/syft/internal" - "github.com/anchore/syft/internal/unknown" "github.com/anchore/syft/syft/artifact" "github.com/anchore/syft/syft/file" "github.com/anchore/syft/syft/pkg" @@ -26,22 +25,13 @@ func parseSafeTensorsFile(_ context.Context, _ file.Resolver, _ *generic.Environ return nil, nil, fmt.Errorf("failed to read safetensors header: %w", err) } - // ShardCount is intentionally not set here: the merge processor is the single - // owner of ShardCount and derives it from the number of shards in the group. - md := pkg.SafeTensorsModelInfo{ - Format: "safetensors", - TensorCount: uint64(len(header.tensors)), - Parameters: header.parameterCount(), - Quantization: normalizeDType(header.dominantDType()), - UserMetadata: userMetadataKeyValues(header.metadata), - MetadataHash: header.metadataHash(), - } + md := header.modelInfo() p := newSafeTensorsPackage( &md, reader.WithAnnotation(pkg.EvidenceAnnotationKey, pkg.PrimaryEvidenceAnnotation), ) - return []pkg.Package{p}, nil, unknown.IfEmptyf([]pkg.Package{p}, "unable to parse safetensors file") + return []pkg.Package{p}, nil, nil } // integrity check diff --git a/syft/pkg/cataloger/ai/parse_safetensors_oci.go b/syft/pkg/cataloger/ai/parse_safetensors_oci.go index 5b517b846..8aa04e5b0 100644 --- a/syft/pkg/cataloger/ai/parse_safetensors_oci.go +++ b/syft/pkg/cataloger/ai/parse_safetensors_oci.go @@ -8,7 +8,6 @@ import ( "strings" "github.com/anchore/syft/internal" - "github.com/anchore/syft/internal/unknown" "github.com/anchore/syft/syft/artifact" "github.com/anchore/syft/syft/file" "github.com/anchore/syft/syft/pkg" @@ -81,7 +80,7 @@ func parseSafeTensorsOCIConfig(_ context.Context, _ file.Resolver, _ *generic.En &md, reader.WithAnnotation(pkg.EvidenceAnnotationKey, pkg.PrimaryEvidenceAnnotation), ) - return []pkg.Package{p}, nil, unknown.IfEmptyf([]pkg.Package{p}, "unable to parse docker AI safetensors config") + return []pkg.Package{p}, nil, nil } // parseSafeTensorsOCILayer decodes the JSON header of a SafeTensors weight @@ -94,14 +93,7 @@ func parseSafeTensorsOCILayer(_ context.Context, _ file.Resolver, _ *generic.Env return nil, nil, fmt.Errorf("failed to read safetensors layer header: %w", err) } - md := pkg.SafeTensorsModelInfo{ - Format: "safetensors", - TensorCount: uint64(len(header.tensors)), - Parameters: header.parameterCount(), - Quantization: normalizeDType(header.dominantDType()), - UserMetadata: userMetadataKeyValues(header.metadata), - MetadataHash: header.metadataHash(), - } + md := header.modelInfo() p := newSafeTensorsPackage( &md, diff --git a/syft/pkg/cataloger/ai/parse_safetensors_test.go b/syft/pkg/cataloger/ai/parse_safetensors_test.go index 7b98cb915..caf7d21d9 100644 --- a/syft/pkg/cataloger/ai/parse_safetensors_test.go +++ b/syft/pkg/cataloger/ai/parse_safetensors_test.go @@ -6,6 +6,7 @@ import ( "encoding/binary" "encoding/json" "io" + "math" "os" "path/filepath" "slices" @@ -1146,8 +1147,9 @@ func TestSafeTensorsHeader_parameterCountAndDType(t *testing.T) { "small": {DType: "F32", Shape: []int64{16, 16}}, "scalar": {DType: "F32", Shape: []int64{}}, // empty shape contributes 1 }} - assert.Equal(t, uint64(1000*16+16*16+1), h.parameterCount()) - assert.Equal(t, "BF16", h.dominantDType()) + params, dtype := h.parameterStats() + assert.Equal(t, uint64(1000*16+16*16+1), params) + assert.Equal(t, "BF16", dtype) } func TestSafeTensorsHeader_parameterCount_nonPositiveDims(t *testing.T) { @@ -1159,7 +1161,29 @@ func TestSafeTensorsHeader_parameterCount_nonPositiveDims(t *testing.T) { "zero": {DType: "F32", Shape: []int64{4, 0}}, "negative": {DType: "F32", Shape: []int64{-1, 8}}, }} - assert.Equal(t, uint64(16), h.parameterCount()) + params, _ := h.parameterStats() + assert.Equal(t, uint64(16), params) +} + +func TestSafeTensorsHeader_parameterStats_overflow(t *testing.T) { + // a shape whose product overflows uint64 is skipped instead of wrapping (the + // first tensor would wrap to 0) and can't flip the dominant dtype; the total + // saturates instead of wrapping. + h := &safeTensorsHeader{tensors: map[string]safeTensorsEntry{ + "good": {DType: "BF16", Shape: []int64{4, 4}}, + "wraps": {DType: "F32", Shape: []int64{4294967296, 4294967296}}, + "overflow": {DType: "F32", Shape: []int64{math.MaxInt64, 3}}, + }} + params, dtype := h.parameterStats() + assert.Equal(t, uint64(16), params) + assert.Equal(t, "BF16", dtype) + + h = &safeTensorsHeader{tensors: map[string]safeTensorsEntry{ + "a": {DType: "F32", Shape: []int64{math.MaxInt64, 2}}, + "b": {DType: "F32", Shape: []int64{math.MaxInt64, 2}}, + }} + params, _ = h.parameterStats() + assert.Equal(t, uint64(math.MaxUint64), params) } func TestSafeTensorsHeader_dominantDType_tieBreak(t *testing.T) { @@ -1169,7 +1193,8 @@ func TestSafeTensorsHeader_dominantDType_tieBreak(t *testing.T) { "a": {DType: "F32", Shape: []int64{4, 4}}, "b": {DType: "BF16", Shape: []int64{4, 4}}, }} - assert.Equal(t, "BF16", h.dominantDType()) + _, dtype := h.parameterStats() + assert.Equal(t, "BF16", dtype) } func TestNormalizeDType(t *testing.T) { diff --git a/syft/pkg/cataloger/ai/processor_test.go b/syft/pkg/cataloger/ai/processor_test.go index 6d216ae95..e99f9a9ce 100644 --- a/syft/pkg/cataloger/ai/processor_test.go +++ b/syft/pkg/cataloger/ai/processor_test.go @@ -1,6 +1,7 @@ package ai import ( + "errors" "testing" "github.com/stretchr/testify/assert" @@ -61,3 +62,20 @@ func Test_ggufMergeProcessor(t *testing.T) { }) } } + +func Test_ggufMergeProcessor_mergesDespiteError(t *testing.T) { + // one layer failing to parse must not drop the rest of the model + parseErr := errors.New("one layer failed to parse") + pkgs := []pkg.Package{ + {Name: "model", Metadata: pkg.GGUFFileHeader{MetadataKeyValuesHash: "abc"}}, + {Name: "", Metadata: pkg.GGUFFileHeader{MetadataKeyValuesHash: "part1"}}, + } + + got, _, err := ggufMergeProcessor(pkgs, nil, parseErr) + require.ErrorIs(t, err, parseErr) + require.Len(t, got, 1) + assert.Equal(t, "model", got[0].Name) + header, ok := got[0].Metadata.(pkg.GGUFFileHeader) + require.True(t, ok) + assert.Len(t, header.Parts, 1) +} From ae7843e17c676c15cf9a64202b20df4935efd6fd Mon Sep 17 00:00:00 2001 From: Alex Goodman Date: Tue, 29 Sep 2026 10:03:31 -0400 Subject: [PATCH 22/40] fix(ai): handle malformed GGUF headers without panicking (#5345) - pre-scan the GGUF metadata KV section and reject values gguf-parser-go would panic on, recurse on without bound, or overallocate for: wrong typed well-known keys, zero alignment, nested or oversized fully decoded arrays, an unreasonable `split.tensors.count`, and empty strings the parser would misread. Keys are matched the way the parser sees them. The KV count, fully read string bytes, and skipped string counts are bounded too. The file becomes an unknown instead of taking down the scan. - `copyHeader` now bounds its own read to the max header size. - non-finite floats in `header` (including inside arrays) become `"NaN"`, `"+Inf"` or `"-Inf"` so JSON encoding can't fail. `header` and `metadataHash` are otherwise unchanged. - a `metadataHash` failure is now an error instead of an empty hash. Signed-off-by: Alex Goodman --- syft/pkg/cataloger/ai/parse_gguf.go | 303 +++++++++++++++++- syft/pkg/cataloger/ai/parse_gguf_model.go | 36 ++- syft/pkg/cataloger/ai/parse_gguf_test.go | 251 +++++++++++++++ .../cataloger/ai/parse_safetensors_test.go | 25 -- syft/pkg/cataloger/ai/test_helpers_test.go | 34 +- 5 files changed, 605 insertions(+), 44 deletions(-) create mode 100644 syft/pkg/cataloger/ai/parse_gguf_test.go diff --git a/syft/pkg/cataloger/ai/parse_gguf.go b/syft/pkg/cataloger/ai/parse_gguf.go index 9437cafe2..efcfc232d 100644 --- a/syft/pkg/cataloger/ai/parse_gguf.go +++ b/syft/pkg/cataloger/ai/parse_gguf.go @@ -1,9 +1,14 @@ package ai import ( + "bufio" "encoding/binary" + "errors" "fmt" "io" + "math" + "slices" + "strings" gguf_parser "github.com/gpustack/gguf-parser-go" ) @@ -12,12 +17,41 @@ import ( const ( ggufMagicNumber = 0x46554747 // "GGUF" in little-endian maxHeaderSize = 50 * 1024 * 1024 // 50MB for large tokenizer vocabularies + + // maxGGUFArrayDepth allows arrays of arrays but nothing deeper. + maxGGUFArrayDepth = 2 + // maxGGUFDecodedItems bounds the items, across the whole header, of arrays that gguf-parser-go + // always decodes in full. Real files hold a few per-layer arrays. + maxGGUFDecodedItems = 65536 + // maxGGUFSplitTensors bounds split.tensors.count, which the library preallocates tensor infos for. + // Shards declare the total across all splits, so this can't be the header tensor count. + maxGGUFSplitTensors = 1 << 17 + // maxGGUFKeyLength is the key length limit from the GGUF spec. + maxGGUFKeyLength = 65535 + // maxGGUFKVCount bounds the metadata KV count. Real files hold hundreds; vocabularies live in arrays. + maxGGUFKVCount = 65536 + // maxGGUFReadStringBytes bounds, across the header, the key and string bytes the library reads in + // full and syft then hashes and encodes (chat templates are the largest, at tens of KB). + maxGGUFReadStringBytes = 8 << 20 + // maxGGUFSkippedStrings bounds, across the header, the strings the library skips one at a time + // inside arrays (vocabularies run to a few hundred thousand tokens and merges). + maxGGUFSkippedStrings = 1 << 22 ) -// copyHeader copies the GGUF header from the reader to the writer. +// ggufDecodedArraySuffixes are the keys gguf-parser-go decodes in full even with SkipLargeMetadata +// (hardcoded in its ReadArray as of v0.26.3; re-check on upgrade). +var ggufDecodedArraySuffixes = []string{ + ".feed_forward_length", ".attention.head_count", ".attention.head_count_kv", ".attention.sliding_window_pattern", +} + +// ggufScalarSizes is the encoded size of each value type, indexed by type (0 for strings and arrays). +var ggufScalarSizes = [...]int{1, 1, 2, 2, 4, 4, 4, 1, 0, 0, 8, 8, 8} + +// copyHeader copies at most maxHeaderSize bytes of the GGUF header from the reader to the writer. // It validates the magic number first, then copies the rest of the data. -// The reader should be wrapped with io.LimitedReader to prevent OOM issues. func copyHeader(w io.Writer, r io.Reader) error { + r = io.LimitReader(r, maxHeaderSize) + // Read initial chunk to validate magic number // GGUF format: magic(4) + version(4) + tensor_count(8) + metadata_kv_count(8) + metadata_kvs + tensors_info initialBuf := make([]byte, 24) // Enough for magic, version, tensor count, and kv count @@ -37,7 +71,6 @@ func copyHeader(w io.Writer, r io.Reader) error { } // Copy the rest of the header from reader to writer - // The LimitedReader will return EOF once maxHeaderSize is reached if _, err := io.Copy(w, r); err != nil { return fmt.Errorf("failed to copy GGUF header: %w", err) } @@ -45,6 +78,268 @@ func copyHeader(w io.Writer, r io.Reader) error { return nil } +// validateGGUFHeader walks the metadata KV section and rejects values that gguf-parser-go would panic +// on, recurse on without bound, or overallocate for. It reads without storing anything. +// ponytail: tensor infos are not checked here (zero block size types, dimension overflow); those +// panics are recoverable and are left to the per-file panic recovery in the generic cataloger. +func validateGGUFHeader(r io.Reader) error { + v := &ggufHeaderValidator{ + r: bufio.NewReader(r), + decodedBudget: maxGGUFDecodedItems, + readStringBudget: maxGGUFReadStringBytes, + skippedStringBudget: maxGGUFSkippedStrings, + } + var prefix struct { + Magic, Version uint32 + } + if err := binary.Read(v.r, binary.LittleEndian, &prefix); err != nil { + return fmt.Errorf("reading header prefix: %w", err) + } + v.v1 = prefix.Version <= 1 + // the tensor count is validated by the library + if _, err := v.readLen(); err != nil { + return fmt.Errorf("reading tensor count: %w", err) + } + kvCount, err := v.readLen() + if err != nil { + return fmt.Errorf("reading metadata count: %w", err) + } + if kvCount > maxGGUFKVCount { + return fmt.Errorf("metadata count %d exceeds %d", kvCount, maxGGUFKVCount) + } + + for i := uint64(0); i < kvCount; i++ { + key, err := v.readKey() + if err != nil { + return fmt.Errorf("metadata key %d: %w", i, err) + } + var vt uint32 + if err := binary.Read(v.r, binary.LittleEndian, &vt); err != nil { + return fmt.Errorf("metadata key %q: %w", key, err) + } + if err := v.checkKey(key, gguf_parser.GGUFMetadataValueType(vt)); err != nil { + return fmt.Errorf("metadata key %q: %w", key, err) + } + } + return nil +} + +type ggufHeaderValidator struct { + r *bufio.Reader + v1 bool // v1 uses 32-bit lengths + decodedBudget uint64 + readStringBudget uint64 + skippedStringBudget uint64 +} + +func (v *ggufHeaderValidator) checkKey(key string, vt gguf_parser.GGUFMetadataValueType) error { + decoded := slices.ContainsFunc(ggufDecodedArraySuffixes, func(s string) bool { return strings.HasSuffix(key, s) }) + n, err := v.readValue(vt, decoded, 1) + if err != nil { + return err + } + + // these are read with typed accessors that panic on a type mismatch + switch key { + case "general.architecture", "general.name", "general.license", "general.type", "general.author", + "general.url", "general.description", "controlvector.model_hint": + if vt != gguf_parser.GGUFMetadataValueTypeString { + return fmt.Errorf("expected string, got %v", vt) + } + case "general.quantization_version", "general.file_type": + if !vt.IsNumeric() { + return fmt.Errorf("expected number, got %v", vt) + } + case "general.alignment": + if vt != gguf_parser.GGUFMetadataValueTypeUint32 || n == 0 { + return fmt.Errorf("expected non-zero uint32, got %v %v", vt, n) + } + case "split.tensors.count": + if !vt.IsNumeric() { + return fmt.Errorf("expected number, got %v", vt) + } + if math.IsNaN(n) || n < 0 || n > maxGGUFSplitTensors { + return fmt.Errorf("unreasonable split tensor count %.0f", n) + } + } + return nil +} + +// readValue consumes one value, returning scalar numbers as float64 for the key checks. +func (v *ggufHeaderValidator) readValue(vt gguf_parser.GGUFMetadataValueType, decoded bool, depth int) (float64, error) { + if uint32(vt) >= uint32(len(ggufScalarSizes)) { + return 0, fmt.Errorf("invalid value type %d", vt) + } + switch vt { + case gguf_parser.GGUFMetadataValueTypeString: + // only strings inside skipped arrays are skipped by the library, the rest are read in full + return 0, v.skipString(depth > 1 && !decoded) + case gguf_parser.GGUFMetadataValueTypeArray: + return 0, v.readArray(decoded, depth) + } + size := ggufScalarSizes[vt] + b, err := v.r.Peek(size) + if err != nil { + return 0, err + } + var buf [8]byte + copy(buf[:], b) + if _, err := v.r.Discard(size); err != nil { + return 0, err + } + u := binary.LittleEndian.Uint64(buf[:]) + switch vt { + case gguf_parser.GGUFMetadataValueTypeInt8: + return float64(int8(u)), nil + case gguf_parser.GGUFMetadataValueTypeInt16: + return float64(int16(u)), nil + case gguf_parser.GGUFMetadataValueTypeInt32: + return float64(int32(u)), nil + case gguf_parser.GGUFMetadataValueTypeInt64: + return float64(int64(u)), nil + case gguf_parser.GGUFMetadataValueTypeFloat32: + return float64(math.Float32frombits(uint32(u))), nil + case gguf_parser.GGUFMetadataValueTypeFloat64: + return math.Float64frombits(u), nil + } + return float64(u), nil +} + +func (v *ggufHeaderValidator) readArray(decoded bool, depth int) error { + var itemType uint32 + if err := binary.Read(v.r, binary.LittleEndian, &itemType); err != nil { + return err + } + n, err := v.readLen() + if err != nil { + return err + } + it := gguf_parser.GGUFMetadataValueType(itemType) + switch { + case itemType >= uint32(len(ggufScalarSizes)): + return fmt.Errorf("invalid array item type %d", itemType) + case decoded && n > v.decodedBudget: + return fmt.Errorf("decoded arrays exceed %d items", maxGGUFDecodedItems) + case it == gguf_parser.GGUFMetadataValueTypeArray && (!decoded || depth >= maxGGUFArrayDepth): + // the library can only skip over arrays of scalars + return errors.New("unsupported nested array") + case n > maxHeaderSize: + return fmt.Errorf("array length %d exceeds header size", n) + case !decoded && ggufScalarSizes[itemType] > 0: + // skipped arrays of fixed size items can be passed over in one go + _, err := v.r.Discard(int(n) * ggufScalarSizes[itemType]) + return err + } + if decoded { + v.decodedBudget -= n + } + for i := uint64(0); i < n; i++ { + if _, err := v.readValue(it, decoded, depth+1); err != nil { + return err + } + } + return nil +} + +func (v *ggufHeaderValidator) readLen() (uint64, error) { + if v.v1 { + var n uint32 + err := binary.Read(v.r, binary.LittleEndian, &n) + return uint64(n), err + } + var n uint64 + err := binary.Read(v.r, binary.LittleEndian, &n) + return n, err +} + +// readKey reads a key the way gguf-parser-go will see it, whitespace trimmed. +func (v *ggufHeaderValidator) readKey() (string, error) { + n, err := v.readLen() + if err != nil { + return "", err + } + if n == 0 || n > maxGGUFKeyLength { + return "", fmt.Errorf("invalid key length %d", n) + } + if err := v.spendReadString(n); err != nil { + return "", err + } + buf := make([]byte, n) + if _, err := io.ReadFull(v.r, buf); err != nil { + return "", err + } + return strings.TrimSpace(string(buf)), nil +} + +func (v *ggufHeaderValidator) skipString(allowEmpty bool) error { + n, err := v.readLen() + if err != nil { + return err + } + // gguf-parser-go reads an empty string as a full pool buffer, so everything after it would be + // parsed out of step with this walk + if n == 0 && !allowEmpty { + return errors.New("empty string") + } + if n > maxHeaderSize { + return fmt.Errorf("string length %d exceeds header size", n) + } + if allowEmpty { + if v.skippedStringBudget == 0 { + return fmt.Errorf("skipped strings exceed %d", maxGGUFSkippedStrings) + } + v.skippedStringBudget-- + } else if err := v.spendReadString(n); err != nil { + return err + } + _, err = v.r.Discard(int(n)) + return err +} + +func (v *ggufHeaderValidator) spendReadString(n uint64) error { + if n > v.readStringBudget { + return fmt.Errorf("strings exceed %d bytes", maxGGUFReadStringBytes) + } + v.readStringBudget -= n + return nil +} + +// sanitizeGGUFValue replaces non-finite floats, which JSON can not encode, leaving everything else as is. +func sanitizeGGUFValue(v any) any { + switch t := v.(type) { + case float32: + if f := float64(t); math.IsNaN(f) || math.IsInf(f, 0) { + return nonFiniteString(f) + } + case float64: + if math.IsNaN(t) || math.IsInf(t, 0) { + return nonFiniteString(t) + } + case gguf_parser.GGUFMetadataKVArrayValue: + // keep the library shape as is; items are only present for the few keys the library + // decodes in full, which the pre-scan keeps small and shallow + if len(t.Array) > 0 { + items := make([]any, len(t.Array)) + for i, item := range t.Array { + items[i] = sanitizeGGUFValue(item) + } + t.Array = items + } + return t + } + return v +} + +func nonFiniteString(f float64) string { + switch { + case math.IsInf(f, 1): + return "+Inf" + case math.IsInf(f, -1): + return "-Inf" + } + return "NaN" +} + // helper to convert gguf_parser metadata to simpler types func convertGGUFMetadataKVs(kvs gguf_parser.GGUFMetadataKVs) map[string]any { result := make(map[string]any) @@ -56,7 +351,7 @@ func convertGGUFMetadataKVs(kvs gguf_parser.GGUFMetadataKVs) map[string]any { "general.version", "general.parameter_count", "general.quantization": continue } - result[kv.Key] = kv.Value + result[kv.Key] = sanitizeGGUFValue(kv.Value) } return result diff --git a/syft/pkg/cataloger/ai/parse_gguf_model.go b/syft/pkg/cataloger/ai/parse_gguf_model.go index d879a1165..6b9a89259 100644 --- a/syft/pkg/cataloger/ai/parse_gguf_model.go +++ b/syft/pkg/cataloger/ai/parse_gguf_model.go @@ -13,7 +13,6 @@ import ( gguf_parser "github.com/gpustack/gguf-parser-go" "github.com/anchore/syft/internal" - "github.com/anchore/syft/internal/log" "github.com/anchore/syft/internal/tmpdir" "github.com/anchore/syft/syft/artifact" "github.com/anchore/syft/syft/file" @@ -37,14 +36,20 @@ func parseGGUFModel(ctx context.Context, _ file.Resolver, _ *generic.Environment defer cleanup() tempPath := tempFile.Name() - // Copy and validate the GGUF file header using LimitedReader to prevent OOM - // We use LimitedReader to cap reads at maxHeaderSize (50MB) - limitedReader := &io.LimitedReader{R: reader, N: maxHeaderSize} - if err := copyHeader(tempFile, limitedReader); err != nil { + // copy and validate the GGUF file header (bounded to maxHeaderSize) + if err := copyHeader(tempFile, reader); err != nil { tempFile.Close() return nil, nil, fmt.Errorf("failed to copy GGUF header: %w", err) } + if _, err := tempFile.Seek(0, io.SeekStart); err != nil { + tempFile.Close() + return nil, nil, fmt.Errorf("failed to rewind GGUF header: %w", err) + } + err = validateGGUFHeader(tempFile) tempFile.Close() + if err != nil { + return nil, nil, fmt.Errorf("invalid GGUF header: %w", err) + } // Parse using gguf-parser-go with options to skip unnecessary data ggufFile, err := gguf_parser.ParseGGUFFile(tempPath, @@ -60,6 +65,11 @@ func parseGGUFModel(ctx context.Context, _ file.Resolver, _ *generic.Environment // Extract version separately (will be set on Package.Version) modelVersion := extractVersion(ggufFile.Header.MetadataKV) + metadataHash, err := computeKVMetadataHash(ggufFile.Header.MetadataKV) + if err != nil { + return nil, nil, err + } + // Convert to syft metadata structure syftMetadata := &pkg.GGUFFileHeader{ Architecture: metadata.Architecture, @@ -68,7 +78,7 @@ func parseGGUFModel(ctx context.Context, _ file.Resolver, _ *generic.Environment GGUFVersion: uint32(ggufFile.Header.Version), TensorCount: ggufFile.Header.TensorCount, RemainingKeyValues: convertGGUFMetadataKVs(ggufFile.Header.MetadataKV), - MetadataKeyValuesHash: computeKVMetadataHash(ggufFile.Header.MetadataKV), + MetadataKeyValuesHash: metadataHash, } // If model name is not in metadata, use filename @@ -89,11 +99,14 @@ func parseGGUFModel(ctx context.Context, _ file.Resolver, _ *generic.Environment return []pkg.Package{p}, nil, nil } -// computeKVMetadataHash computes a stable hash of the KV metadata for use as a global identifier -func computeKVMetadataHash(metadata gguf_parser.GGUFMetadataKVs) string { +// computeKVMetadataHash computes a stable hash of the sanitized KV metadata for use as a global identifier +func computeKVMetadataHash(metadata gguf_parser.GGUFMetadataKVs) (string, error) { // Sort the KV pairs by key for stable hashing sortedKVs := make([]gguf_parser.GGUFMetadataKV, len(metadata)) - copy(sortedKVs, metadata) + for i, kv := range metadata { + kv.Value = sanitizeGGUFValue(kv.Value) + sortedKVs[i] = kv + } sort.Slice(sortedKVs, func(i, j int) bool { return sortedKVs[i].Key < sortedKVs[j].Key }) @@ -101,13 +114,12 @@ func computeKVMetadataHash(metadata gguf_parser.GGUFMetadataKVs) string { // Marshal sorted KVs to JSON for stable hashing jsonBytes, err := json.Marshal(sortedKVs) if err != nil { - log.Debugf("failed to marshal metadata for hashing: %v", err) - return "" + return "", fmt.Errorf("failed to marshal GGUF metadata for hashing: %w", err) } // Compute xxhash hash := xxhash.Sum64(jsonBytes) - return fmt.Sprintf("%016x", hash) // 16 hex chars (64 bits) + return fmt.Sprintf("%016x", hash), nil // 16 hex chars (64 bits) } // extractVersion attempts to extract version from metadata KV pairs diff --git a/syft/pkg/cataloger/ai/parse_gguf_test.go b/syft/pkg/cataloger/ai/parse_gguf_test.go new file mode 100644 index 000000000..c5d8d5253 --- /dev/null +++ b/syft/pkg/cataloger/ai/parse_gguf_test.go @@ -0,0 +1,251 @@ +package ai + +import ( + "bytes" + "context" + "encoding/binary" + "encoding/json" + "io" + "math" + "strings" + "testing" + + gguf_parser "github.com/gpustack/gguf-parser-go" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/anchore/syft/internal/tmpdir" + "github.com/anchore/syft/syft/file" + "github.com/anchore/syft/syft/format/syftjson" + "github.com/anchore/syft/syft/pkg" + "github.com/anchore/syft/syft/sbom" +) + +func le(v any) []byte { + buf := new(bytes.Buffer) + binary.Write(buf, binary.LittleEndian, v) + return buf.Bytes() +} + +func ggufString(s string) []byte { + return append(le(uint64(len(s))), s...) +} + +func parseGGUFBytes(t *testing.T, data []byte) ([]pkg.Package, error) { + t.Helper() + ctx, td := tmpdir.Root(context.Background(), "gguf-test") + t.Cleanup(func() { _ = td.Cleanup() }) + reader := file.NewLocationReadCloser(file.NewLocation("/model.gguf"), io.NopCloser(bytes.NewReader(data))) + var pkgs []pkg.Package + var err error + require.NotPanics(t, func() { + pkgs, _, err = parseGGUFModel(ctx, nil, nil, reader) + }) + return pkgs, err +} + +func TestParseGGUFModel_malformedHeader(t *testing.T) { + // nested arrays of arrays, each one level deeper, ending in an empty uint8 array + nested := func(depth int) []byte { + v := ggufArray(ggufTypeUint8, 0, nil) + for i := 1; i < depth; i++ { + v = ggufArray(ggufTypeArray, 1, v) + } + return v + } + + // a header that declares more KVs than the cap, with none following + tooManyKVs := append(append([]byte("GGUF"), le(uint32(3))...), le([]uint64{0, maxGGUFKVCount + 1})...) + + tests := []struct { + name string + builder *testGGUFBuilder + data []byte // used instead of builder when set + wantErr bool + }{ + {name: "too many KVs", data: tooManyKVs, wantErr: true}, + {name: "split tensor count is a string", builder: newTestGGUFBuilder().withStringKV("split.tensors.count", "1"), wantErr: true}, + {name: "split tensor count over the cap", builder: newTestGGUFBuilder().withUint32KV("split.tensors.count", maxGGUFSplitTensors+1), wantErr: true}, + {name: "read strings too long in total", builder: newTestGGUFBuilder().withStringKV("x", strings.Repeat("a", maxGGUFReadStringBytes)), wantErr: true}, + { + name: "skipped strings too many in total", + builder: newTestGGUFBuilder().withRawKV("tokenizer.ggml.tokens", ggufTypeArray, ggufArray(ggufTypeString, maxGGUFSkippedStrings+1, make([]byte, 8*(maxGGUFSkippedStrings+1)))), + wantErr: true, + }, + {name: "v1 header", builder: newTestGGUFBuilder().withVersion(1).withStringKV("general.name", "m").withUint32KV("llama.context_length", 8)}, + {name: "architecture not a string", builder: newTestGGUFBuilder().withUint32KV("general.architecture", 1), wantErr: true}, + {name: "name not a string", builder: newTestGGUFBuilder().withUint32KV("general.name", 1), wantErr: true}, + {name: "license not a string", builder: newTestGGUFBuilder().withUint32KV("general.license", 1), wantErr: true}, + {name: "type not a string", builder: newTestGGUFBuilder().withUint32KV("general.type", 1), wantErr: true}, + {name: "author not a string", builder: newTestGGUFBuilder().withUint32KV("general.author", 1), wantErr: true}, + {name: "url not a string", builder: newTestGGUFBuilder().withUint32KV("general.url", 1), wantErr: true}, + {name: "description not a string", builder: newTestGGUFBuilder().withUint32KV("general.description", 1), wantErr: true}, + {name: "model hint not a string", builder: newTestGGUFBuilder().withUint32KV("controlvector.model_hint", 1), wantErr: true}, + {name: "quantization version is a string", builder: newTestGGUFBuilder().withStringKV("general.quantization_version", "2"), wantErr: true}, + {name: "alignment is a string", builder: newTestGGUFBuilder().withStringKV("general.alignment", "32"), wantErr: true}, + {name: "alignment is uint64", builder: newTestGGUFBuilder().withUint64KV("general.alignment", 32), wantErr: true}, + {name: "alignment is zero", builder: newTestGGUFBuilder().withUint32KV("general.alignment", 0), wantErr: true}, + { + name: "array of arrays on a skipped key", + builder: newTestGGUFBuilder().withRawKV("tokenizer.ggml.merges", ggufTypeArray, ggufArray(ggufTypeArray, 1, ggufArray(ggufTypeUint8, 0, nil))), + wantErr: true, + }, + { + name: "invalid array item type", + builder: newTestGGUFBuilder().withRawKV("tokenizer.ggml.scores", ggufTypeArray, ggufArray(99, 0, nil)), + wantErr: true, + }, + {name: "split tensor count out of range", builder: newTestGGUFBuilder().withUint64KV("split.tensors.count", 1<<62), wantErr: true}, + {name: "split tensor count huge", builder: newTestGGUFBuilder().withUint64KV("split.tensors.count", 1<<40), wantErr: true}, + {name: "split tensor count negative", builder: newTestGGUFBuilder().withRawKV("split.tensors.count", ggufTypeInt64, le(int64(-1))), wantErr: true}, + { + name: "nesting too deep on a decoded key", + builder: newTestGGUFBuilder().withRawKV("llama.attention.head_count", ggufTypeArray, nested(3)), + wantErr: true, + }, + { + name: "decoded array too long", + builder: newTestGGUFBuilder().withRawKV("llama.feed_forward_length", ggufTypeArray, ggufArray(ggufTypeUint8, maxGGUFDecodedItems+1, make([]byte, maxGGUFDecodedItems+1))), + wantErr: true, + }, + {name: "empty string value", builder: newTestGGUFBuilder().withStringKV("general.author", ""), wantErr: true}, + {name: "padded key", builder: newTestGGUFBuilder().withUint32KV(" general.alignment\t", 0), wantErr: true}, + {name: "long padded key", builder: newTestGGUFBuilder().withUint32KV(strings.Repeat(" ", 300)+"general.alignment", 0), wantErr: true}, + {name: "file type is a string", builder: newTestGGUFBuilder().withUint32KV("general.file_type", 1).withStringKV("general.file_type", "x"), wantErr: true}, + { + name: "decoded arrays too long in total", + builder: newTestGGUFBuilder(). + withRawKV("a.feed_forward_length", ggufTypeArray, ggufArray(ggufTypeUint8, maxGGUFDecodedItems, make([]byte, maxGGUFDecodedItems))). + withRawKV("b.feed_forward_length", ggufTypeArray, ggufArray(ggufTypeUint8, 1, make([]byte, 1))), + wantErr: true, + }, + {name: "invalid value type", builder: newTestGGUFBuilder().withRawKV("x", 0xFFFFFFFF, nil), wantErr: true}, + {name: "empty key", builder: newTestGGUFBuilder().withUint32KV("", 1), wantErr: true}, + { + name: "empty string in a decoded array", + builder: newTestGGUFBuilder().withRawKV("llama.attention.head_count", ggufTypeArray, ggufArray(ggufTypeString, 1, le(uint64(0)))), + wantErr: true, + }, + { + name: "empty string in a skipped array", + builder: newTestGGUFBuilder().withRawKV("tokenizer.ggml.tokens", ggufTypeArray, ggufArray(ggufTypeString, 1, le(uint64(0)))), + }, + { + name: "array of arrays on a decoded key", + builder: newTestGGUFBuilder().withRawKV("llama.attention.head_count", ggufTypeArray, nested(2)), + }, + {name: "alignment is a uint32", builder: newTestGGUFBuilder().withUint32KV("general.alignment", 32)}, + { + name: "split keys as written by llama.cpp", + builder: newTestGGUFBuilder(). + withRawKV("split.no", ggufTypeUint16, le(uint16(0))). + withRawKV("split.count", ggufTypeUint16, le(uint16(3))). + withRawKV("split.tensors.count", ggufTypeInt32, le(int32(3000))), + }, + { + name: "split shard declares the total tensor count", + builder: newTestGGUFBuilder().withRawKV("split.tensors.count", ggufTypeInt32, le(int32(3000))), + }, + { + name: "skipped scalar array", + builder: newTestGGUFBuilder().withRawKV("tokenizer.ggml.scores", ggufTypeArray, ggufArray(ggufTypeFloat32, 3, make([]byte, 12))), + }, + // ponytail: zero block size tensor types and dimension overflow panic inside tensor info parsing; + // those are left to the generic cataloger's panic handling rather than checked here. + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + data := tt.data + if data == nil { + data = tt.builder.withStringKV("general.architecture", "llama").build() + } + pkgs, err := parseGGUFBytes(t, data) + if tt.wantErr { + // the pre-scan has to be what rejects these, before the library sees them + require.ErrorContains(t, err, "invalid GGUF header") + assert.Empty(t, pkgs) + return + } + require.NoError(t, err) + require.Len(t, pkgs, 1) + }) + } +} + +func TestParseGGUFModel_nonFiniteFloat(t *testing.T) { + data := newTestGGUFBuilder(). + withStringKV("general.architecture", "llama"). + withRawKV("llama.rope.freq_base", ggufTypeFloat32, le(float32(math.Inf(1)))). + withRawKV("llama.rope.scale", ggufTypeFloat64, le(math.Inf(-1))). + withRawKV("llama.attention.head_count", ggufTypeArray, ggufArray(ggufTypeFloat32, 2, le([]float32{8, float32(math.NaN())}))). + withRawKV("tokenizer.ggml.scores", ggufTypeArray, ggufArray(ggufTypeFloat32, 1, le(float32(0)))). + build() + + pkgs, err := parseGGUFBytes(t, data) + require.NoError(t, err) + require.Len(t, pkgs, 1) + + meta := pkgs[0].Metadata.(pkg.GGUFFileHeader) + assert.NotEmpty(t, meta.MetadataKeyValuesHash) + assert.Equal(t, "+Inf", meta.RemainingKeyValues["llama.rope.freq_base"]) + assert.Equal(t, "-Inf", meta.RemainingKeyValues["llama.rope.scale"]) + headCount := meta.RemainingKeyValues["llama.attention.head_count"].(gguf_parser.GGUFMetadataKVArrayValue) + assert.Equal(t, []any{float32(8), "NaN"}, headCount.Array) + + var buf bytes.Buffer + s := sbom.SBOM{Artifacts: sbom.Artifacts{Packages: pkg.NewCollection(pkgs...)}} + require.NoError(t, syftjson.NewFormatEncoder().Encode(&buf, s)) + assert.NotEmpty(t, buf.Bytes()) +} + +// TestParseGGUFModel_outputUnchanged pins the header output and hash for ordinary values to what syft +// produced as of f7f3c6efc, before malformed header handling. The expected values were captured by +// running this same fixture on that commit. float32 0.1 catches any widening to float64. +func TestParseGGUFModel_outputUnchanged(t *testing.T) { + data := newTestGGUFBuilder(). + withStringKV("general.architecture", "llama"). + withRawKV("tokenizer.ggml.tokens", ggufTypeArray, ggufArray(ggufTypeString, 2, append(ggufString("a"), ggufString("b")...))). + withRawKV("llama.attention.head_count", ggufTypeArray, ggufArray(ggufTypeFloat32, 2, le([]float32{0.1, 0.2}))). + withRawKV("llama.feed_forward_length", ggufTypeArray, ggufArray(ggufTypeArray, 2, append( + ggufArray(ggufTypeUint32, 2, le([]uint32{1, 2})), + ggufArray(ggufTypeUint32, 1, le([]uint32{3}))...))). + withRawKV("llama.rope.freq_base", ggufTypeFloat32, le(float32(0.1))). + build() + + pkgs, err := parseGGUFBytes(t, data) + require.NoError(t, err) + require.Len(t, pkgs, 1) + + meta := pkgs[0].Metadata.(pkg.GGUFFileHeader) + assert.Equal(t, "4f28d55f0985a985", meta.MetadataKeyValuesHash) + header, err := json.Marshal(meta.RemainingKeyValues) + require.NoError(t, err) + assert.JSONEq(t, `{ + "llama.attention.head_count": {"type": 6, "len": 2, "array": [0.1, 0.2], "startOffset": 170, "size": 8}, + "llama.feed_forward_length": {"type": 9, "len": 2, "array": [ + {"type": 4, "len": 2, "array": [1, 2], "startOffset": 239, "size": 8}, + {"type": 4, "len": 1, "array": [3], "startOffset": 259, "size": 4} + ], "startOffset": 227, "size": 36}, + "llama.rope.freq_base": 0.1, + "tokenizer.ggml.tokens": {"type": 8, "len": 2, "startOffset": 102, "size": 18} + }`, string(header)) +} + +// TestCopyHeader_appliesItsOwnBound pins that copyHeader caps the copy at maxHeaderSize itself, +// whatever reader it is handed. +// +// Bytes retained is asserted directly rather than through testutils.MeasureAlloc: the destination is a +// bytes.Buffer, whose doubling growth costs a multiple of what it holds and lands on a different power +// of two depending on the platform. buf.Len() is the same property without that noise. +func TestCopyHeader_appliesItsOwnBound(t *testing.T) { + const payload = 2*maxHeaderSize + 1 + + file := make([]byte, 24, 24+payload) + binary.LittleEndian.PutUint32(file[0:4], ggufMagicNumber) + file = append(file, bytes.Repeat([]byte{'x'}, payload)...) + + var buf bytes.Buffer + require.NoError(t, copyHeader(&buf, bytes.NewReader(file))) + assert.Equal(t, maxHeaderSize, buf.Len(), "copyHeader must cap the copy at maxHeaderSize") +} diff --git a/syft/pkg/cataloger/ai/parse_safetensors_test.go b/syft/pkg/cataloger/ai/parse_safetensors_test.go index caf7d21d9..c5d4457cb 100644 --- a/syft/pkg/cataloger/ai/parse_safetensors_test.go +++ b/syft/pkg/cataloger/ai/parse_safetensors_test.go @@ -1485,28 +1485,3 @@ func TestReadSafeTensorsHeader_capBoundsHeap(t *testing.T) { assert.Less(t, bounded, uint64(8*1024*1024), "the declared-size check has to reject before the body is read, not after") } - -// TestCopyHeader_boundIsTheCallersLimitedReader pins where the GGUF bound actually lives. copyHeader -// io.Copy's whatever it is handed, so maxHeaderSize only holds because every caller wraps the reader -// first. A caller that stops doing that reads the whole file, and nothing in copyHeader would object. -// -// Bytes retained is asserted directly rather than through testutils.MeasureAlloc: the destination is a -// bytes.Buffer, whose doubling growth costs a multiple of what it holds and lands on a different power -// of two depending on the platform. buf.Len() is the same property without that noise. -func TestCopyHeader_boundIsTheCallersLimitedReader(t *testing.T) { - const payload = 2*maxHeaderSize + 1 - - file := make([]byte, 24, 24+payload) - binary.LittleEndian.PutUint32(file[0:4], ggufMagicNumber) - file = append(file, bytes.Repeat([]byte{'x'}, payload)...) - - var unbounded bytes.Buffer - require.NoError(t, copyHeader(&unbounded, bytes.NewReader(file))) - require.Equal(t, len(file), unbounded.Len(), - "an unwrapped reader is copied whole; copyHeader applies no limit of its own") - - var bounded bytes.Buffer - require.NoError(t, copyHeader(&bounded, &io.LimitedReader{R: bytes.NewReader(file), N: maxHeaderSize})) - assert.Equal(t, maxHeaderSize, bounded.Len(), - "a wrapped reader must cap the copy at maxHeaderSize") -} diff --git a/syft/pkg/cataloger/ai/test_helpers_test.go b/syft/pkg/cataloger/ai/test_helpers_test.go index 195e90d92..d11ce8bb1 100644 --- a/syft/pkg/cataloger/ai/test_helpers_test.go +++ b/syft/pkg/cataloger/ai/test_helpers_test.go @@ -72,8 +72,32 @@ func (b *testGGUFBuilder) withUint32KV(key string, value uint32) *testGGUFBuilde return b } +// withRawKV adds a KV whose value bytes are written verbatim, for arrays and malformed values. +func (b *testGGUFBuilder) withRawKV(key string, valueType uint32, raw []byte) *testGGUFBuilder { + b.kvPairs = append(b.kvPairs, testKVPair{key: key, valueType: valueType, value: raw}) + return b +} + +// ggufArray encodes an array value: item type, length, then the already encoded items. +func ggufArray(itemType uint32, n uint64, items []byte) []byte { + buf := new(bytes.Buffer) + binary.Write(buf, binary.LittleEndian, itemType) + binary.Write(buf, binary.LittleEndian, n) + buf.Write(items) + return buf.Bytes() +} + +// writeLen writes a count or length, which is 32-bit in GGUF v1 and 64-bit after. +func (b *testGGUFBuilder) writeLen(n uint64) { + if b.version <= 1 { + binary.Write(b.buf, binary.LittleEndian, uint32(n)) + return + } + binary.Write(b.buf, binary.LittleEndian, n) +} + func (b *testGGUFBuilder) writeString(s string) { - binary.Write(b.buf, binary.LittleEndian, uint64(len(s))) + b.writeLen(uint64(len(s))) b.buf.WriteString(s) } @@ -85,10 +109,10 @@ func (b *testGGUFBuilder) build() []byte { binary.Write(b.buf, binary.LittleEndian, b.version) // Write tensor count - binary.Write(b.buf, binary.LittleEndian, b.tensorCount) + b.writeLen(b.tensorCount) // Write KV count - binary.Write(b.buf, binary.LittleEndian, uint64(len(b.kvPairs))) + b.writeLen(uint64(len(b.kvPairs))) // Write KV pairs for _, kv := range b.kvPairs { @@ -97,6 +121,10 @@ func (b *testGGUFBuilder) build() []byte { // Write value type binary.Write(b.buf, binary.LittleEndian, kv.valueType) // Write value based on type + if raw, ok := kv.value.([]byte); ok { + b.buf.Write(raw) + continue + } switch kv.valueType { case ggufTypeString: b.writeString(kv.value.(string)) From fa6ea7b10cc0a0acc2d935cfcabee2d76ebc5704 Mon Sep 17 00:00:00 2001 From: Alex Goodman Date: Tue, 29 Sep 2026 10:25:30 -0400 Subject: [PATCH 23/40] test(debian): bound the bomb fixture by ratio, not a fixed 1MB (#5350) go1.27's compress/flate encodes 512MB of zeros to ~1MB (1052690 bytes) where go1.26 gives ~510KB, so the fixture sanity check tripped on newer toolchains before the bound was ever exercised. The decompression bound itself is fine (bounded read allocates ~300KB on both). Signed-off-by: Alex Goodman --- syft/pkg/cataloger/debian/parse_deb_archive_test.go | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/syft/pkg/cataloger/debian/parse_deb_archive_test.go b/syft/pkg/cataloger/debian/parse_deb_archive_test.go index 7afa0a455..ad8c00ddf 100644 --- a/syft/pkg/cataloger/debian/parse_deb_archive_test.go +++ b/syft/pkg/cataloger/debian/parse_deb_archive_test.go @@ -375,7 +375,7 @@ func Test_processDataTar_capsCopyrightFiles(t *testing.T) { func Test_decompressionStream_failsEarlyInsteadOfBuffering(t *testing.T) { const expanded = 512 * intFile.MB - // a few KB of input expanding to 512MB, so heap is what separates a streaming bound from a + // well under 1% of input expanding to 512MB, so heap is what separates a streaming bound from a // bound checked after the read completes var gz bytes.Buffer gw := gzip.NewWriter(&gz) @@ -386,7 +386,9 @@ func Test_decompressionStream_failsEarlyInsteadOfBuffering(t *testing.T) { remaining -= int64(n) } require.NoError(t, gw.Close()) - require.Less(t, gz.Len(), 1024*1024, "compressed payload should be tiny relative to what it expands to") + // a ratio rather than a fixed size: deflate output for zeros is toolchain dependent (about 510KB on + // go1.26, about 1MB on go1.27), and only the ratio matters to the test + require.Less(t, gz.Len(), expanded/100, "compressed payload should be tiny relative to what it expands to") // both arms use io.ReadAll on purpose. The stream itself never holds much whatever the cap says, // so copying to io.Discard measures nothing; what the cap actually protects is a consumer that From 236ab25393c9bb826fabb9aa7a6ca554088d7b3b Mon Sep 17 00:00:00 2001 From: Alex Goodman Date: Tue, 29 Sep 2026 11:19:17 -0400 Subject: [PATCH 24/40] test(snap): bound the kernel changelog bomb fixture by ratio, not a fixed 1MB (#5352) Same issue as #5350: go1.27's compress/flate encodes this 512MB fixture to just over 1MB, so the fixture sanity check tripped before the bound was exercised. The line bound itself is fine. Signed-off-by: Alex Goodman --- syft/pkg/cataloger/snap/parse_kernel_changelog_test.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/syft/pkg/cataloger/snap/parse_kernel_changelog_test.go b/syft/pkg/cataloger/snap/parse_kernel_changelog_test.go index 063161d19..a1806a822 100644 --- a/syft/pkg/cataloger/snap/parse_kernel_changelog_test.go +++ b/syft/pkg/cataloger/snap/parse_kernel_changelog_test.go @@ -330,7 +330,9 @@ func TestParseKernelChangelog_gzipBombStaysBounded(t *testing.T) { remaining -= n } require.NoError(t, gw.Close()) - require.Less(t, gz.Len(), 1024*1024, "compressed payload should be tiny relative to what it expands to") + // compress/flate output for this input varies by Go version (about 510KB on go1.26, 1MB on go1.27), so + // bound the fixture by ratio rather than a fixed size + require.Less(t, gz.Len(), expanded/100, "compressed payload should be tiny relative to what it expands to") // prove the fixture is a bomb: the same stream read without a line bound costs its full size unbounded := testutils.MeasureAlloc(t, func() { From c0072873c95e6a8d7af0a52bcfcdf9dcecc74c2b Mon Sep 17 00:00:00 2001 From: huuya Date: Wed, 30 Sep 2026 03:25:04 +0900 Subject: [PATCH 25/40] fix(golang): account for UPX's loader padding when unpacking Go binaries (#5347) UPX pads its output to a 4 byte boundary before it writes the loader stub, and l_lsize counts from that boundary. The block chain resumed at the end of the last PT_LOAD extent plus l_lsize, so whenever the compressed extents ended off a 4 byte boundary it landed 1 to 3 bytes early, read a misaligned b_info, and stopped. The padding between the segments was never filled in, the reconstruction ended at that first hole, and .go.buildinfo in the data segment went with it, so the binary contributed no packages. The loader skip now rounds up to 4 before adding l_lsize, which matches the funpad4 in UPX's own unpacker. The crafted fixture for the tail extents put its stub straight after the head extent, the layout the bug assumed, so it now pads the way UPX does. Signed-off-by: huuyafwww --- syft/pkg/cataloger/golang/upx.go | 5 +++-- syft/pkg/cataloger/golang/upx_security_test.go | 5 +++++ 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/syft/pkg/cataloger/golang/upx.go b/syft/pkg/cataloger/golang/upx.go index 9d3a4ae8c..4b80a26a7 100644 --- a/syft/pkg/cataloger/golang/upx.go +++ b/syft/pkg/cataloger/golang/upx.go @@ -502,13 +502,14 @@ type loaderSkip struct { } // past reports the offset the chain resumes at, and false once the jump has been spent or there is no -// stub to jump over. +// stub to jump over. UPX pads to a 4 byte boundary before it writes the stub, and l_lsize counts from that +// boundary rather than from where the last extent ended. func (l *loaderSkip) past(offset int64) (int64, bool) { if l.done || l.size == 0 { return offset, false } l.done = true - return offset + int64(l.size), true + return (offset+3)&^3 + int64(l.size), true } // decompressUPXBlocks walks the b_info chain, decoding each block into its place in the reconstructed diff --git a/syft/pkg/cataloger/golang/upx_security_test.go b/syft/pkg/cataloger/golang/upx_security_test.go index f4d4f105d..c7e3168c3 100644 --- a/syft/pkg/cataloger/golang/upx_security_test.go +++ b/syft/pkg/cataloger/golang/upx_security_test.go @@ -718,6 +718,9 @@ func TestDecompressUPX_TailExtentsArePlacedPastTheLoader(t *testing.T) { // front of it. Every other crafted fixture here has l_lsize == 0, so without this skipLoader was only // reachable through the Docker-backed fixture. // + // UPX pads to a 4 byte boundary before it writes the stub and l_lsize counts from there, so the head + // extent here deliberately ends off that boundary: its end plus l_lsize lands inside the stub. + // // note: block 1 here is not an ELF, so there are no PT_LOAD offsets and the tail lands sequentially. // The hole-filling path this feeds in production is covered by TestFirstHole and by the sparse // fixtures in upx_bounds_regression_test.go. @@ -727,6 +730,8 @@ func TestDecompressUPX_TailExtentsArePlacedPastTheLoader(t *testing.T) { data := buildUPXHeaderWithLoader(4096, 4096, loaderSize) data = append(data, blockFor(t, head)...) + require.NotZero(t, len(data)%4, "the head extent has to end off a 4 byte boundary for the padding to exist") + data = append(data, make([]byte, 4-len(data)%4)...) // the padding UPX writes in front of the stub data = append(data, make([]byte, 12)...) // end marker: closes the first run of extents data = append(data, make([]byte, loaderSize-12)...) // the loader stub the chain skips over data = append(data, blockFor(t, tail)...) From c60aabad37447894c229f3f48a718954a881aed9 Mon Sep 17 00:00:00 2001 From: Will Murphy Date: Tue, 29 Sep 2026 15:00:42 -0400 Subject: [PATCH 26/40] test(golang): make the UPX fixture exercise loader padding (#5354) UPX pads to a 4 byte boundary before it writes the loader stub. The image-small-upx build happened to need no padding, so the cataloger tests over it passed with or without the fix in #5347. - build the fixture with -X main.Version=1.0.11, which needs 2 bytes of padding, so the existing cataloger tests fail without the fix - add TestImageSmallUPXNeedsLoaderPadding, which fails if a change to the fixture's inputs means it no longer needs padding - run the crafted loader test on both sides of the boundary, so a skip that always rounds up is caught too Signed-off-by: Will Murphy --- syft/pkg/cataloger/golang/cataloger_test.go | 25 ++++---- .../testdata/image-small-upx/Dockerfile | 4 +- .../pkg/cataloger/golang/upx_security_test.go | 60 +++++++++++++------ syft/pkg/cataloger/golang/upx_test.go | 34 +++++++++++ 4 files changed, 92 insertions(+), 31 deletions(-) diff --git a/syft/pkg/cataloger/golang/cataloger_test.go b/syft/pkg/cataloger/golang/cataloger_test.go index 860171f70..4d504ff65 100644 --- a/syft/pkg/cataloger/golang/cataloger_test.go +++ b/syft/pkg/cataloger/golang/cataloger_test.go @@ -59,9 +59,10 @@ func Test_PackageCataloger_Binary(t *testing.T) { { name: "upx compressed binary", fixture: "image-small-upx", + // v1.0.11, not v1.0.0: see the fixture's Dockerfile wantErr: require.NoError, expectedPkgs: []string{ - "anchore.io/not/real @ v1.0.0 (/run-me)", + "anchore.io/not/real @ v1.0.11 (/run-me)", "github.com/andybalholm/brotli @ v1.1.1 (/run-me)", "github.com/dsnet/compress @ v0.0.2-0.20210315054119-f66993602bf5 (/run-me)", "github.com/golang/snappy @ v0.0.4 (/run-me)", @@ -75,17 +76,17 @@ func Test_PackageCataloger_Binary(t *testing.T) { "github.com/anchore/archiver/v3 @ v3.5.3-0.20241210171143-5b1d8d1c7c51 (/run-me)", }, expectedRels: []string{ - "github.com/andybalholm/brotli @ v1.1.1 (/run-me) [dependency-of] anchore.io/not/real @ v1.0.0 (/run-me)", - "github.com/dsnet/compress @ v0.0.2-0.20210315054119-f66993602bf5 (/run-me) [dependency-of] anchore.io/not/real @ v1.0.0 (/run-me)", - "github.com/golang/snappy @ v0.0.4 (/run-me) [dependency-of] anchore.io/not/real @ v1.0.0 (/run-me)", - "github.com/klauspost/compress @ v1.17.11 (/run-me) [dependency-of] anchore.io/not/real @ v1.0.0 (/run-me)", - "github.com/klauspost/pgzip @ v1.2.6 (/run-me) [dependency-of] anchore.io/not/real @ v1.0.0 (/run-me)", - "github.com/anchore/archiver/v3 @ v3.5.3-0.20241210171143-5b1d8d1c7c51 (/run-me) [dependency-of] anchore.io/not/real @ v1.0.0 (/run-me)", - "github.com/nwaples/rardecode @ v1.1.3 (/run-me) [dependency-of] anchore.io/not/real @ v1.0.0 (/run-me)", - "github.com/pierrec/lz4/v4 @ v4.1.21 (/run-me) [dependency-of] anchore.io/not/real @ v1.0.0 (/run-me)", - "github.com/ulikunitz/xz @ v0.5.12 (/run-me) [dependency-of] anchore.io/not/real @ v1.0.0 (/run-me)", - "github.com/xi2/xz @ v0.0.0-20171230120015-48954b6210f8 (/run-me) [dependency-of] anchore.io/not/real @ v1.0.0 (/run-me)", - "stdlib @ go1.23.2 (/run-me) [dependency-of] anchore.io/not/real @ v1.0.0 (/run-me)", + "github.com/andybalholm/brotli @ v1.1.1 (/run-me) [dependency-of] anchore.io/not/real @ v1.0.11 (/run-me)", + "github.com/dsnet/compress @ v0.0.2-0.20210315054119-f66993602bf5 (/run-me) [dependency-of] anchore.io/not/real @ v1.0.11 (/run-me)", + "github.com/golang/snappy @ v0.0.4 (/run-me) [dependency-of] anchore.io/not/real @ v1.0.11 (/run-me)", + "github.com/klauspost/compress @ v1.17.11 (/run-me) [dependency-of] anchore.io/not/real @ v1.0.11 (/run-me)", + "github.com/klauspost/pgzip @ v1.2.6 (/run-me) [dependency-of] anchore.io/not/real @ v1.0.11 (/run-me)", + "github.com/anchore/archiver/v3 @ v3.5.3-0.20241210171143-5b1d8d1c7c51 (/run-me) [dependency-of] anchore.io/not/real @ v1.0.11 (/run-me)", + "github.com/nwaples/rardecode @ v1.1.3 (/run-me) [dependency-of] anchore.io/not/real @ v1.0.11 (/run-me)", + "github.com/pierrec/lz4/v4 @ v4.1.21 (/run-me) [dependency-of] anchore.io/not/real @ v1.0.11 (/run-me)", + "github.com/ulikunitz/xz @ v0.5.12 (/run-me) [dependency-of] anchore.io/not/real @ v1.0.11 (/run-me)", + "github.com/xi2/xz @ v0.0.0-20171230120015-48954b6210f8 (/run-me) [dependency-of] anchore.io/not/real @ v1.0.11 (/run-me)", + "stdlib @ go1.23.2 (/run-me) [dependency-of] anchore.io/not/real @ v1.0.11 (/run-me)", }, }, { diff --git a/syft/pkg/cataloger/golang/testdata/image-small-upx/Dockerfile b/syft/pkg/cataloger/golang/testdata/image-small-upx/Dockerfile index bdd8901d7..7ee71b619 100644 --- a/syft/pkg/cataloger/golang/testdata/image-small-upx/Dockerfile +++ b/syft/pkg/cataloger/golang/testdata/image-small-upx/Dockerfile @@ -9,7 +9,9 @@ COPY go.mod go.sum ./ RUN go mod download COPY main.go main.go -RUN CGO_ENABLED=0 GOOS=linux go build -ldflags "-X main.Version=1.0.0" -o run-me . +# 1.0.11 is chosen so UPX has to pad before its loader stub (1.0.0 needed none, which hid a bug). +# TestImageSmallUPXNeedsLoaderPadding checks the padding is still there. +RUN CGO_ENABLED=0 GOOS=linux go build -ldflags "-X main.Version=1.0.11" -o run-me . RUN upx --best --lzma --exact run-me FROM scratch diff --git a/syft/pkg/cataloger/golang/upx_security_test.go b/syft/pkg/cataloger/golang/upx_security_test.go index c7e3168c3..dd703ec6d 100644 --- a/syft/pkg/cataloger/golang/upx_security_test.go +++ b/syft/pkg/cataloger/golang/upx_security_test.go @@ -718,33 +718,57 @@ func TestDecompressUPX_TailExtentsArePlacedPastTheLoader(t *testing.T) { // front of it. Every other crafted fixture here has l_lsize == 0, so without this skipLoader was only // reachable through the Docker-backed fixture. // - // UPX pads to a 4 byte boundary before it writes the stub and l_lsize counts from there, so the head - // extent here deliberately ends off that boundary: its end plus l_lsize lands inside the stub. + // UPX pads to a 4 byte boundary before it writes the stub and l_lsize counts from there, so each case + // ends the head extent the given number of bytes short of a boundary. The head is a stored block so its + // length alone decides where it ends; a compressed one ends wherever the encoder happens to stop. // // note: block 1 here is not an ELF, so there are no PT_LOAD offsets and the tail lands sequentially. // The hole-filling path this feeds in production is covered by TestFirstHole and by the sparse // fixtures in upx_bounds_regression_test.go. const loaderSize = 64 - head := bytes.Repeat([]byte("H"), 128) tail := bytes.Repeat([]byte("T"), 32) - data := buildUPXHeaderWithLoader(4096, 4096, loaderSize) - data = append(data, blockFor(t, head)...) - require.NotZero(t, len(data)%4, "the head extent has to end off a 4 byte boundary for the padding to exist") - data = append(data, make([]byte, 4-len(data)%4)...) // the padding UPX writes in front of the stub - data = append(data, make([]byte, 12)...) // end marker: closes the first run of extents - data = append(data, make([]byte, loaderSize-12)...) // the loader stub the chain skips over - data = append(data, blockFor(t, tail)...) - data = append(data, make([]byte, 12)...) // end marker for the tail run - data = padTo(data, 1024) + tests := []struct { + name string + padding int + }{ + {name: "no padding", padding: 0}, + {name: "1 byte of padding", padding: 1}, + {name: "2 bytes of padding", padding: 2}, + {name: "3 bytes of padding", padding: 3}, + } - out, err := unpack(t, data) - // 160 bytes of a declared 4096, short by construction like every other crafted fixture here - require.ErrorIs(t, err, errUPXPartial) - got := readAll(t, out) + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + data := buildUPXHeaderWithLoader(4096, 4096, loaderSize) - assert.Equal(t, head, got[:len(head)], "the head extent stays where it was placed") - assert.Contains(t, string(got), string(tail), "the extent behind the loader must be placed too") + // trim a 128 byte head so that it, plus its b_info, ends tt.padding bytes short of a boundary + headLen := 128 - (len(data)+12+128+tt.padding)%4 + head := bytes.Repeat([]byte("H"), headLen) + stored := make([]byte, 12) + binary.LittleEndian.PutUint32(stored[0:4], uint32(headLen)) // sz_unc + binary.LittleEndian.PutUint32(stored[4:8], uint32(headLen)) // sz_cpr, equal for a stored block + stored[8] = upxMethodStored + + data = append(data, stored...) + data = append(data, head...) + require.Equal(t, tt.padding, (4-len(data)%4)%4, "the head should end tt.padding bytes short of a boundary") + data = append(data, make([]byte, tt.padding)...) // the padding UPX writes in front of the stub + data = append(data, make([]byte, 12)...) // end marker: closes the first run of extents + data = append(data, make([]byte, loaderSize-12)...) // the loader stub the chain skips over + data = append(data, blockFor(t, tail)...) + data = append(data, make([]byte, 12)...) // end marker for the tail run + data = padTo(data, 1024) + + out, err := unpack(t, data) + // a few hundred bytes of a declared 4096, short by construction like every other crafted fixture here + require.ErrorIs(t, err, errUPXPartial) + got := readAll(t, out) + + assert.Equal(t, head, got[:len(head)], "the head extent stays where it was placed") + assert.Contains(t, string(got), string(tail), "the extent behind the loader must be placed too") + }) + } } func TestDecompressUPX_UnsupportedMethodIsNotReportable(t *testing.T) { diff --git a/syft/pkg/cataloger/golang/upx_test.go b/syft/pkg/cataloger/golang/upx_test.go index 2def6178a..05df518e2 100644 --- a/syft/pkg/cataloger/golang/upx_test.go +++ b/syft/pkg/cataloger/golang/upx_test.go @@ -2,10 +2,14 @@ package golang import ( "bytes" + "io" "testing" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + + stereoscopeFile "github.com/anchore/stereoscope/pkg/file" + "github.com/anchore/stereoscope/pkg/imagetest" ) // every case here is one the magic scan must reject, asserted on parseUPXInfo's errNotUPX result. @@ -104,3 +108,33 @@ func TestParseUPXInfo_ValidHeader(t *testing.T) { assert.Equal(t, uint8(22), info.format) assert.Equal(t, uint32(0x100000), info.originalSize) } + +// TestImageSmallUPXNeedsLoaderPadding checks the fixture rather than the code. UPX pads to a 4 byte +// boundary before its loader stub, and the cataloger tests over image-small-upx only catch an unpacker that +// ignores that padding if there is some. A build that needs none passes either way, which is how the bug +// went unnoticed. The case without padding is covered by TestDecompressUPX_TailExtentsArePlacedPastTheLoader. +func TestImageSmallUPXNeedsLoaderPadding(t *testing.T) { + img := imagetest.GetFixtureImage(t, "docker-archive", "image-small-upx") + rc, err := img.OpenPathFromSquash(stereoscopeFile.Path("/run-me")) + require.NoError(t, err) + data, err := io.ReadAll(rc) + require.NoError(t, rc.Close()) + require.NoError(t, err) + + r := bytes.NewReader(data) + info, err := parseUPXInfo(r, int64(len(data))) + require.NoError(t, err) + require.NotZero(t, info.loaderSize, "real UPX output carries a loader stub") + + // walk the first run of extents to the end marker in front of the stub + extentsEnd := info.firstBlockOff + for blockNum := 0; ; blockNum++ { + block, _, err := readChainBlock(r, extentsEnd, blockNum, info.inputLen) + require.NoError(t, err) + if block == nil { + break + } + extentsEnd = block.dataOffset + int64(block.compressedSize) + } + require.NotZero(t, extentsEnd%4, "the fixture no longer needs padding: change the version string in its Dockerfile") +} From 57bd6a5f9cfb11752acfd759a8d72ba56857e74b Mon Sep 17 00:00:00 2001 From: Alex Goodman Date: Tue, 29 Sep 2026 16:01:21 -0400 Subject: [PATCH 27/40] fix(cpe): bound CPE candidate generation for pathological package metadata (#5356) * fix(cpe): cap sub-selection prefixes to bound candidate cross-product a name with thousands of `-`/`_` separators produced one vendor and product candidate per segment, and the vendor x product cross-product made CPE generation roughly cubic in time and memory (8KB name: ~250s, 4.8GB RSS). `generateSubSelections` now stops after 10 prefixes. The full name is still a candidate, so real package names are unaffected. Signed-off-by: Alex Goodman * fix(cpe): bound candidate count and field length before the cross-product the prefix cap only covered one way to inflate the vendor x product cross-product in `FromPackageAttributes`. Java groupIDs add a vendor (and, with no artifactID, a product) per dot segment, and .NET executables add one of each per file, so a 1000-segment `Bundle-SymbolicName` still produced ~1M CPEs and several GB. - vendor and product candidates are each capped at 64 (shortest first, so truncation is deterministic) - names, versions, and candidates over 512 bytes are skipped, since every byte is paid once per CPE in key building and regex validation real packages produce around a dozen candidates with names well under 100 bytes, so output for real inputs is unchanged. Also corrects the earlier comment: prefixes only become vendor candidates and the cost was quadratic, not cubic. Signed-off-by: Alex Goodman --------- Signed-off-by: Alex Goodman --- .../internal/cpegenerate/generate.go | 48 +++++++++++++-- .../internal/cpegenerate/generate_test.go | 61 +++++++++++++++++++ 2 files changed, 105 insertions(+), 4 deletions(-) diff --git a/syft/pkg/cataloger/internal/cpegenerate/generate.go b/syft/pkg/cataloger/internal/cpegenerate/generate.go index 097cf4e32..4495e17c1 100644 --- a/syft/pkg/cataloger/internal/cpegenerate/generate.go +++ b/syft/pkg/cataloger/internal/cpegenerate/generate.go @@ -3,10 +3,12 @@ package cpegenerate import ( "bufio" "bytes" + "cmp" _ "embed" "encoding/json" "fmt" "regexp" + "slices" "sort" "strings" "sync" @@ -145,8 +147,14 @@ func FromPackageAttributes(p pkg.Package) []cpe.CPE { return nil } - vendors := candidateVendors(p) - products := candidateProducts(p) + // real CPE fields are short, and every byte of a field is paid once per generated CPE (key building and regex + // validation), so a hostile name or version is not worth generating candidates for at all. + if len(p.Name) > maxCandidateFieldLength || len(p.Version) > maxCandidateFieldLength { + return nil + } + + vendors := boundCandidates(candidateVendors(p)) + products := boundCandidates(candidateProducts(p)) targetSWs := candidateTargetSw(p) if len(products) == 0 { return nil @@ -183,6 +191,32 @@ func FromPackageAttributes(p pkg.Package) []cpe.CPE { return result } +// maxCandidateFieldLength bounds the name, version, and each vendor and product candidate that reaches the +// cross-product in FromPackageAttributes. Real values are well under this. +const maxCandidateFieldLength = 512 + +// maxCandidates bounds how many vendor and product candidates each feed the cross-product in FromPackageAttributes, +// so a single package yields at most maxCandidates² CPEs. Some sources add a candidate per input element with no bound +// of their own (e.g. one vendor per java groupID segment, one product per .NET executable), and real packages produce +// a dozen or so candidates at most. +const maxCandidates = 64 + +// boundCandidates drops candidates longer than maxCandidateFieldLength and keeps at most maxCandidates of the rest. +// Candidates are ordered shortest first (then lexically) so that truncation is deterministic and keeps the short, +// generic values that tend to be real vendors and products. +func boundCandidates(candidates []string) []string { + candidates = slices.DeleteFunc(candidates, func(c string) bool { + return len(c) > maxCandidateFieldLength + }) + if len(candidates) <= maxCandidates { + return candidates + } + slices.SortFunc(candidates, func(a, b string) int { + return cmp.Or(cmp.Compare(len(a), len(b)), strings.Compare(a, b)) + }) + return candidates[:maxCandidates] +} + func candidateTargetSw(p pkg.Package) []string { switch p.Type { case pkg.WordpressPluginPkg: @@ -359,13 +393,19 @@ func addAllSubSelections(fields fieldCandidateSet) { } } +// maxSubSelections bounds how many prefixes generateSubSelections returns. Each prefix becomes a vendor candidate, so +// without a bound a name with N separators yields O(N) vendors of O(N) length each (quadratic in time and memory). +// Real package names rarely exceed a handful of segments. +const maxSubSelections = 10 + // generateSubSelections attempts to split a field by hyphens and underscores and return a list of sensible sub-selections -// that can be used as product or vendor candidates. E.g. jenkins-ci-tools -> [jenkins-ci-tools, jenkins-ci, jenkins]. +// that can be used as product or vendor candidates. E.g. jenkins-ci-tools -> [jenkins, jenkins-ci, jenkins-ci-tools]. +// At most maxSubSelections prefixes are returned, shortest first. func generateSubSelections(field string) (results []string) { scanner := bufio.NewScanner(strings.NewReader(field)) scanner.Split(scanByHyphenOrUnderscore) var lastToken uint8 - for scanner.Scan() { + for len(results) < maxSubSelections && scanner.Scan() { rawCandidate := scanner.Text() if len(rawCandidate) == 0 { break diff --git a/syft/pkg/cataloger/internal/cpegenerate/generate_test.go b/syft/pkg/cataloger/internal/cpegenerate/generate_test.go index d0ace066c..cdf32d9b7 100644 --- a/syft/pkg/cataloger/internal/cpegenerate/generate_test.go +++ b/syft/pkg/cataloger/internal/cpegenerate/generate_test.go @@ -1121,6 +1121,11 @@ func Test_generateSubSelections(t *testing.T) { field: "_", expected: nil, }, + { + // capped at maxSubSelections, keeping the shortest prefixes (the full name is not returned) + field: "a_b_c_d_e_f_g_h_i_j_k_l", + expected: []string{"a", "a_b", "a_b_c", "a_b_c_d", "a_b_c_d_e", "a_b_c_d_e_f", "a_b_c_d_e_f_g", "a_b_c_d_e_f_g_h", "a_b_c_d_e_f_g_h_i", "a_b_c_d_e_f_g_h_i_j"}, + }, } for _, test := range tests { t.Run(test.field, func(t *testing.T) { @@ -1265,3 +1270,59 @@ func TestAddBinaryPackageDigitVariations(t *testing.T) { }) } } + +func TestFromPackageAttributes_boundsSubSelectionBlowup(t *testing.T) { + // a name with many separators used to produce a vendor candidate per segment, each O(name length), which was + // quadratic in time and memory. + name := strings.Repeat("a_", 200) + "a" + p := pkg.Package{Name: name, Version: "1.0", Type: pkg.PythonPkg, Language: pkg.Python} + + cpes := FromPackageAttributes(p) + assert.NotEmpty(t, cpes) + assert.LessOrEqual(t, len(cpes), 1000) +} + +func TestFromPackageAttributes_boundsCandidateCrossProduct(t *testing.T) { + // every groupID segment becomes a vendor, and with no artifactID every segment past the second also becomes a + // product, so the vendor x product cross-product is quadratic in segment count without a bound. + var segments []string + for i := range 1000 { + segments = append(segments, fmt.Sprintf("s%d", i)) + } + p := pkg.Package{ + Name: "thing", + Version: "1.0", + Type: pkg.JavaPkg, + Language: pkg.Java, + Metadata: pkg.JavaArchive{ + Manifest: &pkg.JavaManifest{ + Main: pkg.KeyValues{{Key: "Bundle-SymbolicName", Value: "com.x." + strings.Join(segments, ".")}}, + }, + }, + } + + cpes := FromPackageAttributes(p) + assert.NotEmpty(t, cpes) + assert.LessOrEqual(t, len(cpes), maxCandidates*maxCandidates) +} + +func TestFromPackageAttributes_skipsOversizedFields(t *testing.T) { + long := strings.Repeat("a", maxCandidateFieldLength+1) + + assert.Empty(t, FromPackageAttributes(pkg.Package{Name: long, Version: "1.0", Type: pkg.PythonPkg})) + assert.Empty(t, FromPackageAttributes(pkg.Package{Name: "thing", Version: long, Type: pkg.PythonPkg})) + assert.NotEmpty(t, FromPackageAttributes(pkg.Package{Name: "thing", Version: "1.0", Type: pkg.PythonPkg})) +} + +func Test_boundCandidates(t *testing.T) { + long := strings.Repeat("a", maxCandidateFieldLength+1) + assert.Equal(t, []string{"b", "a"}, boundCandidates([]string{"b", long, "a"}), "under the cap order is untouched") + + var many []string + for i := range maxCandidates + 10 { + many = append(many, strings.Repeat("x", maxCandidates+10-i)) + } + got := boundCandidates(many) + assert.Len(t, got, maxCandidates) + assert.Equal(t, "x", got[0], "shortest candidates are kept") +} From d0e1e7be05f3f182a2306c87fd1511333b558c2c Mon Sep 17 00:00:00 2001 From: Alex Goodman Date: Tue, 29 Sep 2026 16:26:42 -0400 Subject: [PATCH 28/40] Harden cataloger parsers against truncated input (do not panic) (#5355) * fix(javascript): don't panic on a yarn v1 lockfile line missing its value A v1 `yarn.lock` dependency line without a version constraint, or a `version`/`resolved`/`integrity` line without a value, indexed past the end of the split line. Such dependencies are now skipped and empty fields left empty. Signed-off-by: Alex Goodman * fix(alpine): don't panic on an apk db ACL or checksum line before its entry An `M:`, `a:` or `Z:` line that comes before any directory or file entry in `lib/apk/db/installed` indexed `files[-1]`. Such a line is now skipped. Signed-off-by: Alex Goodman * fix(erlang): don't panic on a truncated rebar.lock or .app file The term parser read past the end of the input when a file ended mid-term: whitespace only, an unclosed list, a lone `<`, or a comment ending in `\r`. These now return an "unexpected end of input" error. Signed-off-by: Alex Goodman * fix(lua): don't panic on a truncated .rockspec The rockspec parser read past the end of the input when a file ended mid-statement: a lone `[`, an unclosed `[[...]]` value or `[...]` index, or a `local` assignment cut off before its `=`, value, or line end. These now return an "unexpected end of input" error. Signed-off-by: Alex Goodman * fix(swift): don't panic on a Podfile.lock pod without a version Signed-off-by: Alex Goodman * fix(gentoo): don't panic on a short obj line in portage CONTENTS Signed-off-by: Alex Goodman * fix(spdx): don't panic on an opam checksum without an algorithm The opam cataloger stores checksums verbatim, so a malformed one reached the SPDX encoder and panicked there, outside the cataloger's recover. Checksums without an `=` are now skipped. Signed-off-by: Alex Goodman * refactor: use strings.Cut where a split was only indexed after a Contains check Also guards `windows.FromPosix` against a path with no separator. Signed-off-by: Alex Goodman * test: guard untyped package metadata fields - new `any`-typed metadata fields must be allowlisted with a reason - allowlisted untyped fields are pushed through `SetID` and every encoder with non-finite, deeply nested and large values Signed-off-by: Alex Goodman * test: fuzz every generic cataloger parser One fuzz target over every glob-selected parser in the capabilities data, run through its cataloger task. Each input must not panic, must finish in time and must keep allocations proportional to its size. Seeds come from each parser's testdata plus the string literals in its package. Signed-off-by: Alex Goodman * test: scan malformed inputs under memory and time limits A directory of small malformed files (deep nesting, oversized declared counts, truncation, type mismatches, non-finite floats) is scanned by the CLI with a memory limit and an RSS ceiling. The scan must exit 0 and write an SBOM. Signed-off-by: Alex Goodman * test: stop a fuzz input once its live heap passes the allocation limit The allocation check only ran after the parser returned, so an input on its way to exhausting memory ran the machine into swap first. The harness now polls the live heap while the parser runs and panics past the same limit, which the fuzz engine saves as a crasher. Signed-off-by: Alex Goodman * fix: bound nesting depth and error context in the erlang and lua parsers - a deeply nested rebar.lock or .rockspec overflowed the goroutine stack, which is fatal rather than a recoverable panic, so one file could take down the whole scan. Both parsers now stop at 1000 levels with an error. - parse errors echoed whole input lines into the SBOM as unknowns, and lua appended another copy at every nesting level. `PrintError` now clips to a window around the error column, and lua decorates once at the top. - the Podfile.lock error no longer echoes the pod entry. Signed-off-by: Alex Goodman * test: close ways the malformed-input checks could pass quietly - a ctx-aware parser hitting the fuzz timeout now fails the input instead of racing the hard deadline - the heap watchdog measures growth from a post-GC baseline rather than an absolute process-wide number - a saved fuzz input naming a target that no longer exists fails instead of being skipped - the CLI malformed-input scan fails on a recovered parser panic, not only on a non-zero exit Signed-off-by: Alex Goodman --------- Signed-off-by: Alex Goodman --- .../packagemetadata/untyped_fields_test.go | 74 ++++ .../common/spdxhelpers/to_format_model.go | 9 +- .../spdxhelpers/to_format_model_test.go | 14 + syft/format/untyped_metadata_test.go | 94 ++++ syft/internal/parsing/basic_parsing.go | 31 +- syft/internal/windows/path.go | 4 + syft/linux/identify_release.go | 4 +- syft/pkg/cataloger/alpine/parse_apk_db.go | 12 + .../pkg/cataloger/alpine/parse_apk_db_test.go | 26 ++ .../manager/internal/config/application.go | 8 +- syft/pkg/cataloger/erlang/erlang_parser.go | 31 +- .../cataloger/erlang/erlang_parser_test.go | 33 ++ syft/pkg/cataloger/gentoo/cataloger_test.go | 15 + .../gentoo/parse_portage_contents.go | 3 +- .../var/db/pkg/app-misc/foo-1.0/CONTENTS | 4 + .../cataloger/javascript/parse_yarn_lock.go | 47 +- .../javascript/parse_yarn_lock_test.go | 34 ++ syft/pkg/cataloger/lua/rockspec_parser.go | 118 ++++-- .../pkg/cataloger/lua/rockspec_parser_test.go | 33 ++ syft/pkg/cataloger/parsers_fuzz_test.go | 400 ++++++++++++++++++ .../cataloger/python/parse_requirements.go | 10 +- syft/pkg/cataloger/snap/parse_base_dpkg.go | 7 +- .../cataloger/snap/parse_system_manifest.go | 7 +- .../pkg/cataloger/swift/parse_podfile_lock.go | 9 +- .../swift/parse_podfile_lock_test.go | 17 + .../fuzz/FuzzGenericParsers/e6edf122b3cfae05 | 3 + test/cli/malformed_input_test.go | 78 ++++ test/cli/testdata/.gitignore | 1 + .../malformed/gguf-declared-count/model.gguf | Bin 0 -> 108 bytes .../malformed/gguf-non-finite/model.gguf | Bin 0 -> 174 bytes .../malformed/gguf-truncated/model.gguf | Bin 0 -> 101 bytes .../malformed/gguf-type-mismatch/model.gguf | Bin 0 -> 97 bytes .../malformed/json-deep-nesting/package.json | 1 + .../malformed/list-for-scalar/Cargo.lock | 6 + .../malformed/list-for-scalar/package.json | 1 + .../model.safetensors | Bin 0 -> 10 bytes .../malformed/truncated-archive/a-1.0.jar | Bin 0 -> 388 bytes .../truncated-lockfile/package-lock.json | 14 + .../yaml-deep-nesting/pnpm-lock.yaml | 3 + .../malformed/yaml-merge-keys/pnpm-lock.yaml | 8 + 40 files changed, 1060 insertions(+), 99 deletions(-) create mode 100644 internal/packagemetadata/untyped_fields_test.go create mode 100644 syft/format/untyped_metadata_test.go create mode 100644 syft/pkg/cataloger/gentoo/testdata/short-obj-line/var/db/pkg/app-misc/foo-1.0/CONTENTS create mode 100644 syft/pkg/cataloger/parsers_fuzz_test.go create mode 100644 syft/pkg/cataloger/testdata/fuzz/FuzzGenericParsers/e6edf122b3cfae05 create mode 100644 test/cli/malformed_input_test.go create mode 100644 test/cli/testdata/malformed/gguf-declared-count/model.gguf create mode 100644 test/cli/testdata/malformed/gguf-non-finite/model.gguf create mode 100644 test/cli/testdata/malformed/gguf-truncated/model.gguf create mode 100644 test/cli/testdata/malformed/gguf-type-mismatch/model.gguf create mode 100644 test/cli/testdata/malformed/json-deep-nesting/package.json create mode 100644 test/cli/testdata/malformed/list-for-scalar/Cargo.lock create mode 100644 test/cli/testdata/malformed/list-for-scalar/package.json create mode 100644 test/cli/testdata/malformed/safetensors-declared-length/model.safetensors create mode 100644 test/cli/testdata/malformed/truncated-archive/a-1.0.jar create mode 100644 test/cli/testdata/malformed/truncated-lockfile/package-lock.json create mode 100644 test/cli/testdata/malformed/yaml-deep-nesting/pnpm-lock.yaml create mode 100644 test/cli/testdata/malformed/yaml-merge-keys/pnpm-lock.yaml diff --git a/internal/packagemetadata/untyped_fields_test.go b/internal/packagemetadata/untyped_fields_test.go new file mode 100644 index 000000000..c4bde3718 --- /dev/null +++ b/internal/packagemetadata/untyped_fields_test.go @@ -0,0 +1,74 @@ +package packagemetadata + +import ( + "reflect" + "strings" + "testing" +) + +// untypedFieldAllowlist holds package metadata fields that are allowed to carry untyped (interface) values. +// Values in these fields come straight from whatever a parser decoded, so they are not shaped by syft +// and can hold anything an encoder or hasher chokes on. Every entry is part of existing SBOM output and +// cannot be retyped without breaking it; new entries need a reason and a review. +var untypedFieldAllowlist = map[string]string{ + "GGUFFileHeader.RemainingKeyValues": "raw GGUF header key-values; retyping changes the header output and metadataHash", +} + +// TestNoUntypedMetadataFields fails when a package metadata type gains a field typed any, []any, +// map[string]any (or any other interface) that is not on the allowlist. +func TestNoUntypedMetadataFields(t *testing.T) { + var hits []string + for _, ty := range AllTypes() { + rt := reflect.TypeOf(ty) + collectUntyped(rt, rt.Name(), map[reflect.Type]bool{}, &hits) + } + + seen := map[string]bool{} + for _, h := range hits { + seen[h] = true + if _, ok := untypedFieldAllowlist[h]; !ok { + t.Errorf("untyped field %s: use a concrete type, or add it to untypedFieldAllowlist with a reason", h) + } + } + for k := range untypedFieldAllowlist { + if !seen[k] { + t.Errorf("stale untypedFieldAllowlist entry %s: field no longer exists or is now typed", k) + } + } +} + +// collectUntyped records "Struct.Field" for every field whose type reaches an interface, descending +// through pointers, slices, arrays, maps and nested syft-owned structs. +func collectUntyped(rt reflect.Type, path string, visited map[reflect.Type]bool, hits *[]string) { + switch rt.Kind() { + case reflect.Interface: + *hits = append(*hits, path) + case reflect.Pointer, reflect.Slice, reflect.Array: + collectUntyped(rt.Elem(), path, visited, hits) + case reflect.Map: + collectUntyped(rt.Key(), path, visited, hits) + collectUntyped(rt.Elem(), path, visited, hits) + case reflect.Struct: + if visited[rt] { + return + } + if rt.Name() != "" && !strings.HasPrefix(rt.PkgPath(), "github.com/anchore/syft/") { + // a third-party type in the model carries whatever shape the library gives it (the same leak as an + // interface field), so it counts as untyped. The standard library is left alone. + if strings.Contains(strings.Split(rt.PkgPath(), "/")[0], ".") { + *hits = append(*hits, path) + } + return + } + visited[rt] = true + for i := 0; i < rt.NumField(); i++ { + f := rt.Field(i) + // unexported embedded structs still promote their exported fields to the encoders and the hasher. A + // field is only out of reach when it is neither encoded nor hashed by SetID. + if (!f.IsExported() && !f.Anonymous) || (f.Tag.Get("json") == "-" && f.Tag.Get("hash") == "ignore") { + continue + } + collectUntyped(f.Type, rt.Name()+"."+f.Name, visited, hits) + } + } +} diff --git a/syft/format/common/spdxhelpers/to_format_model.go b/syft/format/common/spdxhelpers/to_format_model.go index 12e67ec5c..426f9e10c 100644 --- a/syft/format/common/spdxhelpers/to_format_model.go +++ b/syft/format/common/spdxhelpers/to_format_model.go @@ -581,10 +581,13 @@ func toPackageChecksums(p pkg.Package) ([]spdx.Checksum, bool) { }) case pkg.OpamPackage: for _, checksum := range meta.Checksums { - parts := strings.Split(checksum, "=") + algorithm, value, ok := strings.Cut(checksum, "=") + if !ok { + continue + } checksums = append(checksums, spdx.Checksum{ - Algorithm: spdx.ChecksumAlgorithm(strings.ToUpper(parts[0])), - Value: parts[1], + Algorithm: spdx.ChecksumAlgorithm(strings.ToUpper(algorithm)), + Value: value, }) } } diff --git a/syft/format/common/spdxhelpers/to_format_model_test.go b/syft/format/common/spdxhelpers/to_format_model_test.go index 57ca27db2..302fe81ae 100644 --- a/syft/format/common/spdxhelpers/to_format_model_test.go +++ b/syft/format/common/spdxhelpers/to_format_model_test.go @@ -516,6 +516,20 @@ func Test_toPackageChecksums(t *testing.T) { }, filesAnalyzed: false, }, + { + name: "Opam Package with a checksum missing its algorithm separator", + pkg: pkg.Package{ + Name: "test", + Version: "1.0.0", + Metadata: pkg.OpamPackage{ + Checksums: []string{"f5f1c0b4", "md5=abc"}, + }, + }, + expected: []spdx.Checksum{ + {Algorithm: "MD5", Value: "abc"}, + }, + filesAnalyzed: false, + }, { name: "Package with no metadata type", pkg: pkg.Package{ diff --git a/syft/format/untyped_metadata_test.go b/syft/format/untyped_metadata_test.go new file mode 100644 index 000000000..a1a28b9d0 --- /dev/null +++ b/syft/format/untyped_metadata_test.go @@ -0,0 +1,94 @@ +package format + +import ( + "bytes" + "fmt" + "math" + "testing" + + "github.com/anchore/syft/syft/format/syftjson" + "github.com/anchore/syft/syft/pkg" + "github.com/anchore/syft/syft/sbom" + "github.com/anchore/syft/syft/source" +) + +// TestEncodersHandleUntypedMetadataValues pushes awkward values through the untyped metadata fields +// allowlisted in internal/packagemetadata (see untypedFieldAllowlist) and asserts that SetID and every +// encoder neither panic nor fail. A new allowlist entry should get a case here too. +func TestEncodersHandleUntypedMetadataValues(t *testing.T) { + cases := []struct { + name string + value any + // encoding/json rejects non-finite floats, so syft-json returns an error for them. Producers + // have to replace these before they reach the model (the GGUF cataloger does); the case stays + // here so the expectation flips visibly if syft-json ever learns to encode them. + syftJSONRejects bool + }{ + {name: "NaN", value: math.NaN(), syftJSONRejects: true}, + {name: "+Inf float32", value: float32(math.Inf(1)), syftJSONRejects: true}, + {name: "-Inf", value: math.Inf(-1), syftJSONRejects: true}, + {name: "NaN in array", value: []any{1.0, math.NaN()}, syftJSONRejects: true}, + {name: "invalid utf-8", value: "\xff\xfe"}, + {name: "NUL and control characters", value: "a\x00b\x1b"}, + {name: "max uint64", value: uint64(math.MaxUint64)}, + {name: "deep array", value: nested(2000, func(v any) any { return []any{v} })}, + {name: "deep map", value: nested(2000, func(v any) any { return map[string]any{"k": v} })}, + {name: "large array", value: make([]any, 100_000)}, + } + + for _, tt := range cases { + t.Run(tt.name, func(t *testing.T) { + p := pkg.Package{ + Name: "model", + Version: "1.0", + Type: pkg.ModelPkg, + Metadata: pkg.GGUFFileHeader{ + RemainingKeyValues: map[string]any{"general.value": tt.value}, + }, + } + if !noPanic(t, "SetID", p.SetID) { + return + } + + s := sbom.SBOM{ + Artifacts: sbom.Artifacts{Packages: pkg.NewCollection(p)}, + Source: source.Description{Metadata: source.DirectoryMetadata{Path: "/models"}}, + } + for _, enc := range Encoders() { + id := fmt.Sprintf("%s@%s", enc.ID(), enc.Version()) + var buf bytes.Buffer + noPanic(t, id, func() { + err := enc.Encode(&buf, s) + switch { + case tt.syftJSONRejects && enc.ID() == syftjson.ID: + if err == nil { + t.Errorf("%s: expected an encode error for a non-finite float", id) + } + case err != nil: + t.Errorf("%s: encode failed: %v", id, err) + } + }) + } + }) + } +} + +func nested(depth int, wrap func(any) any) any { + var v any = "leaf" + for range depth { + v = wrap(v) + } + return v +} + +func noPanic(t *testing.T, what string, fn func()) (ok bool) { + t.Helper() + defer func() { + if r := recover(); r != nil { + t.Errorf("%s: panic: %v", what, r) + ok = false + } + }() + fn() + return true +} diff --git a/syft/internal/parsing/basic_parsing.go b/syft/internal/parsing/basic_parsing.go index 27a1175c8..7cc794f86 100644 --- a/syft/internal/parsing/basic_parsing.go +++ b/syft/internal/parsing/basic_parsing.go @@ -22,6 +22,15 @@ func SkipWhitespace(data []byte, i *int) { } } +// MaxDepth bounds nesting in the recursive hand-written parsers. Real files nest a handful of levels; without a +// bound, deep input overflows the goroutine stack, which is fatal rather than a recoverable panic. +const MaxDepth = 1000 + +// errorContext is how many bytes of each echoed line PrintError keeps around the error column, so an error in a +// huge single-line input stays small (and does not copy the file into the SBOM as an unknown). +const errorContext = 80 + +// PrintError describes where in data offset i is, echoing up to two lines of context clipped around the column. func PrintError(data []byte, i int) string { line := 1 char := 1 @@ -57,15 +66,23 @@ func PrintError(data []byte, i int) string { sep := ": " - lines := "" - if len(prev) > 1 { - lines += fmt.Sprintf("%s%s%s\n", l1, sep, prev[len(prev)-2]) - } - if len(prev) > 0 { - lines += fmt.Sprintf("%s%s%s\n", l2, sep, prev[len(prev)-1]) + start := max(0, char-1-errorContext/2) + clip := func(s string) string { + if start >= len(s) { + return "" + } + return s[start:min(len(s), start+errorContext)] } - pointer := strings.Repeat(" ", len(l2)+len(sep)+char-1) + "^" + lines := "" + if len(prev) > 1 { + lines += fmt.Sprintf("%s%s%s\n", l1, sep, clip(prev[len(prev)-2])) + } + if len(prev) > 0 { + lines += fmt.Sprintf("%s%s%s\n", l2, sep, clip(prev[len(prev)-1])) + } + + pointer := strings.Repeat(" ", len(l2)+len(sep)+char-1-start) + "^" return fmt.Sprintf("line: %v, char: %v\n%s%s", line, char, lines, pointer) } diff --git a/syft/internal/windows/path.go b/syft/internal/windows/path.go index 409917103..c091b89fd 100644 --- a/syft/internal/windows/path.go +++ b/syft/internal/windows/path.go @@ -36,6 +36,10 @@ func FromPosix(posixPath string) (windowsPath string) { // decode the volume (e.g. /c/ --> C:\\) - There should always be a volume name. // The volume may be a UNC path (e.g. /\\localhost\C$\ --> \\localhost\C$\) pathFields := strings.Split(posixPath, "/") + if len(pathFields) < 2 { + // no separator means no volume to decode + return filepath.Clean(posixPath) + } rootPath := strings.ToUpper(pathFields[1]) volumeName := AppendRootTerminator(rootPath) diff --git a/syft/linux/identify_release.go b/syft/linux/identify_release.go index 5025cdde5..de45c2db7 100644 --- a/syft/linux/identify_release.go +++ b/syft/linux/identify_release.go @@ -156,8 +156,8 @@ var busyboxVersionMatcher = regexp.MustCompile(`BusyBox v[\d.]+`) func parseBusyBox(contents string) (*Release, error) { matches := busyboxVersionMatcher.FindAllString(contents, -1) for _, match := range matches { - parts := strings.Split(match, " ") - version := strings.ReplaceAll(parts[1], "v", "") + _, version, _ := strings.Cut(match, " ") + version = strings.ReplaceAll(version, "v", "") return simpleRelease(match, "busybox", version, ""), nil } diff --git a/syft/pkg/cataloger/alpine/parse_apk_db.go b/syft/pkg/cataloger/alpine/parse_apk_db.go index e661797a1..bf721c68d 100644 --- a/syft/pkg/cataloger/alpine/parse_apk_db.go +++ b/syft/pkg/cataloger/alpine/parse_apk_db.go @@ -270,6 +270,10 @@ func (f apkField) apply(p *parsedData, ctx *apkFileParsingContext) { ctx.indexOfLatestDirectory = len(ctx.files) - 1 case "M": i := ctx.indexOfLatestDirectory + if i < 0 { + log.Debugf("APK field %q appears before any directory entry", f.name) + return + } latest := ctx.files[i] var ok bool @@ -296,6 +300,10 @@ func (f apkField) apply(p *parsedData, ctx *apkFileParsingContext) { ctx.indexOfLatestRegularFile = len(ctx.files) - 1 case "a": i := ctx.indexOfLatestRegularFile + if i < 0 { + log.Debugf("APK field %q appears before any file entry", f.name) + return + } latest := ctx.files[i] var ok bool @@ -309,6 +317,10 @@ func (f apkField) apply(p *parsedData, ctx *apkFileParsingContext) { ctx.files[i] = latest case "Z": i := ctx.indexOfLatestRegularFile + if i < 0 { + log.Debugf("APK field %q appears before any file entry", f.name) + return + } latest := ctx.files[i] latest.Digest = processChecksum(f.value) diff --git a/syft/pkg/cataloger/alpine/parse_apk_db_test.go b/syft/pkg/cataloger/alpine/parse_apk_db_test.go index ee146d723..6ff453af7 100644 --- a/syft/pkg/cataloger/alpine/parse_apk_db_test.go +++ b/syft/pkg/cataloger/alpine/parse_apk_db_test.go @@ -869,3 +869,29 @@ https://foo.them.org/alpine/v3.14/community`, }) } } + +func TestParseApkDB_attributeBeforeItsEntry(t *testing.T) { + // ACL (M:, a:) and checksum (Z:) lines describe the most recent directory or file entry, and used to index + // files[-1] when no entry had been seen yet + tests := []struct { + name string + line string + }{ + {name: "directory ACL before any directory", line: "M:0:0:755"}, + {name: "file ACL before any file", line: "a:0:0:755"}, + {name: "checksum before any file", line: "Z:Q1Kz6TXnPVJeDQUfO0IJVxD3HnYmU="}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + contents := strings.Join([]string{"P:foo", "V:1.0-r0", tt.line, ""}, "\n") + reader := file.NewLocationReadCloser(file.NewLocation("installed"), io.NopCloser(strings.NewReader(contents))) + + pkgs, _, err := parseApkDB(context.Background(), nil, new(generic.Environment), reader) + + require.NoError(t, err) + require.Len(t, pkgs, 1) + assert.Equal(t, "foo", pkgs[0].Name) + assert.Empty(t, pkgs[0].Metadata.(pkg.ApkDBEntry).Files, "an orphaned attribute line should not invent a file") + }) + } +} diff --git a/syft/pkg/cataloger/binary/internal/manager/internal/config/application.go b/syft/pkg/cataloger/binary/internal/manager/internal/config/application.go index d758184dd..fe0bd9a4b 100644 --- a/syft/pkg/cataloger/binary/internal/manager/internal/config/application.go +++ b/syft/pkg/cataloger/binary/internal/manager/internal/config/application.go @@ -102,10 +102,10 @@ func (c Application) Validate() error { } func (c Application) GetBinaryFromImage(name, version string) *BinaryFromImage { - if strings.Contains(name, "@") && version == "" { - parts := strings.Split(name, "@") - name = parts[0] - version = parts[1] + if version == "" { + if n, v, ok := strings.Cut(name, "@"); ok { + name, version = n, v + } } for _, entry := range c.FromImages { if entry.Name() == name && entry.Version == version { diff --git a/syft/pkg/cataloger/erlang/erlang_parser.go b/syft/pkg/cataloger/erlang/erlang_parser.go index 819c3bb41..07079d9a8 100644 --- a/syft/pkg/cataloger/erlang/erlang_parser.go +++ b/syft/pkg/cataloger/erlang/erlang_parser.go @@ -15,6 +15,10 @@ type erlangNode struct { var errSkipComments = errors.New("") +var errUnexpectedEnd = errors.New("unexpected end of input") + +var errTooDeep = fmt.Errorf("nesting deeper than %d levels", parsing.MaxDepth) + func (e erlangNode) Slice() []erlangNode { out, ok := e.value.([]erlangNode) if ok { @@ -79,7 +83,7 @@ func parseErlang(reader io.Reader) (erlangNode, error) { } func parseErlangBlock(data []byte, i *int) (erlangNode, error) { - block, err := parseErlangNode(data, i) + block, err := parseErlangNode(data, i, 0) if err != nil { return node(nil), err } @@ -89,13 +93,19 @@ func parseErlangBlock(data []byte, i *int) (erlangNode, error) { return block, nil } -func parseErlangNode(data []byte, i *int) (erlangNode, error) { +func parseErlangNode(data []byte, i *int, depth int) (erlangNode, error) { parsing.SkipWhitespace(data, i) + if *i >= len(data) { + return node(nil), errUnexpectedEnd + } c := data[*i] switch c { case '[', '{': offset := *i + 1 parsing.SkipWhitespace(data, &offset) + if offset >= len(data) { + return node(nil), errUnexpectedEnd + } c2 := data[offset] // Add support for empty lists @@ -104,7 +114,7 @@ func parseErlangNode(data []byte, i *int) (erlangNode, error) { return node(nil), nil } - return parseErlangList(data, i) + return parseErlangList(data, i, depth+1) case '"': fallthrough case '\'': @@ -145,6 +155,9 @@ func parseErlangAngleString(data []byte, i *int) (erlangNode, error) { } func parseErlangString(data []byte, i *int) (erlangNode, error) { + if *i >= len(data) { + return node(nil), errUnexpectedEnd + } delim := data[*i] *i++ var buf bytes.Buffer @@ -167,13 +180,16 @@ func parseErlangString(data []byte, i *int) (erlangNode, error) { return node(nil), fmt.Errorf("unterminated string at %d", *i) } -func parseErlangList(data []byte, i *int) (erlangNode, error) { +func parseErlangList(data []byte, i *int, depth int) (erlangNode, error) { + if depth > parsing.MaxDepth { + return node(nil), errTooDeep + } *i++ out := erlangNode{ value: []erlangNode{}, } for *i < len(data) { - item, err := parseErlangNode(data, i) + item, err := parseErlangNode(data, i, depth) if err != nil { if err == errSkipComments { parsing.SkipWhitespace(data, i) @@ -183,6 +199,9 @@ func parseErlangList(data []byte, i *int) (erlangNode, error) { } out.value = append(out.value.([]erlangNode), item) parsing.SkipWhitespace(data, i) + if *i >= len(data) { + return node(nil), errUnexpectedEnd + } c := data[*i] switch c { case ',': @@ -210,7 +229,7 @@ func parseErlangComment(data []byte, i *int) { // Rest of a line is a comment. Deals with CR, LF and CR/LF if c == '\n' { break - } else if c == '\r' && data[*i] == '\n' { + } else if c == '\r' && *i < len(data) && data[*i] == '\n' { *i++ break } diff --git a/syft/pkg/cataloger/erlang/erlang_parser_test.go b/syft/pkg/cataloger/erlang/erlang_parser_test.go index 9b6950793..9fa30cefc 100644 --- a/syft/pkg/cataloger/erlang/erlang_parser_test.go +++ b/syft/pkg/cataloger/erlang/erlang_parser_test.go @@ -2,6 +2,7 @@ package erlang import ( "bytes" + "strings" "testing" "github.com/stretchr/testify/assert" @@ -128,3 +129,35 @@ func Test_parseErlangString_escapedByte(t *testing.T) { require.NoError(t, err) assert.Equal(t, `a"b`, got.String()) } + +func TestParseErlang_deepNesting(t *testing.T) { + // 4MB of '[' overflowed the goroutine stack (fatal, not recoverable) before nesting was bounded + _, err := parseErlang(strings.NewReader(strings.Repeat("[a,", 4<<20))) + require.ErrorIs(t, err, errTooDeep) + // the error echoes a clipped window of the single-line input, not the whole thing + assert.Less(t, len(err.Error()), 1024) +} + +func Test_parseErlang_truncated(t *testing.T) { + // each of these read past the end of the input before end-of-input was checked + tests := []struct { + name string + content string + wantErr require.ErrorAssertionFunc + }{ + {name: "whitespace only", content: " ", wantErr: require.Error}, + {name: "unterminated list", content: "[a", wantErr: require.Error}, + {name: "unterminated empty list", content: "[ ", wantErr: require.Error}, + {name: "lone binary string opener", content: "<", wantErr: require.Error}, + {name: "comment ending in CR", content: "%\r", wantErr: require.NoError}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + var err error + require.NotPanics(t, func() { + _, err = parseErlang(strings.NewReader(tt.content)) + }) + tt.wantErr(t, err) + }) + } +} diff --git a/syft/pkg/cataloger/gentoo/cataloger_test.go b/syft/pkg/cataloger/gentoo/cataloger_test.go index bff79a412..e35abac8d 100644 --- a/syft/pkg/cataloger/gentoo/cataloger_test.go +++ b/syft/pkg/cataloger/gentoo/cataloger_test.go @@ -3,6 +3,8 @@ package gentoo import ( "testing" + "github.com/stretchr/testify/require" + "github.com/anchore/syft/syft/artifact" "github.com/anchore/syft/syft/file" "github.com/anchore/syft/syft/pkg" @@ -170,3 +172,16 @@ func TestCataloger_Globs(t *testing.T) { }) } } + +func TestPortageCataloger_ShortObjLine(t *testing.T) { + // obj lines missing their path or digest are skipped rather than indexed past their end + pkgtest.NewCatalogTester(). + FromDirectory(t, "testdata/short-obj-line"). + ExpectsAssertion(func(t *testing.T, pkgs []pkg.Package, _ []artifact.Relationship) { + require.Len(t, pkgs, 1) + m := pkgs[0].Metadata.(pkg.PortageEntry) + require.Len(t, m.Files, 1) + require.Equal(t, "/usr/bin/bar", m.Files[0].Path) + }). + TestCataloger(t, NewPortageCataloger()) +} diff --git a/syft/pkg/cataloger/gentoo/parse_portage_contents.go b/syft/pkg/cataloger/gentoo/parse_portage_contents.go index 30ba84049..7d4d1d8c2 100644 --- a/syft/pkg/cataloger/gentoo/parse_portage_contents.go +++ b/syft/pkg/cataloger/gentoo/parse_portage_contents.go @@ -75,7 +75,8 @@ func addFiles(resolver file.Resolver, dbLocation file.Location, entry *pkg.Porta line := strings.Trim(scanner.Text(), "\n") fields := strings.Split(line, " ") - if fields[0] == "obj" { + // an obj line is "obj " + if len(fields) >= 3 && fields[0] == "obj" { record := pkg.PortageFileRecord{ Path: fields[1], } diff --git a/syft/pkg/cataloger/gentoo/testdata/short-obj-line/var/db/pkg/app-misc/foo-1.0/CONTENTS b/syft/pkg/cataloger/gentoo/testdata/short-obj-line/var/db/pkg/app-misc/foo-1.0/CONTENTS new file mode 100644 index 000000000..dd84618ef --- /dev/null +++ b/syft/pkg/cataloger/gentoo/testdata/short-obj-line/var/db/pkg/app-misc/foo-1.0/CONTENTS @@ -0,0 +1,4 @@ +dir /usr +obj +obj /usr/bin/foo +obj /usr/bin/bar abc123 1600000000 diff --git a/syft/pkg/cataloger/javascript/parse_yarn_lock.go b/syft/pkg/cataloger/javascript/parse_yarn_lock.go index 9213be760..7e65acf68 100644 --- a/syft/pkg/cataloger/javascript/parse_yarn_lock.go +++ b/syft/pkg/cataloger/javascript/parse_yarn_lock.go @@ -206,28 +206,16 @@ func parseYarnV1LockFile(reader io.ReadCloser) ([]yarnPackage, error) { continue } if strings.HasPrefix(line, " ") && !strings.HasPrefix(line, " ") { - line = strings.Trim(line, " ") - array := strings.Split(line, " ") - switch array[0] { - case "version": - pkg.Version = strings.Trim(array[1], "\"") - case "resolved": - name, version, resolved := findResolvedPackageAndVersion(line) - if name != "" && version != "" && resolved != "" { - pkg.Name = name - pkg.Version = version - pkg.Resolved = resolved - } else { - pkg.Resolved = strings.Trim(array[1], "\"") - } - case "integrity": - pkg.Integrity = strings.Trim(array[1], "\"") - } + applyYarnV1Field(&pkg, strings.Trim(line, " ")) continue } if strings.HasPrefix(line, " ") { line = strings.Trim(line, " ") array := strings.Split(line, " ") + if len(array) < 2 { + // a dependency without a version constraint is malformed; skip it rather than guess + continue + } dependencyName := strings.Trim(array[0], "\"") dependencyVersion := strings.Trim(array[1], "\"") dependencies[dependencyName] = dependencyVersion @@ -244,6 +232,31 @@ func parseYarnV1LockFile(reader io.ReadCloser) ([]yarnPackage, error) { return pkgs, nil } +// applyYarnV1Field sets the package field described by a single (already trimmed) yarn v1 field line. +func applyYarnV1Field(pkg *yarnPackage, line string) { + array := strings.Split(line, " ") + // a field line may be missing its value in a malformed lockfile + var value string + if len(array) > 1 { + value = strings.Trim(array[1], "\"") + } + switch array[0] { + case "version": + pkg.Version = value + case "resolved": + name, version, resolved := findResolvedPackageAndVersion(line) + if name != "" && version != "" && resolved != "" { + pkg.Name = name + pkg.Version = version + pkg.Resolved = resolved + } else { + pkg.Resolved = value + } + case "integrity": + pkg.Integrity = value + } +} + func parseYarnLockYaml(reader io.ReadCloser) ([]yarnPackage, error) { var lockfile = map[string]yarnV2PackageEntry{} if err := yaml.NewDecoder(reader, yaml.AllowDuplicateMapKey()).Decode(&lockfile); err != nil { diff --git a/syft/pkg/cataloger/javascript/parse_yarn_lock_test.go b/syft/pkg/cataloger/javascript/parse_yarn_lock_test.go index 93a24c725..7c18eab01 100644 --- a/syft/pkg/cataloger/javascript/parse_yarn_lock_test.go +++ b/syft/pkg/cataloger/javascript/parse_yarn_lock_test.go @@ -6,9 +6,11 @@ import ( "net/http" "net/http/httptest" "os" + "strings" "testing" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" "github.com/anchore/syft/syft/artifact" "github.com/anchore/syft/syft/file" @@ -1091,3 +1093,35 @@ func setupYarnRegistry() (mux *http.ServeMux, serverURL string, teardown func()) return mux, server.URL, server.Close } + +func TestParseYarnV1LockFile_missingValues(t *testing.T) { + tests := []struct { + name string + content string + want []yarnPackage + }{ + { + // a dependency line without a constraint is dropped rather than indexed past its end + name: "dependency without version", + content: "# yarn lockfile v1\n\na@1:\n version \"1\"\n dependencies:\n b\n\n", + want: []yarnPackage{{Name: "a", Version: "1", Dependencies: map[string]string{}}}, + }, + { + // a field line without a value leaves the field empty + name: "field without value", + content: "# yarn lockfile v1\n\na@1:\n version\n\n", + want: []yarnPackage{{Name: "a", Dependencies: map[string]string{}}}, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + var got []yarnPackage + var err error + require.NotPanics(t, func() { + got, err = parseYarnV1LockFile(io.NopCloser(strings.NewReader(tt.content))) + }) + require.NoError(t, err) + assert.Equal(t, tt.want, got) + }) + } +} diff --git a/syft/pkg/cataloger/lua/rockspec_parser.go b/syft/pkg/cataloger/lua/rockspec_parser.go index 14317b5f0..8c42d8f8e 100644 --- a/syft/pkg/cataloger/lua/rockspec_parser.go +++ b/syft/pkg/cataloger/lua/rockspec_parser.go @@ -2,6 +2,7 @@ package lua import ( "bytes" + "errors" "fmt" "io" @@ -33,6 +34,10 @@ func (r rockspecNode) String() string { return "" } +var errUnexpectedEnd = errors.New("unexpected end of input") + +var errTooDeep = fmt.Errorf("nesting deeper than %d levels", parsing.MaxDepth) + var noReturn = rockspec{ value: nil, } @@ -46,10 +51,11 @@ func parseRockspecData(reader io.Reader) (rockspec, error) { i := 0 locals := make(map[string]string) - blocks, err := parseRockspecBlock(data, &i, locals) + blocks, err := parseRockspecBlock(data, &i, locals, 0) if err != nil { - return noReturn, err + // decorate once here rather than at every nesting level, which copied the input once per level + return noReturn, fmt.Errorf("%w\n%s", err, parsing.PrintError(data, i)) } return rockspec{ @@ -57,9 +63,12 @@ func parseRockspecData(reader io.Reader) (rockspec, error) { }, nil } -func parseRockspecBlock(data []byte, i *int, locals map[string]string) ([]rockspecNode, error) { +func parseRockspecBlock(data []byte, i *int, locals map[string]string, depth int) ([]rockspecNode, error) { + if depth > parsing.MaxDepth { + return nil, errTooDeep + } var out []rockspecNode - var iterator func(data []byte, i *int, locals map[string]string) (*rockspecNode, error) + var iterator func(data []byte, i *int, locals map[string]string, depth int) (*rockspecNode, error) parsing.SkipWhitespace(data, i) @@ -92,9 +101,9 @@ func parseRockspecBlock(data []byte, i *int, locals map[string]string) ([]rocksp } for *i < len(data) { - item, err := iterator(data, i, locals) + item, err := iterator(data, i, locals, depth) if err != nil { - return nil, fmt.Errorf("%w\n%s", err, parsing.PrintError(data, *i)) + return nil, err } parsing.SkipWhitespace(data, i) @@ -114,7 +123,7 @@ func parseRockspecBlock(data []byte, i *int, locals map[string]string) ([]rocksp } //nolint:funlen, gocognit -func parseRockspecNode(data []byte, i *int, locals map[string]string) (*rockspecNode, error) { +func parseRockspecNode(data []byte, i *int, locals map[string]string, depth int) (*rockspecNode, error) { parsing.SkipWhitespace(data, i) if *i >= len(data) { @@ -212,7 +221,7 @@ func parseRockspecNode(data []byte, i *int, locals map[string]string) (*rockspec *i = offset parsing.SkipWhitespace(data, i) - obj, err := parseRockspecBlock(data, i, locals) + obj, err := parseRockspecBlock(data, i, locals, depth+1) if err != nil { return nil, err @@ -247,6 +256,9 @@ func parseRockspecNode(data []byte, i *int, locals map[string]string) (*rockspec } value := str.String() + if *i >= len(data) { + return nil, errUnexpectedEnd + } c = data[*i] if c != ']' { @@ -271,7 +283,7 @@ func parseRockspecNode(data []byte, i *int, locals map[string]string) (*rockspec }, nil } -func parseRockspecListItem(data []byte, i *int, locals map[string]string) (*rockspecNode, error) { +func parseRockspecListItem(data []byte, i *int, locals map[string]string, _ int) (*rockspecNode, error) { parsing.SkipWhitespace(data, i) if *i >= len(data) { @@ -378,6 +390,9 @@ out: if err != nil { return "", err } + if *i >= len(data) { + return "", errUnexpectedEnd + } c = data[*i] if c != ']' { return "", fmt.Errorf("unterminated literal at %d", *i) @@ -394,6 +409,9 @@ out: } func parseRockspecString(data []byte, i *int, _ map[string]string) (*rockspecNode, error) { + if *i >= len(data) { + return nil, errUnexpectedEnd + } delim := data[*i] var endDelim byte switch delim { @@ -434,6 +452,39 @@ func parseComment(data []byte, i *int) { } } +// parseLocalValue parses a single value on the right-hand side of a local assignment. Expressions +// resolve to an empty string, and bare references resolve through locals. The caller must ensure *i < len(data). +func parseLocalValue(data []byte, i *int, locals map[string]string) (string, error) { + c := data[*i] + + switch c { + case '"', '\'': + value, err := parseRockspecString(data, i, locals) + if err != nil { + return "", err + } + return value.value.(string), nil + default: + ref, err := parseRockspecLiteral(data, i, locals) + if err != nil { + return "", err + } + + // skip if it's an expression + skipWhitespaceNoNewLine(data, i) + if *i >= len(data) { + return "", errUnexpectedEnd + } + c := data[*i] + + if c != '\n' && c != '\r' { + skipExpression(data, i) + return "", nil + } + return locals[ref], nil + } +} + //nolint:funlen func parseLocal(data []byte, i *int, locals map[string]string) error { keys := []string{} @@ -459,6 +510,9 @@ keys: keys = append(keys, key) parsing.SkipWhitespace(data, i) + if *i >= len(data) { + return errUnexpectedEnd + } c := data[*i] @@ -477,42 +531,22 @@ keys: values: for { skipWhitespaceNoNewLine(data, i) - - c := data[*i] - - switch c { - case '"', '\'': - value, err := parseRockspecString(data, i, locals) - - if err != nil { - return err - } - values = append(values, value.value.(string)) - default: - ref, err := parseRockspecLiteral(data, i, locals) - if err != nil { - return err - } - - // Skip if it's an expression - skipWhitespaceNoNewLine(data, i) - c := data[*i] - - var value string - - if c != '\n' && c != '\r' { - skipExpression(data, i) - value = "" - } else { - value = locals[ref] - } - - values = append(values, value) + if *i >= len(data) { + return errUnexpectedEnd } - skipWhitespaceNoNewLine(data, i) + value, err := parseLocalValue(data, i, locals) + if err != nil { + return err + } + values = append(values, value) - c = data[*i] + skipWhitespaceNoNewLine(data, i) + if *i >= len(data) { + return errUnexpectedEnd + } + + c := data[*i] switch c { case ',': diff --git a/syft/pkg/cataloger/lua/rockspec_parser_test.go b/syft/pkg/cataloger/lua/rockspec_parser_test.go index aca545e00..6a7bd70f7 100644 --- a/syft/pkg/cataloger/lua/rockspec_parser_test.go +++ b/syft/pkg/cataloger/lua/rockspec_parser_test.go @@ -2,6 +2,7 @@ package lua import ( "bytes" + "strings" "testing" "github.com/stretchr/testify/assert" @@ -282,3 +283,35 @@ hello = foo.. `, }) } } + +func TestParseRockspec_deepNesting(t *testing.T) { + // deep input overflowed the goroutine stack (fatal, not recoverable) before nesting was bounded, and each level + // used to append its own copy of the input to the error + _, err := parseRockspecData(strings.NewReader(strings.Repeat("a={", 3<<20))) + require.ErrorIs(t, err, errTooDeep) + assert.Less(t, len(err.Error()), 1024) +} + +func Test_parseRockspecData_truncated(t *testing.T) { + // each of these ends mid-statement, and used to read past the end of the input + tests := []struct { + name string + content string + }{ + {name: "unterminated long string value", content: "a=[[x]"}, + {name: "unterminated index literal", content: `a["x"`}, + {name: "local missing equals", content: "local a"}, + {name: "local missing value", content: "local a="}, + {name: "local reference at end of input", content: "local a=b"}, + {name: "local string at end of input", content: `local a="x"`}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + var err error + require.NotPanics(t, func() { + _, err = parseRockspecData(strings.NewReader(tt.content)) + }) + require.Error(t, err) + }) + } +} diff --git a/syft/pkg/cataloger/parsers_fuzz_test.go b/syft/pkg/cataloger/parsers_fuzz_test.go new file mode 100644 index 000000000..2d1ef1056 --- /dev/null +++ b/syft/pkg/cataloger/parsers_fuzz_test.go @@ -0,0 +1,400 @@ +package cataloger_test + +import ( + "context" + "encoding/json" + "errors" + "flag" + "fmt" + "go/ast" + "go/parser" + "go/token" + "io/fs" + "os" + "path/filepath" + "regexp" + "runtime" + "runtime/debug" + "runtime/metrics" + "runtime/pprof" + "sort" + "strconv" + "strings" + "testing" + "time" + + "github.com/bmatcuk/doublestar/v4" + + gosync "github.com/anchore/go-sync" + "github.com/anchore/syft/internal/capabilities" + "github.com/anchore/syft/internal/sbomsync" + "github.com/anchore/syft/internal/task" + "github.com/anchore/syft/internal/unknown" + "github.com/anchore/syft/syft/pkg" + "github.com/anchore/syft/syft/pkg/cataloger/internal/pkgtest" + "github.com/anchore/syft/syft/sbom" + "github.com/anchore/syft/syft/source" + "github.com/anchore/syft/syft/source/directorysource" +) + +const ( + // per-input limits. These are generous on purpose: the net is for inputs that take seconds or + // hundreds of MB, not for small inefficiencies. + fuzzTimeout = 10 * time.Second + // fuzzHangSlack is how long past fuzzTimeout a parser that ignores ctx gets before the process is taken down, + // so a ctx-aware parser always reports its timeout through the normal path first + fuzzHangSlack = 5 * time.Second + fuzzAllocBase = 256 << 20 + fuzzAllocPerByte = 1024 + fuzzMaxSeedBytes = 256 << 10 + fuzzMaxSeedsPerFn = 8 +) + +type fuzzTarget struct { + cataloger string + glob string + path string // a concrete relative path that matches glob + dir string // the cataloger package directory, relative to this package, for seeds + task task.Task +} + +func (ft fuzzTarget) String() string { return ft.cataloger + " " + ft.glob } + +// FuzzGenericParsers feeds one input at a time to each glob-selected parser of every generic cataloger, by writing +// it to a path the parser's glob matches and running the cataloger task over that directory. Catalogers that are not +// generic, and processor panics (which the generic cataloger only logs), are not covered. +// +// # What fuzzing is +// +// A fuzz test is a test whose input is generated. Go's fuzzer starts from a set of example inputs (the seed corpus), +// mutates them (flipping bytes, splicing pieces together, inserting values) and keeps any mutation that reaches code +// it has not reached before, so over time it works its way into parser branches that hand-written tests never try. +// Each input is one parser and one file's worth of bytes. An input fails when the parser: +// +// - panics (including a panic the generic cataloger recovers and reports as an unknown) +// - takes longer than fuzzTimeout +// - allocates much more than the input size warrants (see fuzzAllocBase and fuzzAllocPerByte) +// +// A parser returning an error is fine: rejecting bad input is the correct behavior. +// +// # Plain go test (no fuzzing) +// +// Without -fuzz, go test does not generate anything. It runs every seed once, like a table test, and reports them +// as subtests named seed#0, seed#1, ... (go names seeds by position; the failure message says which parser it was). +// The seeds are each parser's own testdata files plus a few inputs built from the string literals in its package, +// which give the parser its own key names and prefixes to trip over. This is what runs in CI, and takes seconds. +// It also replays every saved failure in testdata/fuzz/FuzzGenericParsers/ (see below). +// +// # Running the fuzzer +// +// Fuzzing is only done on demand, locally. Pick one parser with SYFT_FUZZ_TARGET, a regexp over +// " " (cataloger names are listed by `syft cataloger list`), and give it a time budget: +// +// SYFT_FUZZ_TARGET='^apk-db-cataloger ' go test ./syft/pkg/cataloger -run '^$' -fuzz FuzzGenericParsers -fuzztime 60s +// +// -run '^$' skips the ordinary tests, and -fuzztime bounds the run (without it the fuzzer runs until stopped). +// Without SYFT_FUZZ_TARGET every parser shares one run, so each gets a small slice of the time. The fuzzer's +// "new interesting" inputs are kept in the go build cache, not the repo, so later runs pick up where earlier ones +// left off. +// +// # When the fuzzer finds a failure +// +// It shrinks the input to a small version that still fails, prints the failure, and writes it to +// testdata/fuzz/FuzzGenericParsers/. That file is two lines of text, the parser name and the input bytes, so +// it can be read in review. From then on plain go test replays it as the subtest FuzzGenericParsers/, and it +// can be run alone: +// +// go test ./syft/pkg/cataloger -run 'FuzzGenericParsers/' +// +// The lifecycle of a failure: +// +// 1. fix the parser so the input is rejected with an error (or parsed correctly) rather than panicking or hanging +// 2. commit the saved file in the same change as the fix, as the regression test for it; do not commit one without +// its fix, since it fails every test run until the fix lands +// 3. fuzz that parser again, since one fix often uncovers the next failure behind it +// +// Saved files are kept indefinitely, like any other regression test. Delete one only when it no longer means +// anything (for example, the parser it names was removed). A test case in the parser's own _test.go is an equally +// good home for the input, and reads better there. +func FuzzGenericParsers(f *testing.F) { + targets := fuzzTargets(f) + if len(targets) == 0 { + f.Skip("no targets selected") + } + + // inputs name their target rather than index it, so a saved crasher keeps pointing at the same parser when + // the target list changes or is filtered + byName := map[string]fuzzTarget{} + for _, ft := range targets { + byName[ft.String()] = ft + for _, seed := range fuzzSeeds(ft) { + f.Add(ft.String(), seed) + } + } + + f.Fuzz(func(t *testing.T, name string, data []byte) { + ft, ok := byName[name] + if !ok { + // while fuzzing, the mutator rewrites names too, and a filter deliberately drops targets. Otherwise this + // is a saved input whose target was renamed or removed, and skipping it would silently drop a regression. + if os.Getenv("SYFT_FUZZ_TARGET") == "" && flag.Lookup("test.fuzz").Value.String() == "" { + t.Fatalf("saved fuzz input names unknown target %q: update or remove it", name) + } + t.Skip("not a selected target") + } + runFuzzTarget(t, ft, data) + }) +} + +func runFuzzTarget(t *testing.T, ft fuzzTarget, data []byte) { + root := t.TempDir() + p := filepath.Join(root, filepath.FromSlash(ft.path)) + if err := os.MkdirAll(filepath.Dir(p), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(p, data, 0o600); err != nil { + t.Fatal(err) + } + src, err := directorysource.NewFromPath(root) + if err != nil { + t.Fatal(err) + } + resolver, err := src.FileResolver(source.AllLayersScope) + if err != nil { + t.Fatal(err) + } + ctx, cancel := context.WithTimeout(pkgtest.Context(t), fuzzTimeout) + defer cancel() + + var before, after runtime.MemStats + runtime.ReadMemStats(&before) + + // the heap metric is process-wide and counts unswept garbage, so collect first and measure growth from there + heap := []metrics.Sample{{Name: "/memory/classes/heap/objects:bytes"}} + runtime.GC() + metrics.Read(heap) + heapBase := heap[0].Value.Uint64() + + done := make(chan error, 1) + go func() { + defer func() { + if r := recover(); r != nil { + done <- fmt.Errorf("panic: %v\n%s", r, debug.Stack()) + } + }() + done <- ft.task.Execute(ctx, resolver, sbomsync.NewBuilder(&sbom.SBOM{Artifacts: sbom.Artifacts{Packages: pkg.NewCollection()}})) + }() + + limit := uint64(fuzzAllocBase + fuzzAllocPerByte*len(data)) + tick := time.NewTicker(50 * time.Millisecond) + defer tick.Stop() + deadline := time.After(fuzzTimeout + fuzzHangSlack) +wait: + for { + select { + case err = <-done: + break wait + case <-tick.C: + // the after-the-fact TotalAlloc check below can't stop a parser that is on its way to exhausting the + // machine, so watch heap growth while it runs and take the process down once it passes the limit + metrics.Read(heap) + if grown := heap[0].Value.Uint64() - min(heapBase, heap[0].Value.Uint64()); grown > limit { + _ = pprof.Lookup("goroutine").WriteTo(os.Stderr, 2) + panic(fmt.Sprintf("%s: input of %d bytes grew the heap by %d MB (limit %d MB)", ft, len(data), grown>>20, limit>>20)) + } + case <-deadline: + // not every parser watches ctx, so a hung one keeps running and would skew every later input in this + // process. Dump where it is stuck and take the process down; the fuzz engine records that as a crasher. + _ = pprof.Lookup("goroutine").WriteTo(os.Stderr, 2) + panic(fmt.Sprintf("%s: input of %d bytes did not finish within %s", ft, len(data), fuzzTimeout+fuzzHangSlack)) + } + } + + if ctx.Err() != nil { + t.Fatalf("%s: input of %d bytes did not finish within %s", ft, len(data), fuzzTimeout) + } + + // a panic recovered further down (a go-sync PanicError, or the generic per-file recover) still surfaces as an error + if errors.As(err, &gosync.PanicError{}) || hasRecoveredPanic(err) { + t.Fatalf("%s: %v", ft, err) + } + + // TotalAlloc is cumulative, so this bounds churn rather than peak, and it is read after the fact: a single huge + // allocation kills the process before this check, which the fuzz engine still records as a crasher + runtime.ReadMemStats(&after) + if alloc := after.TotalAlloc - before.TotalAlloc; alloc > limit { + t.Fatalf("%s: input of %d bytes allocated %d MB (limit %d MB)", ft, len(data), alloc>>20, limit>>20) + } +} + +// hasRecoveredPanic finds an error anywhere in the tree that starts with "recovered from panic", the wording of the +// generic per-file recover (and of the golang and license recovers). Matching on the start of a node keeps parse +// errors that echo the input from counting. +func hasRecoveredPanic(err error) bool { + if err == nil { + return false + } + if strings.HasPrefix(err.Error(), "recovered from panic") { + return true + } + switch u := err.(type) { + case *unknown.CoordinateError: + // per-file errors arrive wrapped with their location, and CoordinateError does not unwrap + return hasRecoveredPanic(u.Reason) + case interface{ Unwrap() []error }: + for _, e := range u.Unwrap() { + if hasRecoveredPanic(e) { + return true + } + } + case interface{ Unwrap() error }: + return hasRecoveredPanic(u.Unwrap()) + } + return false +} + +// fuzzTargets pairs every glob-selected parser in the capabilities data with the task that runs its cataloger. +func fuzzTargets(tb testing.TB) []fuzzTarget { + tasks, err := task.DefaultPackageTaskFactories().Tasks(task.DefaultCatalogingFactoryConfig()) + if err != nil { + tb.Fatal(err) + } + byName := map[string]task.Task{} + for _, tsk := range tasks { + byName[tsk.Name()] = tsk + } + + entries, err := capabilities.Packages() + if err != nil { + tb.Fatal(err) + } + + var filter *regexp.Regexp + if v := os.Getenv("SYFT_FUZZ_TARGET"); v != "" { + if filter, err = regexp.Compile(v); err != nil { + tb.Fatalf("SYFT_FUZZ_TARGET: %v", err) + } + } + + var targets []fuzzTarget + for _, e := range entries { + tsk, ok := byName[e.Name] + if e.Type != "generic" || !ok { + continue + } + for _, p := range e.Parsers { + // glob parsers only. The mimetype parsers (binary classifiers) need a real executable + // header to be selected at all; add them if a mutating seed turns out to be worth it. + if p.Detector.Method != capabilities.GlobDetection { + continue + } + for _, g := range p.Detector.Criteria { + ft := fuzzTarget{ + cataloger: e.Name, + glob: g, + path: concretePath(g), + dir: strings.TrimPrefix(filepath.Dir(e.Source.File), "syft/pkg/cataloger/"), + task: tsk, + } + if ok, _ := doublestar.Match(g, "/"+ft.path); !ok || !filepath.IsLocal(ft.path) { + tb.Logf("skipping %s: no concrete path for glob", ft) + continue + } + if filter != nil && !filter.MatchString(ft.String()) { + continue + } + targets = append(targets, ft) + } + } + } + if filter != nil && len(targets) == 0 { + tb.Fatalf("SYFT_FUZZ_TARGET %q matches no target", filter) + } + return targets +} + +var braceAlt = regexp.MustCompile(`\{([^,}]*)[^}]*\}`) + +// concretePath turns a glob into one relative path it matches, e.g. `**/*.gguf` -> `x.gguf`. +func concretePath(glob string) string { + s := strings.TrimPrefix(strings.TrimPrefix(glob, "**/"), "/") + s = braceAlt.ReplaceAllString(s, "$1") + s = strings.ReplaceAll(s, "**", "x") + return strings.ReplaceAll(s, "*", "x") +} + +// fuzzSeeds is the parser's own testdata that matches its glob, plus a few inputs built from the string literals in +// its package. Random bytes rarely produce the exact key names a parser branches on, the literals give the mutator +// something to splice. +func fuzzSeeds(ft fuzzTarget) [][]byte { + var seeds [][]byte + // the CLI's malformed fixtures are deliberately not seeds here: before their fixes land some of them allocate + // gigabytes, which in-process means swapping the machine rather than failing a check. The CLI test runs them in + // a subprocess under a memory limit instead. + _ = filepath.WalkDir(ft.dir, func(p string, d fs.DirEntry, err error) error { + if err != nil || len(seeds) >= fuzzMaxSeedsPerFn { + return fs.SkipAll + } + if d.IsDir() || !d.Type().IsRegular() { + return nil + } + if ok, _ := doublestar.Match(ft.glob, "/"+filepath.ToSlash(p)); !ok { + return nil + } + if info, err := d.Info(); err != nil || info.Size() > fuzzMaxSeedBytes { + return nil + } + if b, err := os.ReadFile(p); err == nil { + seeds = append(seeds, b) + } + return nil + }) + + lits := stringLiterals(ft.dir) + if len(lits) == 0 { + return append(seeds, []byte{}) + } + obj := map[string]string{} + var lines, kv strings.Builder + for _, l := range lits { + obj[l] = l + lines.WriteString(l + "\n") + kv.WriteString(strconv.Quote(l) + ": " + strconv.Quote(l) + "\n") + } + js, _ := json.Marshal(obj) + return append(seeds, []byte(lines.String()), []byte(kv.String()), js) +} + +// stringLiterals returns the distinct short string literals in the non-test go files of a package directory. +func stringLiterals(dir string) []string { + entries, err := os.ReadDir(dir) + if err != nil { + return nil + } + set := map[string]struct{}{} + fset := token.NewFileSet() + for _, e := range entries { + name := e.Name() + if e.IsDir() || !strings.HasSuffix(name, ".go") || strings.HasSuffix(name, "_test.go") { + continue + } + f, err := parser.ParseFile(fset, filepath.Join(dir, name), nil, 0) + if err != nil { + continue + } + ast.Inspect(f, func(n ast.Node) bool { + if bl, ok := n.(*ast.BasicLit); ok && bl.Kind == token.STRING { + if s, err := strconv.Unquote(bl.Value); err == nil && len(s) > 1 && len(s) <= 64 { + set[s] = struct{}{} + } + } + return true + }) + } + out := make([]string, 0, len(set)) + for s := range set { + out = append(out, s) + } + sort.Strings(out) + return out +} diff --git a/syft/pkg/cataloger/python/parse_requirements.go b/syft/pkg/cataloger/python/parse_requirements.go index f79737e6a..d167c9d16 100644 --- a/syft/pkg/cataloger/python/parse_requirements.go +++ b/syft/pkg/cataloger/python/parse_requirements.go @@ -214,9 +214,8 @@ func guessVersion(constraint string) string { constraints := strings.Split(constraint, ",") filteredVersions := map[string]struct{}{} for _, part := range constraints { - if strings.Contains(part, "!=") { - parts := strings.Split(part, "!=") - filteredVersions[strings.TrimSpace(parts[1])] = struct{}{} + if _, excluded, ok := strings.Cut(part, "!="); ok { + filteredVersions[strings.TrimSpace(excluded)] = struct{}{} } } @@ -236,9 +235,8 @@ func guessVersion(constraint string) string { continue } - if strings.Contains(part, "==") { - parts := strings.Split(part, "==") - return strings.TrimSpace(parts[1]) + if _, pinned, ok := strings.Cut(part, "=="); ok { + return strings.TrimSpace(pinned) } if closestVersion == nil || version.GreaterThan(*closestVersion) { diff --git a/syft/pkg/cataloger/snap/parse_base_dpkg.go b/syft/pkg/cataloger/snap/parse_base_dpkg.go index 72bb20482..9cc958583 100644 --- a/syft/pkg/cataloger/snap/parse_base_dpkg.go +++ b/syft/pkg/cataloger/snap/parse_base_dpkg.go @@ -60,10 +60,9 @@ func parseBaseDpkgYaml(_ context.Context, _ file.Resolver, _ *generic.Environmen } // Handle architecture suffixes (e.g., "libssl1.1:amd64") - if strings.Contains(name, ":") { - archParts := strings.SplitN(name, ":", 2) - name = archParts[0] - snapMetadata.Architecture = archParts[1] + if n, arch, ok := strings.Cut(name, ":"); ok { + name = n + snapMetadata.Architecture = arch } debPkg := newDebianPackageFromSnap( diff --git a/syft/pkg/cataloger/snap/parse_system_manifest.go b/syft/pkg/cataloger/snap/parse_system_manifest.go index 8a84e3233..72c934647 100644 --- a/syft/pkg/cataloger/snap/parse_system_manifest.go +++ b/syft/pkg/cataloger/snap/parse_system_manifest.go @@ -82,10 +82,9 @@ func parseSystemManifest(_ context.Context, _ file.Resolver, _ *generic.Environm // Handle architecture suffixes if present currentMetadata := snapMetadata - if strings.Contains(name, ":") { - archParts := strings.SplitN(name, ":", 2) - name = archParts[0] - currentMetadata.Architecture = archParts[1] + if n, arch, ok := strings.Cut(name, ":"); ok { + name = n + currentMetadata.Architecture = arch } debPkg := newDebianPackageFromSnap( diff --git a/syft/pkg/cataloger/swift/parse_podfile_lock.go b/syft/pkg/cataloger/swift/parse_podfile_lock.go index d8d094186..e571ca436 100644 --- a/syft/pkg/cataloger/swift/parse_podfile_lock.go +++ b/syft/pkg/cataloger/swift/parse_podfile_lock.go @@ -2,6 +2,7 @@ package swift import ( "context" + "errors" "fmt" "strings" @@ -45,9 +46,11 @@ func parsePodfileLock(_ context.Context, _ file.Resolver, _ *generic.Environment default: return nil, nil, fmt.Errorf("malformed podfile.lock") } - splits := strings.Split(podBlob, " ") - podName := splits[0] - podVersion := strings.TrimSuffix(strings.TrimPrefix(splits[1], "("), ")") + podName, podVersion, ok := strings.Cut(podBlob, " ") + if !ok { + return nil, nil, errors.New("malformed podfile.lock: pod has no version") + } + podVersion = strings.TrimSuffix(strings.TrimPrefix(podVersion, "("), ")") podRootPkg := strings.Split(podName, "/")[0] var pkgHash string diff --git a/syft/pkg/cataloger/swift/parse_podfile_lock_test.go b/syft/pkg/cataloger/swift/parse_podfile_lock_test.go index 7769d85c5..e01ce3ec0 100644 --- a/syft/pkg/cataloger/swift/parse_podfile_lock_test.go +++ b/syft/pkg/cataloger/swift/parse_podfile_lock_test.go @@ -1,8 +1,13 @@ package swift import ( + "context" + "io" + "strings" "testing" + "github.com/stretchr/testify/require" + "github.com/anchore/syft/syft/artifact" "github.com/anchore/syft/syft/file" "github.com/anchore/syft/syft/pkg" @@ -280,3 +285,15 @@ func Test_corruptPodfile(t *testing.T) { WithError(). TestParser(t, parsePodfileLock) } + +func TestParsePodfileLock_podWithoutVersion(t *testing.T) { + // a pod entry is "Name (version)"; one without the version used to index past the end of the split + contents := "PODS:\n - Foo\nSPEC CHECKSUMS:\n Foo: abc\n" + reader := file.NewLocationReadCloser(file.NewLocation("Podfile.lock"), io.NopCloser(strings.NewReader(contents))) + + var err error + require.NotPanics(t, func() { + _, _, err = parsePodfileLock(context.Background(), nil, nil, reader) + }) + require.ErrorContains(t, err, "has no version") +} diff --git a/syft/pkg/cataloger/testdata/fuzz/FuzzGenericParsers/e6edf122b3cfae05 b/syft/pkg/cataloger/testdata/fuzz/FuzzGenericParsers/e6edf122b3cfae05 new file mode 100644 index 000000000..fa31971c5 --- /dev/null +++ b/syft/pkg/cataloger/testdata/fuzz/FuzzGenericParsers/e6edf122b3cfae05 @@ -0,0 +1,3 @@ +go test fuzz v1 +string("lua-rock-cataloger **/*.rockspec") +[]byte("[") diff --git a/test/cli/malformed_input_test.go b/test/cli/malformed_input_test.go new file mode 100644 index 000000000..d831ab363 --- /dev/null +++ b/test/cli/malformed_input_test.go @@ -0,0 +1,78 @@ +package cli + +import ( + "encoding/json" + "os" + "path/filepath" + "runtime" + "syscall" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// maxMalformedScanRSS is far above what scanning a handful of tiny files needs. RSS only sees memory that was actually +// touched, so a large allocation that is never filled can slip under it; the fuzz harness in syft/pkg/cataloger +// measures allocations directly. +const maxMalformedScanRSS = 1 << 30 + +// each directory under testdata/malformed holds a few tiny files shaped to trip parsers up (deep nesting, huge declared +// counts, truncation, type mismatches, non-finite floats). None of them should cost the rest of the SBOM, so each is +// scanned next to a well-formed requirements.txt whose package must still be reported. +func TestMalformedInputsStillProduceSBOM(t *testing.T) { + dirs, err := filepath.Glob("testdata/malformed/*") + require.NoError(t, err) + require.NotEmpty(t, dirs) + + for _, dir := range dirs { + t.Run(filepath.Base(dir), func(t *testing.T) { + root := t.TempDir() + require.NoError(t, os.CopyFS(root, os.DirFS(dir))) + require.NoError(t, os.WriteFile(filepath.Join(root, "requirements.txt"), []byte("well-formed==1.0.0\n"), 0o600)) + + out := filepath.Join(t.TempDir(), "sbom.json") + env := map[string]string{"GOMEMLIMIT": "512MiB"} + // runSyft aborts the process after 60s + cmd, stdout, stderr := runSyft(t, env, "scan", "dir:"+root, "--override-default-catalogers", "all", "-o", "syft-json="+out) + defer logOutputOnFailure(t, cmd, stdout, stderr) + + assertSuccessfulReturnCode(t, stdout, stderr, cmd.ProcessState.ExitCode()) + + if ru, ok := cmd.ProcessState.SysUsage().(*syscall.Rusage); ok { + rss := int64(ru.Maxrss) // int32 on 32-bit linux + if runtime.GOOS == "linux" { + rss *= 1024 // kilobytes on linux, bytes on darwin + } + assert.Less(t, rss, int64(maxMalformedScanRSS), "max RSS %d MB", rss>>20) + } + + b, err := os.ReadFile(out) + require.NoError(t, err, "no SBOM written") + var doc struct { + Artifacts []struct { + Name string `json:"name"` + } `json:"artifacts"` + Files []struct { + Location struct { + Path string `json:"path"` + } `json:"location"` + Unknowns []string `json:"unknowns"` + } `json:"files"` + } + require.NoError(t, json.Unmarshal(b, &doc), "SBOM is not valid JSON") + var names []string + for _, a := range doc.Artifacts { + names = append(names, a.Name) + } + assert.Contains(t, names, "well-formed", "package from the well-formed file is missing") + + // a per-file panic is recovered into an unknown and the scan still exits 0, so look for it explicitly + for _, f := range doc.Files { + for _, u := range f.Unknowns { + assert.NotContains(t, u, "recovered from panic", "parser panicked on %s", f.Location.Path) + } + } + }) + } +} diff --git a/test/cli/testdata/.gitignore b/test/cli/testdata/.gitignore index 3e8e5ed17..a11d22644 100644 --- a/test/cli/testdata/.gitignore +++ b/test/cli/testdata/.gitignore @@ -1,3 +1,4 @@ # negate jar and zip exclusions for test fixtures that need these file types !image-unknowns/*.jar !image-unknowns/*.zip +!malformed/**/*.jar diff --git a/test/cli/testdata/malformed/gguf-declared-count/model.gguf b/test/cli/testdata/malformed/gguf-declared-count/model.gguf new file mode 100644 index 0000000000000000000000000000000000000000..41fc711f4c69ad4e7d5de6cfbe058f81114ed5a6 GIT binary patch literal 108 zcmZ>F4|QW^KmaBvQv^zF4|QW^KmZmfQv^zz^e6%@(WV+(uz_GF4|QW^KmaBvQv^z+5GViu literal 0 HcmV?d00001 diff --git a/test/cli/testdata/malformed/gguf-type-mismatch/model.gguf b/test/cli/testdata/malformed/gguf-type-mismatch/model.gguf new file mode 100644 index 0000000000000000000000000000000000000000..327ca18b47bf283ff4ffd058a872cbfbe59a59c3 GIT binary patch literal 97 zcmZ>F4|QW^KmaBvQv^z_7|vyJ-+A literal 0 HcmV?d00001 diff --git a/test/cli/testdata/malformed/json-deep-nesting/package.json b/test/cli/testdata/malformed/json-deep-nesting/package.json new file mode 100644 index 000000000..b14bbbce8 --- /dev/null +++ b/test/cli/testdata/malformed/json-deep-nesting/package.json @@ -0,0 +1 @@ +{"name":"deep","version":"1.0.0","config":[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]} \ No newline at end of file diff --git a/test/cli/testdata/malformed/list-for-scalar/Cargo.lock b/test/cli/testdata/malformed/list-for-scalar/Cargo.lock new file mode 100644 index 000000000..2a40d3b59 --- /dev/null +++ b/test/cli/testdata/malformed/list-for-scalar/Cargo.lock @@ -0,0 +1,6 @@ +version = 3 + +[[package]] +name = ["a"] +version = 1 +dependencies = "b" diff --git a/test/cli/testdata/malformed/list-for-scalar/package.json b/test/cli/testdata/malformed/list-for-scalar/package.json new file mode 100644 index 000000000..565908124 --- /dev/null +++ b/test/cli/testdata/malformed/list-for-scalar/package.json @@ -0,0 +1 @@ +{"name": ["a"], "version": {"v": 1}, "license": [1, 2], "author": 7, "dependencies": []} \ No newline at end of file diff --git a/test/cli/testdata/malformed/safetensors-declared-length/model.safetensors b/test/cli/testdata/malformed/safetensors-declared-length/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..45f9a3a1fe7eac16207e32f3ff6e8eda6ed010f1 GIT binary patch literal 10 NcmZQz00W2WS^xq)0XYBw literal 0 HcmV?d00001 diff --git a/test/cli/testdata/malformed/truncated-archive/a-1.0.jar b/test/cli/testdata/malformed/truncated-archive/a-1.0.jar new file mode 100644 index 0000000000000000000000000000000000000000..e87a7ecfd1b421560e6effd93d5d72ef45beab42 GIT binary patch literal 388 zcmWIWW@Zs#0DU#RQ>H9kRdAhjSt%Im8E|>#7UZPnrskC-mH@?cLo!QpQmqt9(8W>o1fbZxbNAs+WuToP ztcYxPZem$#o_@N1qJBYsu3kY=enDzcNoHzsdQpC9foFGj9I@tpS4uMi7O~x9BFIyAqX(v`W}b_ E0LX=4)c^nh literal 0 HcmV?d00001 diff --git a/test/cli/testdata/malformed/truncated-lockfile/package-lock.json b/test/cli/testdata/malformed/truncated-lockfile/package-lock.json new file mode 100644 index 000000000..a1d749177 --- /dev/null +++ b/test/cli/testdata/malformed/truncated-lockfile/package-lock.json @@ -0,0 +1,14 @@ +{ + "name": "t", + "version": "1.0.0", + "lockfileVersion": 3, + "packages": { + "": { + "name": "t", + "version": "1.0.0" + }, + "node_modules/a": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/a/-/a-1.0.0.tgz" + } + } diff --git a/test/cli/testdata/malformed/yaml-deep-nesting/pnpm-lock.yaml b/test/cli/testdata/malformed/yaml-deep-nesting/pnpm-lock.yaml new file mode 100644 index 000000000..c9429fe0f --- /dev/null +++ b/test/cli/testdata/malformed/yaml-deep-nesting/pnpm-lock.yaml @@ -0,0 +1,3 @@ +lockfileVersion: '9.0' +packages: + /a@1.0.0: [[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]] diff --git a/test/cli/testdata/malformed/yaml-merge-keys/pnpm-lock.yaml b/test/cli/testdata/malformed/yaml-merge-keys/pnpm-lock.yaml new file mode 100644 index 000000000..d7c514726 --- /dev/null +++ b/test/cli/testdata/malformed/yaml-merge-keys/pnpm-lock.yaml @@ -0,0 +1,8 @@ +lockfileVersion: '9.0' +packages: + /a@1.0.0: &p + resolution: {integrity: sha512-x} + /b@1.0.0: + <<: *p + /c@1.0.0: + <<: [*p, {<<: *p}] From 1dc16fbac463aea94a4d456fb1dacb8ea21bf47b Mon Sep 17 00:00:00 2001 From: Alex Goodman Date: Tue, 29 Sep 2026 16:52:58 -0400 Subject: [PATCH 29/40] fix: report scanner errors and raise the line cap in metadata parsers (#5353) * fix: report scanner errors instead of returning a partial parse as complete A line longer than `bufio.Scanner`'s 64KB buffer ends the scan with `ErrTooLong`, which only `Err()` reports. Parsers now return what they found along with the error (recorded as an unknown for the file), or log it where there is no error to return. Signed-off-by: Alex Goodman * fix: allow lines up to 1MB in line-oriented package metadata parsers Adds `internal.NewLineScanner`, a `bufio.Scanner` capped at 1MB per line rather than the 64KB default, the same cap the alpm parser already uses. Long single-line fields (descriptions, license text, dependency lists) now parse instead of ending the scan early. Signed-off-by: Alex Goodman * fix: use io.ReadFull where a short read was silently accepted - dotnet bundle 7-bit int decoding could read a zero-byte result as a zero byte - wordpress plugin header parsing included the unread tail of its buffer Signed-off-by: Alex Goodman * fix: finish moving line scanners to the 1MB cap and checking scanner errors - java manifest, pom.properties, Gemfile.lock, requirements.txt, homebrew formula, alpm and the purl/cpe decoders now use `internal.NewLineScanner` - snap kernel changelog reports a scan error instead of silently dropping the base kernel package - the attestation output reader keeps draining the pipe after a scan error so cosign can't block on a full pipe Signed-off-by: Alex Goodman * fix: stop logging the full setup.py line once per dependency match The debug log ran once per regex match and included the whole line, so log volume grew with the square of line length. With 1MB lines that is hundreds of GB of debug output from a single crafted file. Signed-off-by: Alex Goodman * fix(gentoo): don't dereference a nil location when a LICENSE scan fails `extractLicenses` accepts a nil location, but the scan error log added earlier on this branch dereferenced it. Signed-off-by: Alex Goodman * fix(gentoo): bound license group expansion on hostile license_groups - dedup within a group uses a set instead of `slices.Contains`, which was quadratic in tokens per line and got much worse under the 1MB line cap - each group is expanded once and memoized, so a chain of groups that reference each other repeatedly no longer expands exponentially - repeated references to the same group within one line are skipped Signed-off-by: Alex Goodman --------- Signed-off-by: Alex Goodman --- cmd/syft/cli/ui/handle_attestation.go | 5 + internal/line_scanner.go | 19 +++ internal/line_scanner_test.go | 24 +++ syft/format/cpes/decoder.go | 6 +- syft/format/purls/decoder.go | 6 +- syft/pkg/cataloger/arch/parse_alpm_db.go | 7 +- syft/pkg/cataloger/debian/parse_copyright.go | 7 +- syft/pkg/cataloger/gentoo/license.go | 151 +++++++++--------- syft/pkg/cataloger/gentoo/license_test.go | 29 ++++ .../gentoo/parse_portage_contents.go | 11 +- syft/pkg/cataloger/golang/parse_go_mod.go | 6 +- .../homebrew/parse_homebrew_formula.go | 4 +- .../internal/dotnet/bundle/bundle.go | 2 +- .../cataloger/java/parse_gradle_lockfile.go | 9 +- .../java/parse_gradle_lockfile_test.go | 18 +++ .../pkg/cataloger/java/parse_java_manifest.go | 4 +- .../cataloger/java/parse_pom_properties.go | 4 +- syft/pkg/cataloger/nix/db_cataloger.go | 2 +- .../cataloger/python/parse_requirements.go | 3 +- syft/pkg/cataloger/python/parse_setup.go | 21 ++- .../python/parse_wheel_egg_metadata.go | 8 +- syft/pkg/cataloger/r/parse_description.go | 8 +- syft/pkg/cataloger/ruby/parse_gemfile_lock.go | 4 +- syft/pkg/cataloger/ruby/parse_gemspec.go | 7 +- .../cataloger/snap/parse_kernel_changelog.go | 7 +- syft/pkg/cataloger/wordpress/parse_plugin.go | 9 +- .../cataloger/wordpress/parse_plugin_test.go | 19 +++ 27 files changed, 280 insertions(+), 120 deletions(-) create mode 100644 internal/line_scanner.go create mode 100644 internal/line_scanner_test.go diff --git a/cmd/syft/cli/ui/handle_attestation.go b/cmd/syft/cli/ui/handle_attestation.go index 838d77c89..4c44410b5 100644 --- a/cmd/syft/cli/ui/handle_attestation.go +++ b/cmd/syft/cli/ui/handle_attestation.go @@ -158,6 +158,11 @@ func (l *backgroundLineReader) read(reader io.Reader, stage *progress.Stage) { l.lock.Unlock() } + if err := s.Err(); err != nil { + log.WithFields("error", err).Debug("unable to read attestation output") + // keep draining so the writer never blocks on a full pipe + _, _ = io.Copy(io.Discard, reader) + } } func (l backgroundLineReader) Lines() []string { diff --git a/internal/line_scanner.go b/internal/line_scanner.go new file mode 100644 index 000000000..ad4da5e29 --- /dev/null +++ b/internal/line_scanner.go @@ -0,0 +1,19 @@ +package internal + +import ( + "bufio" + "io" +) + +// maxScannedLineSize is the longest line NewLineScanner will return before Scan stops with bufio.ErrTooLong. +// The buffer only grows to this size when a line needs it. +const maxScannedLineSize = 1024 * 1024 + +// NewLineScanner returns a bufio.Scanner that accepts lines up to 1MB rather than the 64KB default, since +// package metadata files carry long single-line fields (descriptions, license text, dependency lists). Callers +// must still check Err() after scanning. +func NewLineScanner(r io.Reader) *bufio.Scanner { + s := bufio.NewScanner(r) + s.Buffer(nil, maxScannedLineSize) + return s +} diff --git a/internal/line_scanner_test.go b/internal/line_scanner_test.go new file mode 100644 index 000000000..f29f0928d --- /dev/null +++ b/internal/line_scanner_test.go @@ -0,0 +1,24 @@ +package internal + +import ( + "bufio" + "strings" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestNewLineScanner(t *testing.T) { + long := strings.Repeat("a", 100*1024) + s := NewLineScanner(strings.NewReader(long + "\nb\n")) + require.True(t, s.Scan()) + require.Equal(t, long, s.Text()) + require.True(t, s.Scan()) + require.Equal(t, "b", s.Text()) + require.False(t, s.Scan()) + require.NoError(t, s.Err()) + + s = NewLineScanner(strings.NewReader(strings.Repeat("a", maxScannedLineSize+1))) + require.False(t, s.Scan()) + require.ErrorIs(t, s.Err(), bufio.ErrTooLong) +} diff --git a/syft/format/cpes/decoder.go b/syft/format/cpes/decoder.go index b5d3ab62e..9063f6fae 100644 --- a/syft/format/cpes/decoder.go +++ b/syft/format/cpes/decoder.go @@ -7,6 +7,7 @@ import ( "io" "strings" + syftinternal "github.com/anchore/syft/internal" "github.com/anchore/syft/internal/log" "github.com/anchore/syft/syft/cpe" "github.com/anchore/syft/syft/format/internal" @@ -61,7 +62,7 @@ func toSyftModel(r io.Reader) (*sbom.SBOM, error) { var errs []error pkgs := pkg.NewCollection() - scanner := bufio.NewScanner(r) + scanner := syftinternal.NewLineScanner(r) for scanner.Scan() { line := strings.TrimSpace(scanner.Text()) @@ -86,6 +87,9 @@ func toSyftModel(r io.Reader) (*sbom.SBOM, error) { p.SetID() pkgs.Add(p) } + if err := scanner.Err(); err != nil { + errs = append(errs, err) + } return &sbom.SBOM{ Artifacts: sbom.Artifacts{ diff --git a/syft/format/purls/decoder.go b/syft/format/purls/decoder.go index 9cce2dcf5..89e37f739 100644 --- a/syft/format/purls/decoder.go +++ b/syft/format/purls/decoder.go @@ -8,6 +8,7 @@ import ( "strings" "github.com/anchore/packageurl-go" + syftinternal "github.com/anchore/syft/internal" "github.com/anchore/syft/internal/log" "github.com/anchore/syft/syft/format/internal" "github.com/anchore/syft/syft/pkg" @@ -61,7 +62,7 @@ func toSyftModel(r io.Reader) (*sbom.SBOM, error) { var errs []error pkgs := pkg.NewCollection() - scanner := bufio.NewScanner(r) + scanner := syftinternal.NewLineScanner(r) for scanner.Scan() { line := strings.TrimSpace(scanner.Text()) @@ -84,6 +85,9 @@ func toSyftModel(r io.Reader) (*sbom.SBOM, error) { p.SetID() pkgs.Add(p) } + if err := scanner.Err(); err != nil { + errs = append(errs, err) + } return &sbom.SBOM{ Artifacts: sbom.Artifacts{ diff --git a/syft/pkg/cataloger/arch/parse_alpm_db.go b/syft/pkg/cataloger/arch/parse_alpm_db.go index fe9762087..db7505be6 100644 --- a/syft/pkg/cataloger/arch/parse_alpm_db.go +++ b/syft/pkg/cataloger/arch/parse_alpm_db.go @@ -148,12 +148,7 @@ func parseAlpmDBEntry(reader io.Reader) (*parsedData, error) { } func newScanner(reader io.Reader) *bufio.Scanner { - // This is taken from the apk parser - // https://github.com/anchore/syft/blob/v0.47.0/syft/pkg/cataloger/apkdb/parse_apk_db.go#L37 - const maxScannerCapacity = 1024 * 1024 - bufScan := make([]byte, maxScannerCapacity) - scanner := bufio.NewScanner(reader) - scanner.Buffer(bufScan, maxScannerCapacity) + scanner := internal.NewLineScanner(reader) onDoubleLF := func(data []byte, atEOF bool) (advance int, token []byte, err error) { for i := range data { if i > 0 && data[i-1] == '\n' && data[i] == '\n' { diff --git a/syft/pkg/cataloger/debian/parse_copyright.go b/syft/pkg/cataloger/debian/parse_copyright.go index e6936ce2d..0b5e86647 100644 --- a/syft/pkg/cataloger/debian/parse_copyright.go +++ b/syft/pkg/cataloger/debian/parse_copyright.go @@ -1,7 +1,6 @@ package debian import ( - "bufio" "io" "regexp" "sort" @@ -10,6 +9,7 @@ import ( "github.com/scylladb/go-set/strset" "github.com/anchore/syft/internal" + "github.com/anchore/syft/internal/log" ) // For more information see: https://www.debian.org/doc/packaging-manuals/copyright-format/1.0/#license-syntax @@ -34,7 +34,7 @@ const ( func parseLicensesFromCopyright(reader io.Reader) []string { findings := strset.New() - scanner := bufio.NewScanner(reader) + scanner := internal.NewLineScanner(reader) // per the DEP-5 spec, machine-readable copyright files MUST have a // Format field whose value is a URI for the specification. Only files @@ -69,6 +69,9 @@ func parseLicensesFromCopyright(reader io.Reader) []string { findings.Add(found) } } + if err := scanner.Err(); err != nil { + log.WithFields("error", err).Debug("failed to fully read debian copyright file") + } if !formatVerified { return nil diff --git a/syft/pkg/cataloger/gentoo/license.go b/syft/pkg/cataloger/gentoo/license.go index ee23c0372..4fedf5938 100644 --- a/syft/pkg/cataloger/gentoo/license.go +++ b/syft/pkg/cataloger/gentoo/license.go @@ -5,7 +5,6 @@ import ( "bytes" "fmt" "io" - "slices" "strings" "github.com/scylladb/go-set/strset" @@ -35,7 +34,7 @@ import ( func extractLicenses(resolver file.Resolver, closestLocation *file.Location, reader io.Reader) (string, string) { findings := strset.New() contentsWriter := bytes.Buffer{} - scanner := bufio.NewScanner(io.TeeReader(reader, &contentsWriter)) + scanner := internal.NewLineScanner(io.TeeReader(reader, &contentsWriter)) scanner.Split(bufio.ScanWords) var ( mandatoryLicenses, conditionalLicenses, useflagLicenses []string @@ -69,6 +68,13 @@ func extractLicenses(resolver file.Resolver, closestLocation *file.Location, rea } } } + if err := scanner.Err(); err != nil { + fields := []any{"error", err} + if closestLocation != nil { + fields = append(fields, "path", closestLocation.RealPath) + } + log.WithFields(fields...).Debug("failed to fully read portage LICENSE") + } var licenseGroups map[string][]string if usesGroups { @@ -80,31 +86,22 @@ func extractLicenses(resolver file.Resolver, closestLocation *file.Location, rea findings.Add(conditionalLicenses...) findings.Add(useflagLicenses...) - var mandatoryStatement, conditionalStatement string + return strings.TrimSpace(contentsWriter.String()), licenseExpression(mandatoryLicenses, conditionalLicenses) +} - // attempt to build valid SPDX license expression - if len(mandatoryLicenses) > 0 { - mandatoryStatement = strings.Join(mandatoryLicenses, " AND ") +// licenseExpression attempts to build a valid SPDX license expression +func licenseExpression(mandatoryLicenses, conditionalLicenses []string) string { + mandatoryStatement := strings.Join(mandatoryLicenses, " AND ") + conditionalStatement := strings.Join(conditionalLicenses, " OR ") + + switch { + case mandatoryStatement != "" && conditionalStatement != "": + return mandatoryStatement + " AND (" + conditionalStatement + ")" + case mandatoryStatement != "": + return mandatoryStatement + default: + return conditionalStatement } - if len(conditionalLicenses) > 0 { - conditionalStatement = strings.Join(conditionalLicenses, " OR ") - } - - contents := strings.TrimSpace(contentsWriter.String()) - - if mandatoryStatement != "" && conditionalStatement != "" { - return contents, mandatoryStatement + " AND (" + conditionalStatement + ")" - } - - if mandatoryStatement != "" { - return contents, mandatoryStatement - } - - if conditionalStatement != "" { - return contents, conditionalStatement - } - - return contents, "" } func readLicenseGroups(resolver file.Resolver, closestLocation *file.Location) map[string][]string { @@ -161,10 +158,9 @@ func replaceLicenseGroups(licenses []string, groups map[string][]string) []strin } func parseLicenseGroups(reader io.Reader) (map[string][]string, error) { - result := make(map[string][]string) rawGroups := make(map[string][]string) - scanner := bufio.NewScanner(reader) + scanner := internal.NewLineScanner(reader) // first collect all raw groups for scanner.Scan() { @@ -191,58 +187,69 @@ func parseLicenseGroups(reader io.Reader) (map[string][]string, error) { } // next process each group to expand nested references - for groupName, licenses := range rawGroups { - expanded, err := expandLicenses(groupName, licenses, rawGroups, make(map[string]bool)) + expanded := make(map[string][]string) + for groupName := range rawGroups { + if _, err := expandLicenses(groupName, rawGroups, expanded, make(map[string]bool)); err != nil { + return nil, err + } + } + + return expanded, nil +} + +// expandLicenses handles the recursive expansion of license groups. Each group is expanded once and memoized in +// 'expanded' (the file is untrusted, so re-expanding on every reference would be exponential), and 'visiting' holds +// the groups on the current path to detect cycles. We are always in terms of slices instead of sets to ensure +// original ordering is preserved. +func expandLicenses(currentGroup string, rawGroups, expanded map[string][]string, visiting map[string]bool) ([]string, error) { + if result, ok := expanded[currentGroup]; ok { + return result, nil + } + if visiting[currentGroup] { + return nil, fmt.Errorf("cycle detected in license group definitions for group: %s", currentGroup) + } + visiting[currentGroup] = true + defer delete(visiting, currentGroup) + + result := make([]string, 0) + // sets keep dedup linear, since a single group line can hold hundreds of thousands of tokens + seen := strset.New() + seenRefs := strset.New() + + for _, item := range rawGroups[currentGroup] { + refGroupName, isRef := strings.CutPrefix(item, "@") + if !isRef { + // this is a regular license + if !seen.Has(item) { + seen.Add(item) + result = append(result, item) + } + continue + } + + // this is a reference to another group + if seenRefs.Has(refGroupName) { + continue + } + seenRefs.Add(refGroupName) + + if _, exists := rawGroups[refGroupName]; !exists { + return nil, fmt.Errorf("referenced group not found: %s", refGroupName) + } + + refLicenses, err := expandLicenses(refGroupName, rawGroups, expanded, visiting) if err != nil { return nil, err } - result[groupName] = expanded - } - return result, nil -} - -// expandLicenses handles the recursive expansion of license groups, 'visited' is used to detect cycles. We are always -// in terms of slices instead of sets to ensure original ordering is preserved. -func expandLicenses(currentGroup string, licenses []string, rawGroups map[string][]string, visited map[string]bool) ([]string, error) { - if visited[currentGroup] { - return nil, fmt.Errorf("cycle detected in license group definitions for group: %s", currentGroup) - } - - visited[currentGroup] = true - - result := make([]string, 0) - - for _, item := range licenses { - if strings.HasPrefix(item, "@") { - // this is a reference to another group - refGroupName := item[1:] // remove '@' prefix - - refLicenses, exists := rawGroups[refGroupName] - if !exists { - return nil, fmt.Errorf("referenced group not found: %s", refGroupName) + for _, license := range refLicenses { + if !seen.Has(license) { + seen.Add(license) + result = append(result, license) } - - newVisited := make(map[string]bool) - for k, v := range visited { - newVisited[k] = v - } - - expanded, err := expandLicenses(refGroupName, refLicenses, rawGroups, newVisited) - if err != nil { - return nil, err - } - - for _, license := range expanded { - if !slices.Contains(result, license) { - result = append(result, license) - } - } - } else if !slices.Contains(result, item) { - // ...this is a regular license - result = append(result, item) } } + expanded[currentGroup] = result return result, nil } diff --git a/syft/pkg/cataloger/gentoo/license_test.go b/syft/pkg/cataloger/gentoo/license_test.go index e533c2636..47b0cfda7 100644 --- a/syft/pkg/cataloger/gentoo/license_test.go +++ b/syft/pkg/cataloger/gentoo/license_test.go @@ -2,6 +2,7 @@ package gentoo import ( "bytes" + "fmt" "os" "strings" "testing" @@ -181,3 +182,31 @@ func TestReplaceLicenseGroups(t *testing.T) { }) } } + +func Test_extractLicenses_tokenTooLong(t *testing.T) { + // a nil location must not panic when the scan fails partway through + license := "MIT " + strings.Repeat("a", 2<<20) + _, expression := extractLicenses(nil, nil, strings.NewReader(license)) + assert.Equal(t, "MIT", expression) +} + +func TestParseLicenseGroups_hostileInput(t *testing.T) { + // a single wide group was quadratic to dedup, and a chain where each group references the next + // twice was exponential to expand; both must now finish quickly + var wide strings.Builder + wide.WriteString("WIDE") + for i := range 120_000 { + fmt.Fprintf(&wide, " L%d", i) + } + + var chain strings.Builder + chain.WriteString("G0 MIT\n") + for i := 1; i <= 64; i++ { + fmt.Fprintf(&chain, "G%d @G%d @G%d\n", i, i-1, i-1) + } + + groups, err := parseLicenseGroups(strings.NewReader(wide.String() + "\n" + chain.String())) + require.NoError(t, err) + assert.Len(t, groups["WIDE"], 120_000) + assert.Equal(t, []string{"MIT"}, groups["G64"]) +} diff --git a/syft/pkg/cataloger/gentoo/parse_portage_contents.go b/syft/pkg/cataloger/gentoo/parse_portage_contents.go index 7d4d1d8c2..db9d68035 100644 --- a/syft/pkg/cataloger/gentoo/parse_portage_contents.go +++ b/syft/pkg/cataloger/gentoo/parse_portage_contents.go @@ -1,7 +1,6 @@ package gentoo import ( - "bufio" "context" "fmt" "path" @@ -70,7 +69,7 @@ func addFiles(resolver file.Resolver, dbLocation file.Location, entry *pkg.Porta } defer internal.CloseAndLogError(contentsReader, dbLocation.RealPath) - scanner := bufio.NewScanner(contentsReader) + scanner := internal.NewLineScanner(contentsReader) for scanner.Scan() { line := strings.Trim(scanner.Text(), "\n") fields := strings.Split(line, " ") @@ -87,6 +86,9 @@ func addFiles(resolver file.Resolver, dbLocation file.Location, entry *pkg.Porta entry.Files = append(entry.Files, record) } } + if err := scanner.Err(); err != nil { + log.WithFields("path", dbLocation.RealPath, "error", err).Debug("failed to fully read portage contents") + } } func addLicenses(ctx context.Context, resolver file.Resolver, dbLocation file.Location, entry *pkg.PortageEntry) (pkg.LicenseSet, []file.Location) { @@ -128,7 +130,7 @@ func addSize(resolver file.Resolver, dbLocation file.Location, entry *pkg.Portag } defer internal.CloseAndLogError(sizeReader, location.RealPath) - scanner := bufio.NewScanner(sizeReader) + scanner := internal.NewLineScanner(sizeReader) for scanner.Scan() { line := strings.Trim(scanner.Text(), "\n") size, err := strconv.Atoi(line) @@ -136,6 +138,9 @@ func addSize(resolver file.Resolver, dbLocation file.Location, entry *pkg.Portag entry.InstalledSize = size } } + if err := scanner.Err(); err != nil { + log.WithFields("path", location.RealPath, "error", err).Debug("failed to fully read portage SIZE") + } return []file.Location{location.WithAnnotation(pkg.EvidenceAnnotationKey, pkg.SupportingEvidenceAnnotation)} } diff --git a/syft/pkg/cataloger/golang/parse_go_mod.go b/syft/pkg/cataloger/golang/parse_go_mod.go index 4a40403b1..e9d021729 100644 --- a/syft/pkg/cataloger/golang/parse_go_mod.go +++ b/syft/pkg/cataloger/golang/parse_go_mod.go @@ -1,7 +1,6 @@ package golang import ( - "bufio" "context" "fmt" "go/build" @@ -437,7 +436,7 @@ func parseGoSumFile(resolver file.Resolver, reader file.LocationReadCloser) (map // github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= // github.com/BurntSushi/toml v0.4.1 h1:GaI7EiDXDRfa8VshkTj7Fym7ha+y8/XxIgD2okUIjLw= // github.com/BurntSushi/toml v0.4.1/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ= - scanner := bufio.NewScanner(contents) + scanner := internal.NewLineScanner(contents) // optionally, resize scanner's capacity for lines over 64K, see next example for scanner.Scan() { line := scanner.Text() @@ -449,6 +448,9 @@ func parseGoSumFile(resolver file.Resolver, reader file.LocationReadCloser) (map hash := parts[2] out[nameVersion] = hash } + if err := scanner.Err(); err != nil { + return out, fmt.Errorf("unable to read go.sum: %w", err) + } return out, nil } diff --git a/syft/pkg/cataloger/homebrew/parse_homebrew_formula.go b/syft/pkg/cataloger/homebrew/parse_homebrew_formula.go index 1255ccbc6..93d866746 100644 --- a/syft/pkg/cataloger/homebrew/parse_homebrew_formula.go +++ b/syft/pkg/cataloger/homebrew/parse_homebrew_formula.go @@ -1,11 +1,11 @@ package homebrew import ( - "bufio" "context" "path" "strings" + "github.com/anchore/syft/internal" "github.com/anchore/syft/internal/log" "github.com/anchore/syft/syft/artifact" "github.com/anchore/syft/syft/file" @@ -46,7 +46,7 @@ func parseHomebrewFormula(ctx context.Context, resolver file.Resolver, _ *generi func parseFormulaFile(reader file.LocationReadCloser) (*parsedHomebrewData, error) { pd := parsedHomebrewData{} - scanner := bufio.NewScanner(reader) + scanner := internal.NewLineScanner(reader) for scanner.Scan() { line := strings.TrimSpace(scanner.Text()) if strings.Contains(line, "class ") && strings.Contains(line, " < Formula") { diff --git a/syft/pkg/cataloger/internal/dotnet/bundle/bundle.go b/syft/pkg/cataloger/internal/dotnet/bundle/bundle.go index 3bead6fb9..434b1d2ac 100644 --- a/syft/pkg/cataloger/internal/dotnet/bundle/bundle.go +++ b/syft/pkg/cataloger/internal/dotnet/bundle/bundle.go @@ -209,7 +209,7 @@ func read7BitEncodedInt(r io.Reader) (int, error) { shift := 0 for { var b [1]byte - if _, err := r.Read(b[:]); err != nil { + if _, err := io.ReadFull(r, b[:]); err != nil { return 0, err } result |= int(b[0]&0x7F) << shift diff --git a/syft/pkg/cataloger/java/parse_gradle_lockfile.go b/syft/pkg/cataloger/java/parse_gradle_lockfile.go index 4ba86afb3..4e9e8c694 100644 --- a/syft/pkg/cataloger/java/parse_gradle_lockfile.go +++ b/syft/pkg/cataloger/java/parse_gradle_lockfile.go @@ -1,10 +1,11 @@ package java import ( - "bufio" "context" + "fmt" "strings" + "github.com/anchore/syft/internal" "github.com/anchore/syft/syft/artifact" "github.com/anchore/syft/syft/file" "github.com/anchore/syft/syft/pkg" @@ -22,7 +23,7 @@ func parseGradleLockfile(_ context.Context, _ file.Resolver, _ *generic.Environm var pkgs []pkg.Package // Create a new scanner to read the file - scanner := bufio.NewScanner(reader) + scanner := internal.NewLineScanner(reader) // Create slices to hold the dependencies and plugins dependencies := []lockfileDependency{} @@ -45,6 +46,7 @@ func parseGradleLockfile(_ context.Context, _ file.Resolver, _ *generic.Environm dependencies = append(dependencies, dep) } } + scanErr := scanner.Err() // map the dependencies for _, dep := range dependencies { @@ -72,5 +74,8 @@ func parseGradleLockfile(_ context.Context, _ file.Resolver, _ *generic.Environm pkgs = append(pkgs, mappedPkg) } + if scanErr != nil { + return pkgs, nil, fmt.Errorf("unable to read gradle lockfile: %w", scanErr) + } return pkgs, nil, nil } diff --git a/syft/pkg/cataloger/java/parse_gradle_lockfile_test.go b/syft/pkg/cataloger/java/parse_gradle_lockfile_test.go index d6fc4c2a8..a9208a1b8 100644 --- a/syft/pkg/cataloger/java/parse_gradle_lockfile_test.go +++ b/syft/pkg/cataloger/java/parse_gradle_lockfile_test.go @@ -1,8 +1,15 @@ package java import ( + "bufio" + "context" + "io" + "strings" "testing" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/anchore/syft/syft/file" "github.com/anchore/syft/syft/pkg" "github.com/anchore/syft/syft/pkg/cataloger/internal/pkgtest" @@ -69,3 +76,14 @@ func Test_parserGradleLockfile(t *testing.T) { }) } } + +func Test_parserGradleLockfile_lineTooLong(t *testing.T) { + // packages before an oversized line are kept, and the scan error is still reported + contents := "org.example:before:1.0=compileClasspath\n" + strings.Repeat("a", 2<<20) + "\norg.example:after:2.0=compileClasspath\n" + reader := file.NewLocationReadCloser(file.NewLocation("gradle.lockfile"), io.NopCloser(strings.NewReader(contents))) + + pkgs, _, err := parseGradleLockfile(context.Background(), nil, nil, reader) + require.ErrorIs(t, err, bufio.ErrTooLong) + require.Len(t, pkgs, 1) + assert.Equal(t, "before", pkgs[0].Name) +} diff --git a/syft/pkg/cataloger/java/parse_java_manifest.go b/syft/pkg/cataloger/java/parse_java_manifest.go index cb7945eca..0c1e3129a 100644 --- a/syft/pkg/cataloger/java/parse_java_manifest.go +++ b/syft/pkg/cataloger/java/parse_java_manifest.go @@ -1,12 +1,12 @@ package java import ( - "bufio" "fmt" "io" "strings" "unicode" + "github.com/anchore/syft/internal" "github.com/anchore/syft/internal/log" "github.com/anchore/syft/syft/pkg" ) @@ -26,7 +26,7 @@ func parseJavaManifest(path string, reader io.Reader) (*pkg.JavaManifest, error) } var lastKey string - scanner := bufio.NewScanner(reader) + scanner := internal.NewLineScanner(reader) for scanner.Scan() { line := scanner.Text() diff --git a/syft/pkg/cataloger/java/parse_pom_properties.go b/syft/pkg/cataloger/java/parse_pom_properties.go index 8ef3676be..b4704b099 100644 --- a/syft/pkg/cataloger/java/parse_pom_properties.go +++ b/syft/pkg/cataloger/java/parse_pom_properties.go @@ -1,13 +1,13 @@ package java import ( - "bufio" "fmt" "io" "strings" "github.com/go-viper/mapstructure/v2" + "github.com/anchore/syft/internal" "github.com/anchore/syft/syft/pkg" ) @@ -16,7 +16,7 @@ const pomPropertiesGlob = "**/*pom.properties" func parsePomProperties(path string, reader io.Reader) (*pkg.JavaPomProperties, error) { var props pkg.JavaPomProperties propMap := make(map[string]string) - scanner := bufio.NewScanner(reader) + scanner := internal.NewLineScanner(reader) for scanner.Scan() { line := scanner.Text() diff --git a/syft/pkg/cataloger/nix/db_cataloger.go b/syft/pkg/cataloger/nix/db_cataloger.go index 615e213ef..825895b69 100644 --- a/syft/pkg/cataloger/nix/db_cataloger.go +++ b/syft/pkg/cataloger/nix/db_cataloger.go @@ -95,7 +95,7 @@ func (c dbCataloger) selectDBParser(dbLocation file.Location, resolver file.Reso scanner := bufio.NewScanner(schemaContents) if !scanner.Scan() { - log.WithFields("path", loc.RealPath).Tracef("failed to read Nix database schema file, assuming %d", defaultSchema) + log.WithFields("path", loc.RealPath, "error", scanner.Err()).Tracef("failed to read Nix database schema file, assuming %d", defaultSchema) return c.schemaProcessor[defaultSchema], 0 } diff --git a/syft/pkg/cataloger/python/parse_requirements.go b/syft/pkg/cataloger/python/parse_requirements.go index d167c9d16..6622496fd 100644 --- a/syft/pkg/cataloger/python/parse_requirements.go +++ b/syft/pkg/cataloger/python/parse_requirements.go @@ -1,7 +1,6 @@ package python import ( - "bufio" "context" "fmt" "regexp" @@ -105,7 +104,7 @@ func (rp requirementsParser) parseRequirementsTxt(ctx context.Context, _ file.Re var errs error var packages []pkg.Package - scanner := bufio.NewScanner(reader) + scanner := internal.NewLineScanner(reader) var lastLine string for scanner.Scan() { line := trimRequirementsTxtLine(scanner.Text()) diff --git a/syft/pkg/cataloger/python/parse_setup.go b/syft/pkg/cataloger/python/parse_setup.go index a42c785ac..d8cff06de 100644 --- a/syft/pkg/cataloger/python/parse_setup.go +++ b/syft/pkg/cataloger/python/parse_setup.go @@ -1,11 +1,12 @@ package python import ( - "bufio" "context" + "fmt" "regexp" "strings" + "github.com/anchore/syft/internal" "github.com/anchore/syft/internal/log" "github.com/anchore/syft/syft/artifact" "github.com/anchore/syft/syft/file" @@ -36,7 +37,7 @@ var unquotedPinnedDependency = regexp.MustCompile(`^\s*(\w+)\s*==\s*([\w\.\-]+)` func (sp setupFileParser) parseSetupFile(ctx context.Context, _ file.Resolver, _ *generic.Environment, reader file.LocationReadCloser) ([]pkg.Package, []artifact.Relationship, error) { var packages []pkg.Package - scanner := bufio.NewScanner(reader) + scanner := internal.NewLineScanner(reader) for scanner.Scan() { line := scanner.Text() @@ -45,20 +46,23 @@ func (sp setupFileParser) parseSetupFile(ctx context.Context, _ file.Resolver, _ packages = sp.processQuotedDependencies(ctx, line, reader, packages) packages = sp.processUnquotedDependency(ctx, line, reader, packages) } + if err := scanner.Err(); err != nil { + return packages, nil, fmt.Errorf("unable to read setup.py: %w", err) + } return packages, nil, nil } func (sp setupFileParser) processQuotedDependencies(ctx context.Context, line string, reader file.LocationReadCloser, packages []pkg.Package) []pkg.Package { for _, match := range pinnedDependency.FindAllString(line, -1) { - if p, ok := sp.parseQuotedDependency(ctx, match, line, reader); ok { + if p, ok := sp.parseQuotedDependency(ctx, match, reader); ok { packages = append(packages, p) } } return packages } -func (sp setupFileParser) parseQuotedDependency(ctx context.Context, match, line string, reader file.LocationReadCloser) (pkg.Package, bool) { +func (sp setupFileParser) parseQuotedDependency(ctx context.Context, match string, reader file.LocationReadCloser) (pkg.Package, bool) { parts := strings.Split(match, "==") if len(parts) != 2 { return pkg.Package{}, false @@ -67,7 +71,7 @@ func (sp setupFileParser) parseQuotedDependency(ctx context.Context, match, line name := cleanDependencyString(parts[0]) version := cleanDependencyString(parts[len(parts)-1]) - return sp.validateAndCreatePackage(ctx, name, version, line, reader) + return sp.validateAndCreatePackage(ctx, name, version, reader) } // processUnquotedDependency extracts and processes an unquoted dependency from a line @@ -80,7 +84,7 @@ func (sp setupFileParser) processUnquotedDependency(ctx context.Context, line st name := strings.TrimSpace(matches[1]) version := strings.TrimSpace(matches[2]) - if p, ok := sp.validateAndCreatePackage(ctx, name, version, line, reader); ok { + if p, ok := sp.validateAndCreatePackage(ctx, name, version, reader); ok { if !isDuplicatePackage(p, packages) { packages = append(packages, p) } @@ -96,14 +100,15 @@ func cleanDependencyString(s string) string { return s } -func (sp setupFileParser) validateAndCreatePackage(ctx context.Context, name, version, line string, reader file.LocationReadCloser) (pkg.Package, bool) { +func (sp setupFileParser) validateAndCreatePackage(ctx context.Context, name, version string, reader file.LocationReadCloser) (pkg.Package, bool) { if hasTemplateDirective(name) || hasTemplateDirective(version) { // this can happen in more dynamic setup.py where there is templating return pkg.Package{}, false } if name == "" || version == "" { - log.WithFields("path", reader.RealPath).Debugf("unable to parse package in setup.py line: %q", line) + // the raw line is not logged since this runs once per match, which is quadratic on a long line + log.WithFields("path", reader.RealPath, "name", name, "version", version).Debug("unable to parse package in setup.py") return pkg.Package{}, false } diff --git a/syft/pkg/cataloger/python/parse_wheel_egg_metadata.go b/syft/pkg/cataloger/python/parse_wheel_egg_metadata.go index 0cb060a23..5b0572e3a 100644 --- a/syft/pkg/cataloger/python/parse_wheel_egg_metadata.go +++ b/syft/pkg/cataloger/python/parse_wheel_egg_metadata.go @@ -1,7 +1,6 @@ package python import ( - "bufio" "fmt" "path/filepath" "strings" @@ -9,6 +8,7 @@ import ( "github.com/bmatcuk/doublestar/v4" "github.com/go-viper/mapstructure/v2" + "github.com/anchore/syft/internal" "github.com/anchore/syft/internal/log" "github.com/anchore/syft/syft/file" "github.com/anchore/syft/syft/pkg" @@ -68,7 +68,7 @@ func extractRFC5322Fields(locationReader file.LocationReadCloser) (map[string]an // though this spec is governed by RFC 5322 (mail message), the metadata files are not guaranteed to be compliant. // We must survive parsing as much info as possible without failing and dropping the data. - scanner := bufio.NewScanner(locationReader) + scanner := internal.NewLineScanner(locationReader) for scanner.Scan() { line := scanner.Text() line = strings.TrimRight(line, "\n") @@ -108,6 +108,10 @@ func extractRFC5322Fields(locationReader file.LocationReadCloser) (map[string]an } } } + if err := scanner.Err(); err != nil { + // the caller discards everything on error, so keep the partial fields and log instead + log.WithFields("path", locationReader.Path(), "error", err).Debug("failed to fully read python wheel/egg metadata") + } return fields, nil } diff --git a/syft/pkg/cataloger/r/parse_description.go b/syft/pkg/cataloger/r/parse_description.go index 01aa89f99..340e395ec 100644 --- a/syft/pkg/cataloger/r/parse_description.go +++ b/syft/pkg/cataloger/r/parse_description.go @@ -1,12 +1,13 @@ package r import ( - "bufio" "context" "io" "regexp" "strings" + "github.com/anchore/syft/internal" + "github.com/anchore/syft/internal/log" "github.com/anchore/syft/syft/artifact" "github.com/anchore/syft/syft/file" "github.com/anchore/syft/syft/pkg" @@ -105,7 +106,7 @@ func extractFieldsFromDescriptionFile(reader io.Reader) map[string]string { result := make(map[string]string) key := "" var valueFragment strings.Builder - scanner := bufio.NewScanner(reader) + scanner := internal.NewLineScanner(reader) for scanner.Scan() { line := scanner.Text() @@ -139,6 +140,9 @@ func extractFieldsFromDescriptionFile(reader io.Reader) map[string]string { if key != "" { result[key] = valueFragment.String() } + if err := scanner.Err(); err != nil { + log.WithFields("error", err).Debug("failed to fully read R DESCRIPTION file") + } return result } diff --git a/syft/pkg/cataloger/ruby/parse_gemfile_lock.go b/syft/pkg/cataloger/ruby/parse_gemfile_lock.go index d8fe50850..29bc1cac3 100644 --- a/syft/pkg/cataloger/ruby/parse_gemfile_lock.go +++ b/syft/pkg/cataloger/ruby/parse_gemfile_lock.go @@ -1,12 +1,12 @@ package ruby import ( - "bufio" "context" "strings" "github.com/scylladb/go-set/strset" + "github.com/anchore/syft/internal" "github.com/anchore/syft/internal/unknown" "github.com/anchore/syft/syft/artifact" "github.com/anchore/syft/syft/file" @@ -21,7 +21,7 @@ var sectionsOfInterest = strset.New("GEM", "GIT", "PATH", "PLUGIN SOURCE") // parseGemFileLockEntries is a parser function for Gemfile.lock contents, returning all Gems discovered. func parseGemFileLockEntries(_ context.Context, _ file.Resolver, _ *generic.Environment, reader file.LocationReadCloser) ([]pkg.Package, []artifact.Relationship, error) { var pkgs []pkg.Package - scanner := bufio.NewScanner(reader) + scanner := internal.NewLineScanner(reader) var currentSection string diff --git a/syft/pkg/cataloger/ruby/parse_gemspec.go b/syft/pkg/cataloger/ruby/parse_gemspec.go index 92b586987..144fe7878 100644 --- a/syft/pkg/cataloger/ruby/parse_gemspec.go +++ b/syft/pkg/cataloger/ruby/parse_gemspec.go @@ -1,7 +1,6 @@ package ruby import ( - "bufio" "context" "encoding/json" "fmt" @@ -75,7 +74,7 @@ func processList(s string) []string { func parseGemSpecEntries(ctx context.Context, resolver file.Resolver, _ *generic.Environment, reader file.LocationReadCloser) ([]pkg.Package, []artifact.Relationship, error) { var pkgs []pkg.Package var fields = make(map[string]any) - scanner := bufio.NewScanner(reader) + scanner := internal.NewLineScanner(reader) for scanner.Scan() { line := scanner.Text() @@ -101,6 +100,7 @@ func parseGemSpecEntries(ctx context.Context, resolver file.Resolver, _ *generic } } } + scanErr := scanner.Err() resolveRubyInterpolationsInFields(fields) @@ -121,6 +121,9 @@ func parseGemSpecEntries(ctx context.Context, resolver file.Resolver, _ *generic ) } + if scanErr != nil { + return pkgs, nil, fmt.Errorf("unable to read gemspec: %w", scanErr) + } return pkgs, nil, nil } diff --git a/syft/pkg/cataloger/snap/parse_kernel_changelog.go b/syft/pkg/cataloger/snap/parse_kernel_changelog.go index 5b5a81b4f..7ad37f300 100644 --- a/syft/pkg/cataloger/snap/parse_kernel_changelog.go +++ b/syft/pkg/cataloger/snap/parse_kernel_changelog.go @@ -1,13 +1,13 @@ package snap import ( - "bufio" "compress/gzip" "context" "fmt" "regexp" "strings" + "github.com/anchore/syft/internal" "github.com/anchore/syft/syft/artifact" "github.com/anchore/syft/syft/file" "github.com/anchore/syft/syft/pkg" @@ -32,7 +32,7 @@ func parseKernelChangelog(_ context.Context, _ file.Resolver, _ *generic.Environ } defer gzReader.Close() - scanner := bufio.NewScanner(gzReader) + scanner := internal.NewLineScanner(gzReader) // read the first line to extract kernel version // Format: "linux (5.4.0-195.215) focal; urgency=medium" @@ -65,6 +65,9 @@ func parseKernelChangelog(_ context.Context, _ file.Resolver, _ *generic.Environ break } } + if err := scanner.Err(); err != nil { + return packages, nil, fmt.Errorf("failed to read changelog content: %w", err) + } return packages, nil, nil } diff --git a/syft/pkg/cataloger/wordpress/parse_plugin.go b/syft/pkg/cataloger/wordpress/parse_plugin.go index 4956dc6f8..a3de5060c 100644 --- a/syft/pkg/cataloger/wordpress/parse_plugin.go +++ b/syft/pkg/cataloger/wordpress/parse_plugin.go @@ -2,7 +2,9 @@ package wordpress import ( "context" + "errors" "fmt" + "io" "path/filepath" "regexp" "strings" @@ -42,12 +44,13 @@ func parseWordpressPluginFiles(ctx context.Context, resolver file.Resolver, _ *g var pkgs []pkg.Package buffer := make([]byte, contentBufferSize) - _, err := reader.Read(buffer) - if err != nil { + // a file shorter than the buffer is expected, so only the header it does have is parsed + n, err := io.ReadFull(reader, buffer) + if err != nil && !errors.Is(err, io.ErrUnexpectedEOF) { return nil, nil, fmt.Errorf("failed to read %s file: %w", reader.Path(), err) } - fields := extractFields(string(buffer)) + fields := extractFields(string(buffer[:n])) name, nameOk := fields["name"] version, versionOk := fields["version"] diff --git a/syft/pkg/cataloger/wordpress/parse_plugin_test.go b/syft/pkg/cataloger/wordpress/parse_plugin_test.go index 640f1d42b..97a3941c4 100644 --- a/syft/pkg/cataloger/wordpress/parse_plugin_test.go +++ b/syft/pkg/cataloger/wordpress/parse_plugin_test.go @@ -2,9 +2,13 @@ package wordpress import ( "context" + "io" + "os" "testing" + "testing/iotest" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" "github.com/anchore/syft/syft/file" "github.com/anchore/syft/syft/pkg" @@ -58,3 +62,18 @@ func Test_extractFields(t *testing.T) { }) } } + +func TestParseWordpressPluginFiles_shortReads(t *testing.T) { + // a reader that returns one byte per Read must still yield the full header + fixture := "testdata/glob-paths/wp-content/plugins/akismet/akismet.php" + f, err := os.Open(fixture) + require.NoError(t, err) + t.Cleanup(func() { _ = f.Close() }) + reader := file.NewLocationReadCloser(file.NewLocation(fixture), io.NopCloser(iotest.OneByteReader(f))) + + pkgs, _, err := parseWordpressPluginFiles(context.Background(), nil, nil, reader) + require.NoError(t, err) + require.Len(t, pkgs, 1) + assert.Equal(t, "Akismet Anti-spam: Spam Protection", pkgs[0].Name) + assert.Equal(t, "5.3", pkgs[0].Version) +} From 26927eab0271bf2a08cb332b987d55aec405ea99 Mon Sep 17 00:00:00 2001 From: Pujitha Paladugu Date: Tue, 29 Sep 2026 14:15:36 -0700 Subject: [PATCH 30/40] fix: suppress misleading upstream CPE for PHP extension binaries (#5102) Motivation: syft's PHP interpreter cataloger reports each bundled PHP C extension (openssl, ldap, sqlite3, etc.) as its own package, using the PHP interpreter's own version number as the package version (since these extensions are compiled into a specific PHP release and have no independent version of their own). The classifier already attaches an explicit CPE like cpe:2.3:a:php-openssl:php-openssl:8.5.7:... to make clear this refers to the bundled PHP extension, not upstream OpenSSL. However, that CPE was marked with Source: cpe.GeneratedSource. Syft's package-finalization pipeline (internal/task/package_task_factory.go) only skips its generic, name-based CPE generation when a package already has an "authoritative" CPE (any Source other than GeneratedSource) - so it also generated and appended a second CPE, cpe:2.3:a:openssl:openssl:8.5.7:..., derived from the bare package name. That second CPE is indistinguishable from a real standalone OpenSSL/LDAP/SQLite installation and can cause vulnerability scanners that consume this SBOM to falsely flag PHP hosts as running vulnerable upstream libraries at version "8.5.7" (a PHP version, not a real OpenSSL/LDAP/SQLite release). See anchore/syft#5014 for a full repro. Approach: Mark the classifier's explicit CPE with Source: cpe.DeclaredSource instead of cpe.GeneratedSource, so it is treated as authoritative and the extra name-derived CPE is no longer generated. This mirrors the existing pattern in syft/pkg/cataloger/java/parse_jvm_release.go's newJvmCpe, which uses the same DeclaredSource + comment convention to mark a syft-synthesized (not literally file-declared) CPE as trustworthy and suppress further generation. It also brings this classifier in line with its sibling php-cli/php-fpm/php-apache classifiers in the same file, which already mark their CPEs as authoritative (via NVDDictionaryLookupSource, not applicable here since php- is not a real NVD dictionary entry). This change only suppresses the extra generated CPE; it does not change the package name, type, version, or PURL, and the existing php-:php- CPE is unaffected. Validation: go build ./... go test ./syft/pkg/cataloger/php/... ./internal/task/... ./syft/pkg/cataloger/internal/cpegenerate/... All pass except the two pre-existing docker-image-fixture subtests of Test_InterpreterCataloger, which fail in this sandbox only because the `docker` binary isn't available (exec: "docker": executable file not found in $PATH); confirmed via `git stash` that these two subtests fail identically on main without this change, i.e. this is a sandbox limitation and not a regression. Added a new unit test, Test_getClassifier_declaredCPESource, that directly asserts the classifier's CPE now carries Source: cpe.DeclaredSource. Fixes #5014 Signed-off-by: Pujitha Paladugu <10557236+pujitha24@users.noreply.github.com> Co-authored-by: Pujitha Paladugu <10557236+pujitha24@users.noreply.github.com> --- syft/pkg/cataloger/php/interpreter_cataloger.go | 5 ++++- .../cataloger/php/interpreter_cataloger_test.go | 17 +++++++++++++++++ 2 files changed, 21 insertions(+), 1 deletion(-) diff --git a/syft/pkg/cataloger/php/interpreter_cataloger.go b/syft/pkg/cataloger/php/interpreter_cataloger.go index 80ae69d21..f9022d358 100644 --- a/syft/pkg/cataloger/php/interpreter_cataloger.go +++ b/syft/pkg/cataloger/php/interpreter_cataloger.go @@ -236,7 +236,10 @@ func (p interpreterCataloger) getClassifier(realPath string) (string, *binutils. Vendor: fmt.Sprintf("php-%s", name), Product: fmt.Sprintf("php-%s", name), }, - Source: cpe.GeneratedSource, + // note: we must use a declared source here so that this CPE is treated as authoritative and no + // further CPEs are generated from the (misleading) package name alone, e.g. cpe:...:openssl:openssl + // for the openssl PHP extension, which would imply this is the upstream OpenSSL project. + Source: cpe.DeclaredSource, }, }, } diff --git a/syft/pkg/cataloger/php/interpreter_cataloger_test.go b/syft/pkg/cataloger/php/interpreter_cataloger_test.go index 8d6d9e6d7..bff331d5c 100644 --- a/syft/pkg/cataloger/php/interpreter_cataloger_test.go +++ b/syft/pkg/cataloger/php/interpreter_cataloger_test.go @@ -3,6 +3,10 @@ package php import ( "testing" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/anchore/syft/syft/cpe" "github.com/anchore/syft/syft/pkg/cataloger/internal/pkgtest" ) @@ -147,3 +151,16 @@ func Test_InterpreterCataloger(t *testing.T) { }) } } + +func Test_getClassifier_declaredCPESource(t *testing.T) { + // regression test for https://github.com/anchore/syft/issues/5014 + // the CPE assigned to a PHP extension must be declared (not generated) so that downstream CPE + // generation does not also add a misleading CPE derived from the bare extension name alone + // (e.g. cpe:2.3:a:openssl:openssl:... for the openssl extension, implying the upstream OpenSSL project). + c := interpreterCataloger{name: "php-interpreter-cataloger"} + name, cls := c.getClassifier("/usr/lib/php85/modules/openssl.so") + require.Equal(t, "openssl", name) + require.NotNil(t, cls) + require.Len(t, cls.CPEs, 1) + assert.Equal(t, cpe.DeclaredSource, cls.CPEs[0].Source) +} From 37d45de9fffaac359bea15f450702d0a03a8f545 Mon Sep 17 00:00:00 2001 From: Dev M Date: Wed, 30 Sep 2026 02:53:22 +0530 Subject: [PATCH 31/40] fix(python): link uv.lock dependents to the locked version they name (#5351) * fix(python): link uv.lock dependents to the locked version they name A forked uv.lock can lock several versions of one package, and uv records the version on each dependency entry in that case. The parser dropped that field, so every pandas version depended on every numpy version. Carry the locked version through provides and requires as name@version (bare name stays for unique locks) and keep marker text unchanged. Fixes #5340 Signed-off-by: Dev M * fix(python): pair uv.lock versions without widening public metadata - the locked version on a uv dependency entry is only needed to pair relationships, and the relationships already carry it, so it stays on the internal toml struct instead of `pkg.PythonUvLockDependencyEntry`. The previous approach hid it from JSON but still hashed it into package IDs. - dependency specs are now built from the raw lock entries at parse time rather than read back off package metadata - optional-dependencies get the same version pairing, so an extra that names one locked version no longer links to all of them Signed-off-by: Alex Goodman --------- Signed-off-by: Dev M Signed-off-by: Alex Goodman Co-authored-by: Alex Goodman --- syft/pkg/cataloger/python/dependency.go | 2 +- syft/pkg/cataloger/python/parse_uv_lock.go | 126 ++++++++++-------- .../cataloger/python/parse_uv_lock_test.go | 73 ++++++++++ .../testdata/uv/forked-versions/uv.lock | 54 ++++++++ 4 files changed, 200 insertions(+), 55 deletions(-) create mode 100644 syft/pkg/cataloger/python/testdata/uv/forked-versions/uv.lock diff --git a/syft/pkg/cataloger/python/dependency.go b/syft/pkg/cataloger/python/dependency.go index 50b7d6c4a..29671ed59 100644 --- a/syft/pkg/cataloger/python/dependency.go +++ b/syft/pkg/cataloger/python/dependency.go @@ -15,7 +15,7 @@ import ( "github.com/anchore/syft/syft/pkg/cataloger/internal/dependency" ) -func poetryLockDependencySpecifier(p pkg.Package) dependency.Specification { //nolint:dupl // this is very similar to the uv lock dependency specifier, but should remain separate +func poetryLockDependencySpecifier(p pkg.Package) dependency.Specification { meta, ok := p.Metadata.(pkg.PythonPoetryLockEntry) if !ok { log.Tracef("cataloger failed to extract poetry lock metadata for package %+v", p.Name) diff --git a/syft/pkg/cataloger/python/parse_uv_lock.go b/syft/pkg/cataloger/python/parse_uv_lock.go index 5ac0b3a1c..26c2492e6 100644 --- a/syft/pkg/cataloger/python/parse_uv_lock.go +++ b/syft/pkg/cataloger/python/parse_uv_lock.go @@ -5,12 +5,13 @@ import ( "context" "fmt" "io" + "maps" + "slices" "sort" "strings" "github.com/BurntSushi/toml" - "github.com/anchore/syft/internal/log" "github.com/anchore/syft/internal/unknown" "github.com/anchore/syft/syft/artifact" "github.com/anchore/syft/syft/file" @@ -45,8 +46,12 @@ type uvPackage struct { Metadata uvMetadata `toml:"metadata"` } -type uvDependencies []struct { - Name string `toml:"name"` +type uvDependencies []uvDependency + +type uvDependency struct { + Name string `toml:"name"` + // Version is only written by uv when more than one version of this name is locked + Version string `toml:"version"` Extras []string `toml:"extra"` Markers string `toml:"marker"` } @@ -83,12 +88,16 @@ func newUvLockParser(cfg CatalogerConfig) uvLockParser { // parseUvLock is a parser function for uv.lock contents, returning all the pakcages discovered func (ulp uvLockParser) parseUvLock(ctx context.Context, _ file.Resolver, _ *generic.Environment, reader file.LocationReadCloser) ([]pkg.Package, []artifact.Relationship, error) { - pkgs, err := ulp.uvLockPackages(ctx, reader) + pkgs, specs, err := ulp.uvLockPackages(ctx, reader) if err != nil { return nil, nil, err } - return pkgs, dependency.Resolve(uvLockDependencySpecifier, pkgs), err + specifier := func(p pkg.Package) dependency.Specification { + return specs[p.ID()] + } + + return pkgs, dependency.Resolve(specifier, pkgs), err } func extractUvIndex(p uvPackage) string { @@ -139,105 +148,114 @@ func newPythonUvLockEntry(p uvPackage) pkg.PythonUvLockEntry { } } -func (ulp uvLockParser) uvLockPackages(ctx context.Context, reader file.LocationReadCloser) ([]pkg.Package, error) { +// uvLockPackages returns the packages in the lock along with the dependency specification for each (by package ID). +func (ulp uvLockParser) uvLockPackages(ctx context.Context, reader file.LocationReadCloser) ([]pkg.Package, map[artifact.ID]dependency.Specification, error) { var parsedLockFileVersion uvLockFileVersion // we cannot use the reader twice, so we read the contents first --uv.lock files tend to be small enough contents, err := io.ReadAll(reader) //nolint:gocritic // multi-pass parse requires []byte if err != nil { - return nil, unknown.New(reader.Location, fmt.Errorf("failed to read uv lock file: %w", err)) + return nil, nil, unknown.New(reader.Location, fmt.Errorf("failed to read uv lock file: %w", err)) } _, err = toml.NewDecoder(bytes.NewReader(contents)).Decode(&parsedLockFileVersion) if err != nil { - return nil, fmt.Errorf("failed to read uv lock version: %w", err) + return nil, nil, fmt.Errorf("failed to read uv lock version: %w", err) } // We will need to add some logic to parse and branch on different // lock file versions should they arise, but this gets us // started down this road for now. if parsedLockFileVersion.Version > 1 { - return nil, fmt.Errorf("could not parse uv lock file version %d", parsedLockFileVersion.Version) + return nil, nil, fmt.Errorf("could not parse uv lock file version %d", parsedLockFileVersion.Version) } var parsedLockFile uvLockFile _, err = toml.NewDecoder(bytes.NewReader(contents)).Decode(&parsedLockFile) if err != nil { - return nil, fmt.Errorf("failed to parse uv lock packages: %w", err) - } - - // The uv lock file doesn't store the dependency version in the dependency structure. - // Thus, we need a name -> version map for invoking extractUvDependencies. - // We then, of course, have to pass it down the call stack. - var pkgVerMap = make(map[string]string) - for _, p := range parsedLockFile.Packages { - pkgVerMap[p.Name] = p.Version + return nil, nil, fmt.Errorf("failed to parse uv lock packages: %w", err) } var pkgs []pkg.Package + specs := make(map[artifact.ID]dependency.Specification) for _, p := range parsedLockFile.Packages { - pkgs = append(pkgs, - newPackageForIndexWithMetadata( - ctx, - ulp.licenseResolver, - p.Name, - p.Version, - newPythonUvLockEntry(p), - reader.WithAnnotation(pkg.EvidenceAnnotationKey, pkg.PrimaryEvidenceAnnotation), - ), + np := newPackageForIndexWithMetadata( + ctx, + ulp.licenseResolver, + p.Name, + p.Version, + newPythonUvLockEntry(p), + reader.WithAnnotation(pkg.EvidenceAnnotationKey, pkg.PrimaryEvidenceAnnotation), ) + pkgs = append(pkgs, np) + specs[np.ID()] = uvLockDependencySpecification(p) } - return pkgs, unknown.IfEmptyf(pkgs, "unable to determine packages") + return pkgs, specs, unknown.IfEmptyf(pkgs, "unable to determine packages") } -func isDependencyForUvExtra(dep pkg.PythonUvLockDependencyEntry) bool { +func isDependencyForUvExtra(dep uvDependency) bool { return strings.Contains(dep.Markers, "extra ==") } -// This is identical to poetryLockDependencySpecifier since it operates on identical -// data structures. Keeping it separate for now since it's always possible for data -// structures to change down the line. -// It *is* possible we may be able to merge the Uv and Poetry data structures -func uvLockDependencySpecifier(p pkg.Package) dependency.Specification { //nolint:dupl // this is very similar to the poetry lock dependency specifier, but should remain separate - meta, ok := p.Metadata.(pkg.PythonUvLockEntry) - if !ok { - log.Tracef("cataloger failed to extract UV lock metadata for package %+v", p.Name) - return dependency.Specification{} - } - - provides := []string{packageRef(p.Name, "")} - +// uvLockDependencySpecification is built from the raw lock entry rather than the package metadata. When a lock +// holds several versions of one name (a forked resolution), uv records which version each dependency entry means, +// and that is the only way to pair a dependent with the right one. The pairing is carried by the resulting +// relationships, so the version does not need to live on the metadata. +func uvLockDependencySpecification(p uvPackage) dependency.Specification { var requires []string - - for _, dep := range meta.Dependencies { + for _, dep := range p.Dependencies { if isDependencyForUvExtra(dep) { continue } - - requires = append(requires, packageRef(dep.Name, "")) - - for _, extra := range dep.Extras { - requires = append(requires, packageRef(dep.Name, extra)) - } + requires = append(requires, uvLockRequires(dep)...) } var variants []dependency.ProvidesRequires - for _, extra := range meta.Extras { + for _, extra := range slices.Sorted(maps.Keys(p.OptionalDependencies)) { + var extraRequires []string + for _, dep := range p.OptionalDependencies[extra] { + extraRequires = append(extraRequires, uvLockRequires(dep)...) + } variants = append(variants, dependency.ProvidesRequires{ - Provides: []string{packageRef(p.Name, extra.Name)}, - Requires: extractPackageNames(extra.Dependencies), + Provides: uvLockProvides(p, extra), + Requires: extraRequires, }, ) } return dependency.Specification{ ProvidesRequires: dependency.ProvidesRequires{ - Provides: provides, + Provides: uvLockProvides(p, ""), Requires: requires, }, Variants: variants, } } + +// uvLockProvides offers both the bare ref (for dependency entries without a version) and the versioned ref (for +// dependency entries uv pinned to this version). +func uvLockProvides(p uvPackage, extra string) []string { + return []string{packageRef(p.Name, extra), uvLockPackageRef(p.Name, p.Version, extra)} +} + +// uvLockRequires always requires the base package, plus each extra individually (name[extra1] and name[extra2], +// never name[extra1,extra2]). +func uvLockRequires(dep uvDependency) []string { + refs := []string{uvLockPackageRef(dep.Name, dep.Version, "")} + for _, extra := range dep.Extras { + refs = append(refs, uvLockPackageRef(dep.Name, dep.Version, extra)) + } + return refs +} + +func uvLockPackageRef(name, version, extra string) string { + ref := packageRef(name, extra) + version = strings.TrimSpace(version) + if version == "" { + return ref + } + return ref + "@" + version +} diff --git a/syft/pkg/cataloger/python/parse_uv_lock_test.go b/syft/pkg/cataloger/python/parse_uv_lock_test.go index 582ac9eb6..9aafc2471 100644 --- a/syft/pkg/cataloger/python/parse_uv_lock_test.go +++ b/syft/pkg/cataloger/python/parse_uv_lock_test.go @@ -129,6 +129,79 @@ func TestParseUvLock(t *testing.T) { pkgtest.TestFileParser(t, fixture, uvLockParser.parseUvLock, expectedPkgs, expectedRelationships) } +func TestParseUvLockForkedVersions(t *testing.T) { + fixture := "testdata/uv/forked-versions/uv.lock" + locations := file.NewLocationSet(file.NewLocation(fixture)) + index := "https://pypi.org/simple" + + newPkg := func(name, version string, meta pkg.PythonUvLockEntry) pkg.Package { + return pkg.Package{ + Name: name, + Version: version, + Locations: locations, + PURL: "pkg:pypi/" + name + "@" + version, + Language: pkg.Python, + Type: pkg.PythonPkg, + Metadata: meta, + } + } + + forkdemo := newPkg("forkdemo", "0.1.0", pkg.PythonUvLockEntry{ + Index: ".", + Dependencies: []pkg.PythonUvLockDependencyEntry{ + {Name: "pandas", Markers: "python_full_version < '3.11'"}, + {Name: "pandas", Markers: "python_full_version >= '3.11'"}, + }, + Extras: []pkg.PythonUvLockExtraEntry{ + {Name: "legacy", Dependencies: []string{"numpy"}}, + }, + }) + numpy1264 := newPkg("numpy", "1.26.4", pkg.PythonUvLockEntry{Index: index}) + numpy226 := newPkg("numpy", "2.2.6", pkg.PythonUvLockEntry{Index: index}) + numpy246 := newPkg("numpy", "2.4.6", pkg.PythonUvLockEntry{Index: index}) + numpy253 := newPkg("numpy", "2.5.3", pkg.PythonUvLockEntry{Index: index}) + pandas233 := newPkg("pandas", "2.3.3", pkg.PythonUvLockEntry{ + Index: index, + Dependencies: []pkg.PythonUvLockDependencyEntry{ + {Name: "numpy", Markers: "python_full_version < '3.10'"}, + {Name: "numpy", Markers: "python_full_version == '3.10.*'"}, + }, + }) + pandas306 := newPkg("pandas", "3.0.6", pkg.PythonUvLockEntry{ + Index: index, + Dependencies: []pkg.PythonUvLockDependencyEntry{ + {Name: "numpy", Markers: "python_full_version == '3.11.*'"}, + {Name: "numpy", Markers: "python_full_version >= '3.12'"}, + }, + }) + + dependencyOf := func(from, to pkg.Package) artifact.Relationship { + return artifact.Relationship{From: from, To: to, Type: artifact.DependencyOfRelationship} + } + + // each dependent links only to the versions its entries name, including through an optional dependency + expectedRelationships := []artifact.Relationship{ + dependencyOf(pandas233, forkdemo), + dependencyOf(pandas306, forkdemo), + dependencyOf(numpy1264, forkdemo), + dependencyOf(numpy1264, pandas233), + dependencyOf(numpy226, pandas233), + dependencyOf(numpy246, pandas306), + dependencyOf(numpy253, pandas306), + } + + uvLockParser := newUvLockParser(DefaultCatalogerConfig()) + pkgtest.TestFileParser(t, fixture, uvLockParser.parseUvLock, []pkg.Package{ + forkdemo, + numpy1264, + numpy226, + numpy246, + numpy253, + pandas233, + pandas306, + }, expectedRelationships) +} + func TestParseUvLockWithLicenseEnrichment(t *testing.T) { ctx := context.TODO() fixture := "testdata/pypi-remote/uv.lock" diff --git a/syft/pkg/cataloger/python/testdata/uv/forked-versions/uv.lock b/syft/pkg/cataloger/python/testdata/uv/forked-versions/uv.lock new file mode 100644 index 000000000..bcc73c082 --- /dev/null +++ b/syft/pkg/cataloger/python/testdata/uv/forked-versions/uv.lock @@ -0,0 +1,54 @@ +version = 1 +requires-python = ">=3.9" + +[[package]] +name = "forkdemo" +version = "0.1.0" +source = { virtual = "." } +dependencies = [ + { name = "pandas", version = "2.3.3", marker = "python_full_version < '3.11'" }, + { name = "pandas", version = "3.0.6", marker = "python_full_version >= '3.11'" }, +] + +[package.optional-dependencies] +legacy = [ + { name = "numpy", version = "1.26.4" }, +] + +[[package]] +name = "numpy" +version = "1.26.4" +source = { registry = "https://pypi.org/simple" } + +[[package]] +name = "numpy" +version = "2.2.6" +source = { registry = "https://pypi.org/simple" } + +[[package]] +name = "numpy" +version = "2.4.6" +source = { registry = "https://pypi.org/simple" } + +[[package]] +name = "numpy" +version = "2.5.3" +source = { registry = "https://pypi.org/simple" } + +[[package]] +name = "pandas" +version = "2.3.3" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "numpy", version = "1.26.4", marker = "python_full_version < '3.10'" }, + { name = "numpy", version = "2.2.6", marker = "python_full_version == '3.10.*'" }, +] + +[[package]] +name = "pandas" +version = "3.0.6" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "numpy", version = "2.4.6", marker = "python_full_version == '3.11.*'" }, + { name = "numpy", version = "2.5.3", marker = "python_full_version >= '3.12'" }, +] From e4ff7f5973df0c85d19ebedb768d4cb00ff4a1ce Mon Sep 17 00:00:00 2001 From: "Antoni (Tony) Jagodka" Date: Tue, 29 Sep 2026 16:30:52 -0500 Subject: [PATCH 32/40] fix(spdx): write a name for the root package when the source has none (#5349) * fix(spdx): write a name for the root package when the source has none A source read from another SBOM (a purl list, a CycloneDX document without metadata.component, an SPDX document without a DESCRIBES relationship) can have no name. The SPDX encoder then wrote the document root package with an empty PackageName, which SPDX requires. In tag-value output the PackageName line is omitted, so the root package's fields land in the previous section and syft cannot read its own output back ("received unknown tag SPDXID in CreationInfo section"). Fall back to the document name for the root package, and make the document name fall back to "unknown" for every source type rather than only for unrecognized ones. Signed-off-by: Tony * fix(spdx): drop syft's root package on decode for every SPDX version - the root package name now comes from one fallback chain (source name, image repo name, source ID, document name), so the root purl no longer gets an empty name for a source decoded from CycloneDX - decoding strips the root syft writes for an unknown source, so `syft convert` round trips stop adding an `unknown` package each pass - for SPDX 2.1 and 2.2, which have no `primaryPackagePurpose`, the purpose is taken from the source type in syft's `DocumentRoot--` ID **behavior change:** decoding a syft-written SPDX 2.1/2.2 document no longer returns the root as a package or its `CONTAINS` relationships, matching 2.3. Signed-off-by: Alex Goodman --------- Signed-off-by: Tony Signed-off-by: Alex Goodman Co-authored-by: Alex Goodman --- .../common/spdxhelpers/to_format_model.go | 31 +++++++--- .../spdxhelpers/to_format_model_test.go | 46 ++++++++++++++ .../common/spdxhelpers/to_syft_model.go | 38 +++++++++++- .../spdxutil/helpers/document_name.go | 15 +++-- .../spdxutil/helpers/document_name_test.go | 16 ++++- syft/format/spdxjson/encoder_test.go | 25 +------- .../TestSPDX22JSONRequredProperties.golden | 10 ++-- syft/format/spdxtagvalue/encoder_test.go | 60 ++++++++++++------- 8 files changed, 176 insertions(+), 65 deletions(-) diff --git a/syft/format/common/spdxhelpers/to_format_model.go b/syft/format/common/spdxhelpers/to_format_model.go index 426f9e10c..2a175ce9d 100644 --- a/syft/format/common/spdxhelpers/to_format_model.go +++ b/syft/format/common/spdxhelpers/to_format_model.go @@ -2,6 +2,7 @@ package spdxhelpers import ( + "cmp" "crypto/sha1" "fmt" "path" @@ -179,7 +180,8 @@ func toRootRelationships(rootPackage *spdx.Package, packages []*spdx.Package) (o func toRootPackage(s source.Description) *spdx.Package { var prefix string - name := s.Name + // the SPDX package name is mandatory, and a source decoded from another SBOM may not carry one + name := cmp.Or(s.Name, imageRepoName(s.Metadata), s.ID, helpers.DocumentName(s)) version := s.Version var purl *packageurl.PackageURL @@ -210,7 +212,7 @@ func toRootPackage(s source.Description) *spdx.Package { checksums = append(checksums, *c) purl = &packageurl.PackageURL{ Type: "oci", - Name: s.Name, + Name: name, Version: m.ManifestDigest, Qualifiers: qualifiers, } @@ -240,7 +242,7 @@ func toRootPackage(s source.Description) *spdx.Package { checksums = append(checksums, *c) purl = &packageurl.PackageURL{ Type: "oci", - Name: s.Name, + Name: name, Version: m.ManifestDigest, Qualifiers: qualifiers, } @@ -275,10 +277,6 @@ func toRootPackage(s source.Description) *spdx.Package { default: prefix = prefixUnknown purpose = spdxPrimaryPurposeOther - - if name == "" { - name = s.ID - } } p := &spdx.Package{ @@ -308,6 +306,25 @@ func toRootPackage(s source.Description) *spdx.Package { return p } +// imageRepoName returns the repository name from an image or model user input, the same way the image source names +// itself, so a source decoded from another SBOM (which may only carry the user input) gets a name without the tag. +func imageRepoName(metadata any) string { + var userInput string + switch m := metadata.(type) { + case source.ImageMetadata: + userInput = m.UserInput + case source.OCIModelMetadata: + userInput = m.UserInput + default: + return "" + } + ref, _ := reference.Parse(userInput) + if named, ok := ref.(reference.Named); ok { + return named.Name() + } + return "" +} + func toSPDXID(identifiable artifact.Identifiable) spdx.ElementID { id := string(identifiable.ID()) if after, ok := strings.CutPrefix(id, "SPDXRef-"); ok { diff --git a/syft/format/common/spdxhelpers/to_format_model_test.go b/syft/format/common/spdxhelpers/to_format_model_test.go index 302fe81ae..aede4dbee 100644 --- a/syft/format/common/spdxhelpers/to_format_model_test.go +++ b/syft/format/common/spdxhelpers/to_format_model_test.go @@ -1145,3 +1145,49 @@ func Test_otherLicenses(t *testing.T) { }) } } + +func Test_toRootPackage_name(t *testing.T) { + tests := []struct { + name string + src source.Description + wantName string + wantPurl string + }{ + { + name: "source name wins", + src: source.Description{Name: "named", ID: "id", Metadata: source.ImageMetadata{UserInput: "alpine:3.18"}}, + wantName: "named", + }, + { + // a CycloneDX container component decodes to an image source with only the user input set + name: "image user input without a source name", + src: source.Description{Metadata: source.ImageMetadata{ + UserInput: "alpine:3.18", + ManifestDigest: "sha256:0e4b2e0d6b2f3c2b1d3bb3a1c8f1c4d9e6a9b1e8c6f4d2b3a1f0e9d8c7b6a5f4", + }}, + wantName: "alpine", + wantPurl: "pkg:oci/alpine@sha256%3A0e4b2e0d6b2f3c2b1d3bb3a1c8f1c4d9e6a9b1e8c6f4d2b3a1f0e9d8c7b6a5f4?arch=&tag=3.18", + }, + { + name: "id without a source name", + src: source.Description{ID: "some-id"}, + wantName: "some-id", + }, + { + name: "nothing to name it by", + src: source.Description{Metadata: source.DirectoryMetadata{}}, + wantName: "unknown", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + p := toRootPackage(tt.src) + assert.Equal(t, tt.wantName, p.PackageName) + var purl string + if len(p.PackageExternalReferences) > 0 { + purl = p.PackageExternalReferences[0].Locator + } + assert.Equal(t, tt.wantPurl, purl) + }) + } +} diff --git a/syft/format/common/spdxhelpers/to_syft_model.go b/syft/format/common/spdxhelpers/to_syft_model.go index e8b3e2bfb..eaddc9650 100644 --- a/syft/format/common/spdxhelpers/to_syft_model.go +++ b/syft/format/common/spdxhelpers/to_syft_model.go @@ -147,12 +147,24 @@ func extractSource(spdxIDMap map[string]any, doc *spdx.Document) source.Descript } p := rootPackages[0] + rootType := syftRootType(p.PackageSPDXIdentifier) - switch p.PrimaryPackagePurpose { + purpose := p.PrimaryPackagePurpose + if purpose == "" { + // SPDX 2.1 and 2.2 have no primary package purpose, so fall back to the source type syft puts in the root's ID + purpose = syftRootPurpose(rootType) + } + + switch purpose { case spdxPrimaryPurposeContainer: src = containerSource(p) case spdxPrimaryPurposeFile: src = fileSource(p) + case spdxPrimaryPurposeOther: + // syft writes a root package even for a source it knows nothing about, which is not a package in the SBOM + if rootType != prefixUnknown { + return src + } default: return src } @@ -165,6 +177,30 @@ func extractSource(spdxIDMap map[string]any, doc *spdx.Document) source.Descript return src } +// syftRootType returns the source type syft encodes in a root package ID ("DocumentRoot--"), or "" for +// any other ID. +func syftRootType(id spdx.ElementID) string { + rest, ok := strings.CutPrefix(string(id), "DocumentRoot-") + if !ok { + return "" + } + rootType, _, _ := strings.Cut(rest, "-") + return rootType +} + +// syftRootPurpose returns the primary package purpose syft writes for a root package of the given source type. +func syftRootPurpose(rootType string) string { + switch rootType { + case prefixImage, prefixOCIModel, prefixSnap: + return spdxPrimaryPurposeContainer + case prefixDirectory, prefixFile: + return spdxPrimaryPurposeFile + case prefixUnknown: + return spdxPrimaryPurposeOther + } + return "" +} + func containerSource(p *spdx.Package) source.Description { id := string(p.PackageSPDXIdentifier) diff --git a/syft/format/internal/spdxutil/helpers/document_name.go b/syft/format/internal/spdxutil/helpers/document_name.go index b34c7a445..afba5a5a7 100644 --- a/syft/format/internal/spdxutil/helpers/document_name.go +++ b/syft/format/internal/spdxutil/helpers/document_name.go @@ -9,16 +9,21 @@ func DocumentName(src source.Description) string { return src.Name } + var name string switch metadata := src.Metadata.(type) { case source.ImageMetadata: - return metadata.UserInput + name = metadata.UserInput case source.OCIModelMetadata: - return metadata.UserInput + name = metadata.UserInput case source.DirectoryMetadata: - return metadata.Path + name = metadata.Path case source.FileMetadata: - return metadata.Path - default: + name = metadata.Path + } + + // the SPDX document name is mandatory, and a source read from another SBOM may not have one + if name == "" { return "unknown" } + return name } diff --git a/syft/format/internal/spdxutil/helpers/document_name_test.go b/syft/format/internal/spdxutil/helpers/document_name_test.go index 26dbf1ebf..eab8fc18f 100644 --- a/syft/format/internal/spdxutil/helpers/document_name_test.go +++ b/syft/format/internal/spdxutil/helpers/document_name_test.go @@ -1,8 +1,6 @@ package helpers import ( - "fmt" - "strings" "testing" "github.com/stretchr/testify/assert" @@ -73,11 +71,23 @@ func Test_DocumentName(t *testing.T) { }, expected: "some/name", }, + { + name: "no name or metadata", + srcMetadata: source.Description{}, + expected: "unknown", + }, + { + name: "metadata without a name", + srcMetadata: source.Description{ + Metadata: source.ImageMetadata{}, + }, + expected: "unknown", + }, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { actual := DocumentName(test.srcMetadata) - assert.True(t, strings.HasPrefix(actual, test.expected), fmt.Sprintf("actual name %q", actual)) + assert.Equal(t, test.expected, actual) // track each scheme tested (passed or not) tracker.Tested(t, test.srcMetadata.Metadata) diff --git a/syft/format/spdxjson/encoder_test.go b/syft/format/spdxjson/encoder_test.go index be4680942..42912d5d5 100644 --- a/syft/format/spdxjson/encoder_test.go +++ b/syft/format/spdxjson/encoder_test.go @@ -236,23 +236,6 @@ func TestSupportedVersions(t *testing.T) { subject := testutil.DirectoryInput(t, t.TempDir()) dec := NewFormatDecoder() - relationshipOffsetPerVersion := map[string]int{ - // the package representing the source gets a relationship from the source package to all other packages found - // these relationships cannot be removed until the primaryPackagePurpose info is available in 2.3 - "2.1": 2, - "2.2": 2, - // the source-to-package relationships can be removed since the primaryPackagePurpose info is available in 2.3 - "2.3": 0, - "3.0": 0, - } - - pkgCountOffsetPerVersion := map[string]int{ - "2.1": 1, // the source is mapped as a package, but cannot distinguish it since the primaryPackagePurpose info is not available until 2.3 - "2.2": 1, // the source is mapped as a package, but cannot distinguish it since the primaryPackagePurpose info is not available until 2.3 - "2.3": 0, // the source package can be removed since the primaryPackagePurpose info is available - "3.0": 0, - } - for _, enc := range encs { t.Run(enc.Version(), func(t *testing.T) { require.Contains(t, versions, enc.Version()) @@ -274,13 +257,9 @@ func TestSupportedVersions(t *testing.T) { require.NotEmpty(t, s.Artifacts.Packages.PackageCount()) - offset := relationshipOffsetPerVersion[enc.Version()] + assert.Equal(t, len(subject.Relationships), len(s.Relationships), "mismatched relationship count") - assert.Equal(t, len(subject.Relationships)+offset, len(s.Relationships), "mismatched relationship count") - - offset = pkgCountOffsetPerVersion[enc.Version()] - - if !assert.Equal(t, subject.Artifacts.Packages.PackageCount()+offset, s.Artifacts.Packages.PackageCount(), "mismatched package count") { + if !assert.Equal(t, subject.Artifacts.Packages.PackageCount(), s.Artifacts.Packages.PackageCount(), "mismatched package count") { t.Logf("expected: %d", subject.Artifacts.Packages.PackageCount()) for _, p := range subject.Artifacts.Packages.Sorted() { t.Logf(" - %s", p.String()) diff --git a/syft/format/spdxjson/testdata/snapshot/TestSPDX22JSONRequredProperties.golden b/syft/format/spdxjson/testdata/snapshot/TestSPDX22JSONRequredProperties.golden index 8587d7bc0..5f2b09b5b 100644 --- a/syft/format/spdxjson/testdata/snapshot/TestSPDX22JSONRequredProperties.golden +++ b/syft/format/spdxjson/testdata/snapshot/TestSPDX22JSONRequredProperties.golden @@ -47,13 +47,13 @@ "copyrightText": "NOASSERTION" }, { - "SPDXID": "SPDXRef-DocumentRoot-Unknown-", + "SPDXID": "SPDXRef-DocumentRoot-Unknown-unknown", "copyrightText": "NOASSERTION", "downloadLocation": "NOASSERTION", "filesAnalyzed": false, "licenseConcluded": "NOASSERTION", "licenseDeclared": "NOASSERTION", - "name": "", + "name": "unknown", "supplier": "NOASSERTION" } ], @@ -82,18 +82,18 @@ "relationshipType": "CONTAINS" }, { - "spdxElementId": "SPDXRef-DocumentRoot-Unknown-", + "spdxElementId": "SPDXRef-DocumentRoot-Unknown-unknown", "relatedSpdxElement": "SPDXRef-Package-files-analyzed-false-7d37ba9d2f7c574b", "relationshipType": "CONTAINS" }, { - "spdxElementId": "SPDXRef-DocumentRoot-Unknown-", + "spdxElementId": "SPDXRef-DocumentRoot-Unknown-unknown", "relatedSpdxElement": "SPDXRef-Package-files-analyzed-true-035066c2086b8bb4", "relationshipType": "CONTAINS" }, { "spdxElementId": "SPDXRef-DOCUMENT", - "relatedSpdxElement": "SPDXRef-DocumentRoot-Unknown-", + "relatedSpdxElement": "SPDXRef-DocumentRoot-Unknown-unknown", "relationshipType": "DESCRIBES" } ] diff --git a/syft/format/spdxtagvalue/encoder_test.go b/syft/format/spdxtagvalue/encoder_test.go index e5d470ec8..991d1ad1e 100644 --- a/syft/format/spdxtagvalue/encoder_test.go +++ b/syft/format/spdxtagvalue/encoder_test.go @@ -3,6 +3,7 @@ package spdxtagvalue import ( "bytes" "flag" + "fmt" "testing" "github.com/stretchr/testify/assert" @@ -143,21 +144,6 @@ func TestSupportedVersions(t *testing.T) { subject := testutil.DirectoryInput(t, t.TempDir()) dec := NewFormatDecoder() - relationshipOffsetPerVersion := map[string]int{ - // the package representing the source gets a relationship from the source package to all other packages found - // these relationships cannot be removed until the primaryPackagePurpose info is available in 2.3 - "2.1": 2, - "2.2": 2, - // the source-to-package relationships can be removed since the primaryPackagePurpose info is available in 2.3 - "2.3": 0, - } - - pkgCountOffsetPerVersion := map[string]int{ - "2.1": 1, // the source is mapped as a package, but cannot distinguish it since the primaryPackagePurpose info is not available until 2.3 - "2.2": 1, // the source is mapped as a package, but cannot distinguish it since the primaryPackagePurpose info is not available until 2.3 - "2.3": 0, // the source package can be removed since the primaryPackagePurpose info is available - } - for _, enc := range encs { t.Run(enc.Version(), func(t *testing.T) { require.Contains(t, versions, enc.Version()) @@ -178,13 +164,9 @@ func TestSupportedVersions(t *testing.T) { require.NotEmpty(t, s.Artifacts.Packages.PackageCount()) - offset := relationshipOffsetPerVersion[enc.Version()] + assert.Equal(t, len(subject.Relationships), len(s.Relationships), "mismatched relationship count") - assert.Equal(t, len(subject.Relationships)+offset, len(s.Relationships), "mismatched relationship count") - - offset = pkgCountOffsetPerVersion[enc.Version()] - - if !assert.Equal(t, subject.Artifacts.Packages.PackageCount()+offset, s.Artifacts.Packages.PackageCount(), "mismatched package count") { + if !assert.Equal(t, subject.Artifacts.Packages.PackageCount(), s.Artifacts.Packages.PackageCount(), "mismatched package count") { t.Logf("expected: %d", subject.Artifacts.Packages.PackageCount()) for _, p := range subject.Artifacts.Packages.Sorted() { t.Logf(" - %s", p.String()) @@ -209,3 +191,39 @@ func defaultFormatEncoders() []sbom.FormatEncoder { } return encs } + +func TestEncodeDecodeSourceWithoutName(t *testing.T) { + // a source read from another SBOM (a purl list, a CycloneDX document without metadata.component) may have no name, + // but the SPDX package name is mandatory and the tag-value decoder needs it to start the root package section + p := pkg.Package{Name: "left-pad", Version: "1.3.0", Type: pkg.NpmPkg} + p.SetID() + + for _, src := range []source.Description{ + {}, + {Metadata: source.FileMetadata{}}, + {Metadata: source.DirectoryMetadata{}}, + {Metadata: source.ImageMetadata{}}, + {Metadata: source.OCIModelMetadata{}}, + {Metadata: source.SnapMetadata{}}, + } { + for _, enc := range defaultFormatEncoders() { + t.Run(fmt.Sprintf("%T/%s", src.Metadata, enc.Version()), func(t *testing.T) { + var buf bytes.Buffer + require.NoError(t, enc.Encode(&buf, sbom.SBOM{ + Source: src, + Artifacts: sbom.Artifacts{Packages: pkg.NewCollection(p)}, + })) + + s, _, _, err := NewFormatDecoder().Decode(bytes.NewReader(buf.Bytes())) + require.NoError(t, err, "encoded document:\n%s", buf.String()) + + var names []string + for _, decoded := range s.Artifacts.Packages.Sorted() { + names = append(names, decoded.Name) + } + // the root package describes the source, so it must not come back as a package of the SBOM + assert.Equal(t, []string{"left-pad"}, names) + }) + } + } +} From 8c63a2403c903c5c951dc89080999d410dd72305 Mon Sep 17 00:00:00 2001 From: KR Ravindra Date: Tue, 29 Sep 2026 14:43:53 -0700 Subject: [PATCH 33/40] fix: conan expat CPE (#5259) Add libexpat_project vendor and libexpat product candidates for Conan packages named expat, so generated CPEs match the NVD entries used for libexpat vulnerabilities. Signed-off-by: KR Ravindra <42912207+KR-Ravindra@users.noreply.github.com> Co-authored-by: Will Murphy --- .../cpegenerate/candidate_by_package_type.go | 5 +++++ .../candidate_by_package_type_test.go | 7 +++++++ .../internal/cpegenerate/generate_test.go | 16 ++++++++++++++++ 3 files changed, 28 insertions(+) diff --git a/syft/pkg/cataloger/internal/cpegenerate/candidate_by_package_type.go b/syft/pkg/cataloger/internal/cpegenerate/candidate_by_package_type.go index b43906c8b..7a702c071 100644 --- a/syft/pkg/cataloger/internal/cpegenerate/candidate_by_package_type.go +++ b/syft/pkg/cataloger/internal/cpegenerate/candidate_by_package_type.go @@ -518,6 +518,11 @@ var defaultCandidateAdditions = buildCandidateLookup( candidateKey{PkgName: "libxml2"}, candidateAddition{AdditionalVendors: []string{"xmlsoft"}}, }, + { + pkg.ConanPkg, + candidateKey{PkgName: "expat"}, + candidateAddition{AdditionalVendors: []string{"libexpat_project"}, AdditionalProducts: []string{"libexpat"}}, + }, }) var defaultCandidateRemovals = buildCandidateRemovalLookup( diff --git a/syft/pkg/cataloger/internal/cpegenerate/candidate_by_package_type_test.go b/syft/pkg/cataloger/internal/cpegenerate/candidate_by_package_type_test.go index 8f21eeae4..fb73ea799 100644 --- a/syft/pkg/cataloger/internal/cpegenerate/candidate_by_package_type_test.go +++ b/syft/pkg/cataloger/internal/cpegenerate/candidate_by_package_type_test.go @@ -245,3 +245,10 @@ func Test_defaultCandidateAdditions_conan_libxml2(t *testing.T) { vendors := findAdditionalVendors(defaultCandidateAdditions, pkg.ConanPkg, "libxml2", "") assert.Contains(t, vendors, "xmlsoft") } + +func Test_defaultCandidateAdditions_conan_expat(t *testing.T) { + vendors := findAdditionalVendors(defaultCandidateAdditions, pkg.ConanPkg, "expat", "") + assert.Contains(t, vendors, "libexpat_project") + products := findAdditionalProducts(defaultCandidateAdditions, pkg.ConanPkg, "expat") + assert.Contains(t, products, "libexpat") +} diff --git a/syft/pkg/cataloger/internal/cpegenerate/generate_test.go b/syft/pkg/cataloger/internal/cpegenerate/generate_test.go index cdf32d9b7..a4e6d3d08 100644 --- a/syft/pkg/cataloger/internal/cpegenerate/generate_test.go +++ b/syft/pkg/cataloger/internal/cpegenerate/generate_test.go @@ -892,6 +892,22 @@ func TestGeneratePackageCPEs(t *testing.T) { "cpe:2.3:a:rust_package:rust_package:0.5.0:*:*:*:*:rust:*:*", }, }, + { + name: "conan expat: libexpat_project vendor and libexpat product", + p: pkg.Package{ + Name: "expat", + Version: "2.5.0", + Type: pkg.ConanPkg, + }, + expected: []string{ + "cpe:2.3:a:expat:expat:2.5.0:*:*:*:*:*:*:*", + "cpe:2.3:a:expat:libexpat:2.5.0:*:*:*:*:*:*:*", + "cpe:2.3:a:libexpat:expat:2.5.0:*:*:*:*:*:*:*", + "cpe:2.3:a:libexpat:libexpat:2.5.0:*:*:*:*:*:*:*", + "cpe:2.3:a:libexpat_project:expat:2.5.0:*:*:*:*:*:*:*", + "cpe:2.3:a:libexpat_project:libexpat:2.5.0:*:*:*:*:*:*:*", + }, + }, } for _, test := range tests { From b23e4617312a9d3598172956f57884aa703a5b02 Mon Sep 17 00:00:00 2001 From: Darragh McGurk <120114728+dmcg310@users.noreply.github.com> Date: Tue, 29 Sep 2026 22:44:47 +0100 Subject: [PATCH 34/40] Stop reading the mount table for every single-file source (#5257) Signed-off-by: Darragh McGurk <120114728+dmcg310@users.noreply.github.com> --- syft/internal/fileresolver/file_indexer.go | 6 ++- .../fileresolver/file_indexer_bench_test.go | 51 +++++++++++++++++++ 2 files changed, 56 insertions(+), 1 deletion(-) create mode 100644 syft/internal/fileresolver/file_indexer_bench_test.go diff --git a/syft/internal/fileresolver/file_indexer.go b/syft/internal/fileresolver/file_indexer.go index 086dc75a9..2c47b2b48 100644 --- a/syft/internal/fileresolver/file_indexer.go +++ b/syft/internal/fileresolver/file_indexer.go @@ -29,11 +29,15 @@ func newFileIndexer(path, base string, visitors ...PathIndexVisitor) *fileIndexe base: base, tree: filetree.New(), index: filetree.NewIndex(), + // Unlike the directory indexer there is no skipPathsByMountTypeAndName here: it exists + // to keep a directory walk out of /proc, /sys and similar, and to do so it reads the + // whole mount table on construction. A file source indexes exactly one file and its + // parent, so there is nothing to walk into, and the mount table read is paid for + // every file source with nothing in return. pathIndexVisitors: append( []PathIndexVisitor{ requireFileInfo, disallowByFileType, - skipPathsByMountTypeAndName(path), }, visitors..., ), diff --git a/syft/internal/fileresolver/file_indexer_bench_test.go b/syft/internal/fileresolver/file_indexer_bench_test.go new file mode 100644 index 000000000..11befc4cf --- /dev/null +++ b/syft/internal/fileresolver/file_indexer_bench_test.go @@ -0,0 +1,51 @@ +package fileresolver + +import ( + "os" + "path/filepath" + "testing" + + "github.com/moby/sys/mountinfo" +) + +// BenchmarkNewFromFile is the cost of building a resolver for a single-file source, which is +// paid once per file by callers that catalog files individually. The default case is +// NewFromFile as shipped. The with-mount-skipper case attaches skipPathsByMountTypeAndName +// the way the file indexer used to, so the difference is what reading the mount table for +// every file source cost. That cost scales with the mount table, so its size is reported as +// the mounts metric: expect a bigger gap on a container host than on a laptop. +// +// go test ./syft/internal/fileresolver/ -run '^$' -bench NewFromFile -benchmem -count 6 | tee bench.txt +// benchstat -col /variant bench.txt +func BenchmarkNewFromFile(b *testing.B) { + path := filepath.Join(b.TempDir(), "file.txt") + if err := os.WriteFile(path, []byte("hello\n"), 0o644); err != nil { + b.Fatal(err) + } + mounts, err := mountinfo.GetMounts(nil) + if err != nil { + b.Fatal(err) + } + + b.Run("variant=default", func(b *testing.B) { + for b.Loop() { + if _, err := NewFromFile(path); err != nil { + b.Fatal(err) + } + } + b.ReportMetric(float64(len(mounts)), "mounts") + }) + + b.Run("variant=with-mount-skipper", func(b *testing.B) { + for b.Loop() { + r, err := newFromFileWithoutIndex(path, skipPathsByMountTypeAndName(path)) + if err != nil { + b.Fatal(err) + } + if err := r.buildIndex(); err != nil { + b.Fatal(err) + } + } + b.ReportMetric(float64(len(mounts)), "mounts") + }) +} From 6ac7afb439c950ec2fc169103c4850c6dfe74c5a Mon Sep 17 00:00:00 2001 From: Alex Goodman Date: Wed, 30 Sep 2026 09:08:13 -0400 Subject: [PATCH 35/40] Fix missing file hashes when running on windows system (#5341) * test: add windows unit test job and capture dir resolver path bug - add a `Unit tests (Windows)` job running a new `make unit:windows` task, scoped to the packages that own host path handling (tests needing linux docker fixtures are skipped) - `Unit tests` is now a gate job over the Linux and Windows unit jobs - add a resolver test asserting that locations from `AllLocations` resolve again via `FilesByPath` and use posix paths. This is expected to fail on windows today, which is why file digests come back empty for `dir:` scans (#5325) - fix windows-only test files that no longer compiled Signed-off-by: Alex Goodman * chore: drop verbose output from integration tests Signed-off-by: Alex Goodman * fix: windows path handling outside of the file lookup bug - chroot paths are always posix (`ToChrootPath` no longer returns backslash paths on windows), same for the directory source name derived from `base` - `windows.FromPosix` no longer panics on input that isn't volume-encoded (e.g. relative or already native paths) - skip tests on windows that rely on symlink fixtures, shell script fixtures, or emulate windows paths on a posix host - disable `core.autocrlf` on the windows CI job so fixture digests match linux This leaves only the `FilesByPath` lookup failures from #5325. Signed-off-by: Alex Goodman * test: skip posix-only fileresolver tests on windows Signed-off-by: Alex Goodman * fix: split contained paths on posix separators on windows - `allContainedPaths` cleaned with the native separator but split on `/`, so a root reached through a symlink never had its ancestors indexed on windows - index assertions in tests now key on the posix path the index actually stores - skip symlink fixture tests and a posix-only test helper on windows Signed-off-by: Alex Goodman * chore: revert unrelated go.sum churn and allContainedPaths change - restore `go.sum` and `.make/go.sum` (accidentally populated by a local `-mod=mod` invocation) - revert the `allContainedPaths` separator change: production passes native windows paths there, and splitting them on `/` produces drive-relative ancestors, so skip the posix-only test on windows instead - annotate the remaining `filepath` usages that operate on native paths Signed-off-by: Alex Goodman * fix: search the file tree with posix paths on windows (#5325) The directory and file resolvers key their file tree on posix paths (`/c/some/path`), but `FilesByPath`, `HasPath` and root-anchored `FilesByGlob` searched it with the native path from the chroot context (`C:\some\path`). On windows every lookup silently missed, so anything that resolves files by path found nothing. This is why file digests were empty for `dir:` scans. Requests are now converted to posix before searching the tree. Signed-off-by: Alex Goodman --------- Signed-off-by: Alex Goodman --- .gitattributes | 7 + .github/workflows/validations.yaml | 28 ++++ .make/main.go | 20 ++- syft/internal/fileresolver/chroot_context.go | 5 +- .../fileresolver/chroot_context_test.go | 7 + .../fileresolver/directory_indexer_test.go | 48 +++++-- .../fileresolver/directory_roundtrip_test.go | 48 +++++++ .../fileresolver/directory_windows_test.go | 126 ------------------ .../fileresolver/file_indexer_test.go | 17 +-- .../fileresolver/filetree_resolver.go | 29 +++- syft/internal/windows/path.go | 4 + syft/internal/windows/path_windows_test.go | 35 +++++ .../directorysource/directory_source.go | 3 +- .../directorysource/directory_source_test.go | 18 ++- .../directory_source_win_test.go | 5 +- syft/source/filesource/file_source_test.go | 4 + 16 files changed, 242 insertions(+), 162 deletions(-) create mode 100644 .gitattributes create mode 100644 syft/internal/fileresolver/directory_roundtrip_test.go delete mode 100644 syft/internal/fileresolver/directory_windows_test.go create mode 100644 syft/internal/windows/path_windows_test.go diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 000000000..2bcde6f9d --- /dev/null +++ b/.gitattributes @@ -0,0 +1,7 @@ +# check out text files with lf on every platform (regardless of core.autocrlf), letting git detect binary files +* text=auto eol=lf + +# fixtures that must keep their crlf line endings byte for byte +syft/linux/testdata/unprintable -text +syft/pkg/cataloger/bitnami/testdata/no-rel/opt/bitnami/redis/.spdx-redis.spdx -text +syft/pkg/cataloger/java/testdata/jar-metadata/**/META-INF/MANIFEST.MF -text diff --git a/.github/workflows/validations.yaml b/.github/workflows/validations.yaml index 2656ab1f1..358cbadbf 100644 --- a/.github/workflows/validations.yaml +++ b/.github/workflows/validations.yaml @@ -49,6 +49,13 @@ jobs: Unit-Test: # Note: changing this job name requires making the same update in the .github/workflows/release.yaml pipeline name: "Unit tests" + runs-on: ubuntu-latest + needs: [Unit-Test-Linux, Unit-Test-Windows] + steps: + - run: echo All unit tests passed! + + Unit-Test-Linux: + name: "Unit tests (Linux)" runs-on: *test-runner permissions: contents: read @@ -68,6 +75,27 @@ jobs: - name: Check for capability drift run: make check-capability-drift + Unit-Test-Windows: + name: "Unit tests (Windows)" + runs-on: windows-latest + permissions: + contents: read + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 #v7.0.1 + with: + persist-credentials: false + + # go-make's setup only (not ./.github/actions/bootstrap): no apt, binny tools, or fixture cache are needed + - name: Setup go + go-make tooling + uses: anchore/go-make/.github/actions/setup@fa5421b3bf24c9d18f11a736c5eb4172f2b7d683 # v0.8.1 + with: + # keep in sync with the go-version default in ./.github/actions/bootstrap + go-version: "1.26.2" + + # the full unit suite leans on linux docker fixtures, so this runs the windows-relevant subset + - name: Run unit tests + run: make unit:windows + Integration-Test: # Note: changing this job name requires making the same update in the .github/workflows/release.yaml pipeline name: "Integration tests" diff --git a/.make/main.go b/.make/main.go index f0f08700d..58db41a96 100644 --- a/.make/main.go +++ b/.make/main.go @@ -1,6 +1,7 @@ package main import ( + "os" "path/filepath" "runtime" "strconv" @@ -52,7 +53,7 @@ func main() { raceFlag = " -race" } Run( - "go test -count=1 -timeout=30m -v"+raceFlag+" ./cmd/syft/internal/test/integration/...", + "go test -count=1 -timeout=30m"+raceFlag+" ./cmd/syft/internal/test/integration/...", run.Env("GODEBUG", "dontfreezetheworld=1"), ) }, @@ -96,6 +97,23 @@ func main() { }, }, + // windows unit tests: deliberately not hooked into "test" (no RunsOn). The full unit suite leans on + // linux docker fixture images and posix-only assumptions, so this covers the packages that own + // host path handling (where windows bugs live) and skips the tests that need docker image fixtures. + // ponytail: curated package list, widen it as more of the suite is made windows-safe. + Task{ + Name: "unit:windows", + Description: "run the windows-relevant subset of unit tests", + Run: func() { + Run("go test -count=1"+ + ` -skip "Image|Squash|AllLayers|MixFileTypes|GivenCoordinates"`+ + " ./syft/internal/windows/... ./syft/internal/fileresolver/... ./syft/source/directorysource/... ./syft/source/filesource/..."+ + " ./syft/file ./syft/file/cataloger/filedigest/... ./syft/file/cataloger/filecontent/...", + run.Stdout(os.Stderr), + ) + }, + }, + // default validation pipeline (replaces Taskfile `default`/`pr-validations`/`validations`). Task{ Name: "default", diff --git a/syft/internal/fileresolver/chroot_context.go b/syft/internal/fileresolver/chroot_context.go index 88647be3a..deb8cd000 100644 --- a/syft/internal/fileresolver/chroot_context.go +++ b/syft/internal/fileresolver/chroot_context.go @@ -216,5 +216,8 @@ func (r ChrootContext) ToChrootPath(nativePath string) string { responsePath = strings.TrimPrefix(responsePath, prefix) } - return responsePath + // chroot paths are posix regardless of host (these end up as location paths in the SBOM). This uses + // filepath (not path) since responsePath is native here (FromPosix above, then trimmed of a native prefix), + // and only filepath.ToSlash knows the host separator to convert. + return filepath.ToSlash(responsePath) } diff --git a/syft/internal/fileresolver/chroot_context_test.go b/syft/internal/fileresolver/chroot_context_test.go index 06f6e283a..fbe2f20e1 100644 --- a/syft/internal/fileresolver/chroot_context_test.go +++ b/syft/internal/fileresolver/chroot_context_test.go @@ -3,6 +3,7 @@ package fileresolver import ( "os" "path/filepath" + "runtime" "testing" "github.com/stretchr/testify/assert" @@ -10,6 +11,9 @@ import ( ) func Test_ChrootContext_RequestResponse(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("fixture relies on symlinks, which git does not check out as links on windows by default") + } // / // somewhere/ // outside.txt @@ -563,6 +567,9 @@ func TestNewChrootContext_BaseValidation(t *testing.T) { } func TestToNativeGlob(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("chroot context and expectations are posix paths") + } tests := []struct { name string chrootContext ChrootContext diff --git a/syft/internal/fileresolver/directory_indexer_test.go b/syft/internal/fileresolver/directory_indexer_test.go index d24a86201..39685ccce 100644 --- a/syft/internal/fileresolver/directory_indexer_test.go +++ b/syft/internal/fileresolver/directory_indexer_test.go @@ -4,7 +4,6 @@ import ( "fmt" "io/fs" "os" - "path" "path/filepath" "runtime" "sort" @@ -18,6 +17,7 @@ import ( "github.com/wagoodman/go-progress" "github.com/anchore/stereoscope/pkg/file" + "github.com/anchore/syft/syft/internal/windows" ) type indexerMock struct { @@ -138,6 +138,9 @@ func TestDirectoryIndexer_handleFileAccessErr(t *testing.T) { } func TestDirectoryIndexer_IncludeRootPathInIndex(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("indexes the posix filesystem root") + } filterFn := func(_, path string, _ os.FileInfo, _ error) error { if path != "/" { return fs.SkipDir @@ -206,11 +209,7 @@ func TestDirectoryIndexer_index(t *testing.T) { info, err := os.Stat(test.path) assert.NoError(t, err) - // note: the index uses absolute paths, so assertions MUST keep this in mind - cwd, err := os.Getwd() - require.NoError(t, err) - - p := file.Path(path.Join(cwd, test.path)) + p := indexedPath(t, test.path) assert.Equal(t, true, tree.HasPath(p)) exists, ref, err := tree.File(p) assert.Equal(t, true, exists) @@ -226,6 +225,9 @@ func TestDirectoryIndexer_index(t *testing.T) { } func TestDirectoryIndexer_index_for_AncestorSymlinks(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("fixture relies on symlinks, which git does not check out as links on windows by default") + } // note: this test is testing the effects from NewFromDirectory, indexTree, and addPathToIndex _, filename, _, ok := runtime.Caller(0) require.True(t, ok) @@ -256,11 +258,7 @@ func TestDirectoryIndexer_index_for_AncestorSymlinks(t *testing.T) { info, err := os.Stat(test.path) assert.NoError(t, err) - // note: the index uses absolute paths, so assertions MUST keep this in mind - cwd, err := os.Getwd() - require.NoError(t, err) - - p := file.Path(path.Join(cwd, test.path)) + p := indexedPath(t, test.path) assert.Equal(t, true, tree.HasPath(p)) exists, ref, err := tree.File(p) assert.Equal(t, true, exists) @@ -287,6 +285,9 @@ func TestDirectoryIndexer_index_survive_badSymlink(t *testing.T) { } func TestDirectoryIndexer_index_survive_inaccessibleSymlinkTarget(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("relies on symlinks and posix permissions") + } // a symlink that resolves into a directory we don't have permission to traverse should be skipped // with a warning like any other inaccessible path, not abort the entire scan (see #3286). if os.Geteuid() == 0 { @@ -323,6 +324,9 @@ func TestDirectoryIndexer_index_survive_inaccessibleSymlinkTarget(t *testing.T) } func TestDirectoryIndexer_SkipsAlreadyVisitedLinkDestinations(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("fixture relies on symlinks, which git does not check out as links on windows by default") + } var observedPaths []string pathObserver := func(_, p string, _ os.FileInfo, _ error) error { fields := strings.Split(p, "testdata/symlinks-prune-indexing") @@ -430,6 +434,11 @@ func TestDirectoryIndexer_IndexesAllTypes(t *testing.T) { } func Test_allContainedPaths(t *testing.T) { + if runtime.GOOS == "windows" { + // ponytail: production passes native windows paths here, which these posix cases don't represent. + // Ancestor indexing for a root reached through a symlink is likely broken on windows; fix separately. + t.Skip("cases are posix paths") + } tests := []struct { name string @@ -485,6 +494,9 @@ func Test_allContainedPaths(t *testing.T) { } func Test_relativePath(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("exercises a posix-only test helper") + } tests := []struct { name string basePath string @@ -548,3 +560,17 @@ func relativePath(basePath, givenPath string) string { return relPath } + +// indexedPath returns how the given cwd-relative path is keyed in the index (absolute and always posix) +func indexedPath(t *testing.T, rel string) file.Path { + t.Helper() + cwd, err := os.Getwd() + require.NoError(t, err) + // filepath (not path): cwd is a native host path, so it must be joined with the host separator before + // being converted to the posix form the index is keyed on + p := filepath.Join(cwd, rel) + if windows.HostRunningOnWindows() { + p = windows.ToPosix(p) + } + return file.Path(p) +} diff --git a/syft/internal/fileresolver/directory_roundtrip_test.go b/syft/internal/fileresolver/directory_roundtrip_test.go new file mode 100644 index 000000000..22f39cc18 --- /dev/null +++ b/syft/internal/fileresolver/directory_roundtrip_test.go @@ -0,0 +1,48 @@ +package fileresolver + +import ( + "context" + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + stereoscopeFile "github.com/anchore/stereoscope/pkg/file" +) + +// a location handed out by the resolver should be resolvable again by its own path. This is what the file +// digest cataloger relies on, and it silently found nothing on windows (see #5325). +func TestDirectory_LocationPathsRoundTrip(t *testing.T) { + // resolve symlinks up front (e.g. /var -> /private/var on macOS) so the cwd prefix matches the indexed paths + root, err := filepath.EvalSymlinks(t.TempDir()) + require.NoError(t, err) + require.NoError(t, os.MkdirAll(filepath.Join(root, "sub", "dir"), 0o755)) + for _, p := range []string{"top.txt", filepath.Join("sub", "mid.txt"), filepath.Join("sub", "dir", "deep.txt")} { + require.NoError(t, os.WriteFile(filepath.Join(root, p), []byte(p), 0o600)) + } + + // mirror `syft dir:.` from within the scanned directory + t.Chdir(root) + + resolver, err := NewFromDirectory(".", "") + require.NoError(t, err) + + var regular int + for loc := range resolver.AllLocations(context.Background()) { + md, err := resolver.FileMetadataByLocation(loc) + require.NoError(t, err) + if md.Type != stereoscopeFile.TypeRegular { + continue + } + regular++ + + assert.NotContains(t, loc.RealPath, `\`, "location paths should be posix") + + found, err := resolver.FilesByPath(loc.RealPath) + require.NoError(t, err) + assert.Len(t, found, 1, "unable to resolve %q by its own path", loc.RealPath) + } + assert.Equal(t, 3, regular) +} diff --git a/syft/internal/fileresolver/directory_windows_test.go b/syft/internal/fileresolver/directory_windows_test.go deleted file mode 100644 index 115fb30a3..000000000 --- a/syft/internal/fileresolver/directory_windows_test.go +++ /dev/null @@ -1,126 +0,0 @@ -package fileresolver - -import "testing" - -func Test_windowsToPosix(t *testing.T) { - type args struct { - windowsPath string - } - tests := []struct { - name string - args args - wantPosixPath string - }{ - { - name: "basic case", - args: args{ - windowsPath: `C:\some\windows\place`, - }, - wantPosixPath: "/c/some/windows/place", - }, - { - name: "escaped case", - args: args{ - windowsPath: `C:\\some\\windows\\place`, - }, - wantPosixPath: "/c/some/windows/place", - }, - { - name: "forward slash", - args: args{ - windowsPath: `C:/foo/bar`, - }, - wantPosixPath: "/c/foo/bar", - }, - { - name: "mix slash", - args: args{ - windowsPath: `C:\foo/bar\`, - }, - wantPosixPath: "/c/foo/bar", - }, - { - name: "case sensitive case", - args: args{ - windowsPath: `C:\Foo/bAr\`, - }, - wantPosixPath: "/c/Foo/bAr", - }, - { - name: "special char case", - args: args{ - windowsPath: `C:\ふー\バー`, - }, - wantPosixPath: "/c/ふー/バー", - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - if gotPosixPath := windowsToPosix(tt.args.windowsPath); gotPosixPath != tt.wantPosixPath { - t.Errorf("windowsToPosix() = %v, want %v", gotPosixPath, tt.wantPosixPath) - } - }) - } -} - -func Test_posixToWindows(t *testing.T) { - type args struct { - posixPath string - } - tests := []struct { - name string - args args - wantWindowsPath string - }{ - { - name: "basic case", - args: args{ - posixPath: "/c/some/windows/place", - }, - wantWindowsPath: `C:\some\windows\place`, - }, - { - name: "escaped case", - args: args{ - posixPath: "/c/some/windows/place", - }, - wantWindowsPath: `C:\\some\\windows\\place`, - }, - { - name: "forward slash", - args: args{ - posixPath: "/c/foo/bar", - }, - wantWindowsPath: `C:/foo/bar`, - }, - { - name: "mix slash", - args: args{ - posixPath: "/c/foo/bar", - }, - wantWindowsPath: `C:\foo/bar\`, - }, - { - name: "case sensitive case", - args: args{ - posixPath: "/c/Foo/bAr", - }, - wantWindowsPath: `C:\Foo/bAr\`, - }, - { - name: "special char case", - args: args{ - posixPath: "/c/ふー/バー", - }, - wantWindowsPath: `C:\ふー\バー`, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - if gotWindowsPath := posixToWindows(tt.args.posixPath); gotWindowsPath != tt.wantWindowsPath { - t.Errorf("posixToWindows() = %v, want %v", gotWindowsPath, tt.wantWindowsPath) - } - }) - } -} diff --git a/syft/internal/fileresolver/file_indexer_test.go b/syft/internal/fileresolver/file_indexer_test.go index d9ba1d8c9..a4e0364d5 100644 --- a/syft/internal/fileresolver/file_indexer_test.go +++ b/syft/internal/fileresolver/file_indexer_test.go @@ -3,13 +3,11 @@ package fileresolver import ( "io/fs" "os" - "path" + "path/filepath" "testing" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - - "github.com/anchore/stereoscope/pkg/file" ) // - Verify that both the parent and the path are indexed @@ -38,11 +36,7 @@ func Test_index(t *testing.T) { info, err := os.Stat(test.path) assert.NoError(t, err) - // note: the index uses absolute paths, so assertions MUST keep this in mind - cwd, err := os.Getwd() - require.NoError(t, err) - - p := file.Path(path.Join(cwd, test.path)) + p := indexedPath(t, test.path) assert.Equal(t, true, tree.HasPath(p)) exists, ref, err := tree.File(p) assert.Equal(t, true, exists) @@ -71,7 +65,9 @@ func Test_ignoresPathIfFiltered(t *testing.T) { testPath := "testdata/system_paths/target/home/place" cwd, cwdErr := os.Getwd() require.NoError(t, cwdErr) - ignorePath := path.Join(cwd, testPath) + // filepath (not path): filters see the native absolute path (before posix normalization), so the + // expected value must be joined with the host separator to match on windows + ignorePath := filepath.Join(cwd, testPath) filterFn := func(_, path string, _ os.FileInfo, _ error) error { if path == ignorePath { return ErrSkipPath @@ -91,7 +87,8 @@ func Test_ignoresPathIfParentFiltered(t *testing.T) { cwd, cwdErr := os.Getwd() require.NoError(t, cwdErr) - ignorePath := path.Join(cwd, parentPath) + // filepath (not path): see Test_ignoresPathIfFiltered + ignorePath := filepath.Join(cwd, parentPath) filterFn := func(_, path string, _ os.FileInfo, _ error) error { if path == ignorePath { return fs.SkipDir diff --git a/syft/internal/fileresolver/filetree_resolver.go b/syft/internal/fileresolver/filetree_resolver.go index 68b3b8395..20c60b57a 100644 --- a/syft/internal/fileresolver/filetree_resolver.go +++ b/syft/internal/fileresolver/filetree_resolver.go @@ -6,6 +6,7 @@ import ( "fmt" "io" "os" + "path/filepath" stereoscopeFile "github.com/anchore/stereoscope/pkg/file" "github.com/anchore/stereoscope/pkg/filetree" @@ -34,8 +35,17 @@ func nativeOSFileOpener(ref stereoscopeFile.Reference) (io.ReadCloser, error) { return stereoscopeFile.NewLazyReadCloser(filePath), nil } +// requestPath converts a user (chroot) path into the path the file tree is keyed on. The tree is always posix +// (windows paths are volume-encoded, e.g. /c/some/path), while ToNativePath yields a native host path. func (r *FiletreeResolver) requestPath(userPath string) (string, error) { - return r.Chroot.ToNativePath(userPath) + nativePath, err := r.Chroot.ToNativePath(userPath) + if err != nil { + return "", err + } + if windows.HostRunningOnWindows() { + return windows.ToPosix(nativePath), nil + } + return nativePath, nil } // responsePath takes a path from the underlying fs domain and converts it to a path that is relative to the root of the file resolver. @@ -85,10 +95,6 @@ func (r FiletreeResolver) FilesByPath(userPaths ...string) ([]file.Location, err continue } - if windows.HostRunningOnWindows() { - userStrPath = windows.ToPosix(userStrPath) - } - if ref.HasReference() { references = append(references, file.NewVirtualLocationFromDirectory( @@ -103,8 +109,19 @@ func (r FiletreeResolver) FilesByPath(userPaths ...string) ([]file.Location, err return references, nil } +// requestGlob is the glob equivalent of requestPath. func (r FiletreeResolver) requestGlob(pattern string) (string, error) { - return r.Chroot.ToNativeGlob(pattern) + glob, err := r.Chroot.ToNativeGlob(pattern) + if err != nil { + return "", err + } + // only a glob that was anchored to the root comes back native. Globs that start with a wildcard + // (e.g. **/foo) are returned untouched and are already posix, and ToPosix would root them at "/". + // This uses filepath (not path) since we are asking whether this is a native absolute path. + if windows.HostRunningOnWindows() && filepath.IsAbs(glob) { + return windows.ToPosix(glob), nil + } + return glob, nil } // FilesByGlob returns all file.References that match the given path glob pattern from any layer in the image. diff --git a/syft/internal/windows/path.go b/syft/internal/windows/path.go index c091b89fd..fc16cbe5f 100644 --- a/syft/internal/windows/path.go +++ b/syft/internal/windows/path.go @@ -35,6 +35,10 @@ func ToPosix(windowsPath string) (posixPath string) { func FromPosix(posixPath string) (windowsPath string) { // decode the volume (e.g. /c/ --> C:\\) - There should always be a volume name. // The volume may be a UNC path (e.g. /\\localhost\C$\ --> \\localhost\C$\) + if !strings.HasPrefix(posixPath, "/") { + // not a volume-encoded posix path (e.g. relative or already native), so there is nothing to decode + return posixPath + } pathFields := strings.Split(posixPath, "/") if len(pathFields) < 2 { // no separator means no volume to decode diff --git a/syft/internal/windows/path_windows_test.go b/syft/internal/windows/path_windows_test.go new file mode 100644 index 000000000..ad278fb9e --- /dev/null +++ b/syft/internal/windows/path_windows_test.go @@ -0,0 +1,35 @@ +package windows + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestToPosix(t *testing.T) { + tests := map[string]string{ + `C:\some\windows\place`: "/c/some/windows/place", + `C:\\some\\windows\\place`: "/c/some/windows/place", + `C:/foo/bar`: "/c/foo/bar", + `C:\foo/bar\`: "/c/foo/bar", + `C:\Foo/bAr\`: "/c/Foo/bAr", + `C:\ふー\バー`: "/c/ふー/バー", + } + for in, want := range tests { + assert.Equal(t, want, ToPosix(in), in) + } +} + +func TestFromPosix(t *testing.T) { + tests := map[string]string{ + "/c/some/windows/place": `C:\some\windows\place`, + "/c/Foo/bAr": `C:\Foo\bAr`, + "/c/ふー/バー": `C:\ふー\バー`, + // non volume-encoded input is left alone + "relative/path": "relative/path", + `D:\already\native`: `D:\already\native`, + } + for in, want := range tests { + assert.Equal(t, want, FromPosix(in), in) + } +} diff --git a/syft/source/directorysource/directory_source.go b/syft/source/directorysource/directory_source.go index dcbbeaa62..259ab2cac 100644 --- a/syft/source/directorysource/directory_source.go +++ b/syft/source/directorysource/directory_source.go @@ -216,7 +216,8 @@ func cleanDirPath(path, base string) string { relPath, err := filepath.Rel(cleanBase, cleanRoot) if err == nil { - path = relPath + // filepath.Rel yields a native path, but source names are posix like every other path syft reports + path = filepath.ToSlash(relPath) } // this is odd, but this means we can't use base } diff --git a/syft/source/directorysource/directory_source_test.go b/syft/source/directorysource/directory_source_test.go index fc4160e58..88e7b3430 100644 --- a/syft/source/directorysource/directory_source_test.go +++ b/syft/source/directorysource/directory_source_test.go @@ -4,6 +4,7 @@ import ( "io/fs" "os" "path/filepath" + "runtime" "testing" "github.com/google/go-cmp/cmp" @@ -325,6 +326,9 @@ func Test_DirectorySource_Exclusions(t *testing.T) { } func Test_getDirectoryExclusionFunctions_crossPlatform(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("emulates windows paths on a posix host, see directory_source_win_test.go for the windows equivalent") + } testCases := []struct { desc string root string @@ -465,10 +469,11 @@ func Test_DirectorySource_ID(t *testing.T) { testutil.Chdir(t, "..") // run with source/testdata tests := []struct { - name string - cfg Config - want artifact.ID - wantErr require.ErrorAssertionFunc + name string + cfg Config + want artifact.ID + wantErr require.ErrorAssertionFunc + skipOnWindows bool }{ { name: "empty", @@ -486,6 +491,8 @@ func Test_DirectorySource_ID(t *testing.T) { name: "with odd unclean path through non-existent directory", cfg: Config{Path: "testdata/does-not-exist/../"}, wantErr: require.Error, + // windows cleans the path lexically, so the non-existent directory is never visited + skipOnWindows: true, }, { name: "to a file (not a directory)", @@ -543,6 +550,9 @@ func Test_DirectorySource_ID(t *testing.T) { } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { + if tt.skipOnWindows && runtime.GOOS == "windows" { + t.Skip("posix-specific path behavior") + } if tt.wantErr == nil { tt.wantErr = require.NoError } diff --git a/syft/source/directorysource/directory_source_win_test.go b/syft/source/directorysource/directory_source_win_test.go index 8f99a59d3..d7d38f86a 100644 --- a/syft/source/directorysource/directory_source_win_test.go +++ b/syft/source/directorysource/directory_source_win_test.go @@ -56,8 +56,9 @@ func Test_DirectorySource_crossPlatformExclusions(t *testing.T) { require.NoError(t, err) for _, f := range fns { - result := f(test.path, nil, nil) - require.Equal(t, test.match, result) + // a matched exclusion is signaled by a skip error + err := f("", test.path, nil, nil) + require.Equal(t, test.match, err != nil) } }) } diff --git a/syft/source/filesource/file_source_test.go b/syft/source/filesource/file_source_test.go index 87fe2a795..bba5c61e2 100644 --- a/syft/source/filesource/file_source_test.go +++ b/syft/source/filesource/file_source_test.go @@ -6,6 +6,7 @@ import ( "os/exec" "path" "path/filepath" + "runtime" "syscall" "testing" @@ -96,6 +97,9 @@ func TestNewFromFile(t *testing.T) { } func TestNewFromFile_WithArchive(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("archive fixtures are generated with a shell script") + } testutil.Chdir(t, "..") // run with source/testdata testCases := []struct { From fdeb78fc193f22302f9fc201cd21495e32a586ae Mon Sep 17 00:00:00 2001 From: Alex Goodman Date: Wed, 30 Sep 2026 10:58:11 -0400 Subject: [PATCH 36/40] Speed up unit tests in CI (#5360) * ci: drop gen-5 instance families from the test runner m5a runners run the unit suite ~2.3x slower than m8id (15m vs 6.5m for `go test`), and when spot capacity for newer families dries up runs-on falls back to them. Keeping gen-6 through gen-8 leaves a wide spot pool without the slow tail. Signed-off-by: Alex Goodman * test: shrink decompression bomb fixtures The bomb and heap-bound tests pushed 100MB-16GB through gzip and parsers, which under `-race` made them the slowest tests in the unit suite. Each now uses the smallest fixture that still separates the bounded path from the unbounded one. - arch mtree caps are measured at a small cap and scaled to the shipped constants, so raising a cap still fails the budget - snap and debian bombs expand to 32MB instead of 512MB (4x their 8MB budgets) - golang ELF bombs still declare 256MB but deliver 64MB Signed-off-by: Alex Goodman --------- Signed-off-by: Alex Goodman --- .github/workflows/validations.yaml | 2 +- syft/pkg/cataloger/arch/parse_alpm_db_test.go | 57 ++++++++++++------- .../debian/parse_deb_archive_test.go | 6 +- .../cataloger/golang/scan_binary_bomb_test.go | 53 +++++++++-------- .../snap/parse_kernel_changelog_test.go | 4 +- 5 files changed, 76 insertions(+), 46 deletions(-) diff --git a/.github/workflows/validations.yaml b/.github/workflows/validations.yaml index 358cbadbf..95092a476 100644 --- a/.github/workflows/validations.yaml +++ b/.github/workflows/validations.yaml @@ -27,7 +27,7 @@ jobs: # enough; a 4 vCPU instance can't pull much more than that anyway. # note: s3-cache intentionally omitted -- PR runs are untrusted and must not write to the # shared cache backend that the trusted release workflow reads from (cache poisoning). - runs-on: &test-runner "runs-on=${{ github.run_id }}/cpu=4+8/ram=16+32/family=m5+m6+m7+m8+c5+c6+c7+c8/spot=price-capacity-optimized/volume=100gb" + runs-on: &test-runner "runs-on=${{ github.run_id }}/cpu=4+8/ram=16+32/family=m6+m7+m8+c6+c7+c8/spot=price-capacity-optimized/volume=100gb" permissions: contents: read steps: diff --git a/syft/pkg/cataloger/arch/parse_alpm_db_test.go b/syft/pkg/cataloger/arch/parse_alpm_db_test.go index 6b26209e8..c147fa330 100644 --- a/syft/pkg/cataloger/arch/parse_alpm_db_test.go +++ b/syft/pkg/cataloger/arch/parse_alpm_db_test.go @@ -492,12 +492,16 @@ func Test_parseMtree_malformedInput(t *testing.T) { // bytes still costs multiples of its own size in heap. Asserting the error would keep passing if the // line cap were removed, because the byte cap would still reject a large enough input eventually. func Test_parseMtree_lineCapBoundsHeap(t *testing.T) { - // newline-only lines are the worst case: minimum bytes per entry, so the most entries per byte - const payloadSize = 16 * intFile.MB - spec := bytes.Repeat([]byte{'\n'}, payloadSize) + // the property is linear in the cap, so it is measured at a small cap and scaled to the shipped + // one. Running it at 300k lines means an uncapped parse costing ~16GB, which under -race is most + // of a CI minute for a test with no concurrency in it. + const ( + lineCap = 4096 + payloadSize = 64 * lineCap + ) - // well inside the byte cap, so only the line cap can stop this - require.Less(t, int64(payloadSize), int64(maxMtreeSize)) + // newline-only lines are the worst case: minimum bytes per entry, so the most entries per byte + spec := bytes.Repeat([]byte{'\n'}, payloadSize) unbounded := testutils.MeasureAlloc(t, func() { // the same listing with the line cap lifted, to prove the fixture is really a bomb @@ -507,19 +511,29 @@ func Test_parseMtree_lineCapBoundsHeap(t *testing.T) { require.Greater(t, unbounded, uint64(payloadSize), "fixture did not actually cost more than its own size; it is no longer a bomb") - bounded := testutils.MeasureAlloc(t, func() { - _, err := parseMtreeWithLimits(gzipOf(t, spec), maxMtreeSize, maxMtreeLines) - require.ErrorIs(t, err, errTooManyMtreeLines) - }) + boundedAt := func(limit int) uint64 { + gz := gzipOf(t, spec) + return testutils.MeasureAlloc(t, func() { + _, err := parseMtreeWithLimits(gz, maxMtreeSize, limit) + require.ErrorIs(t, err, errTooManyMtreeLines) + }) + } + bounded := boundedAt(lineCap) - t.Logf("unbounded allocated %d bytes, bounded allocated %d bytes", unbounded, bounded) + // the marginal cost between two caps, so fixed setup (gzip reader, parser) does not get scaled + perLine := (boundedAt(2*lineCap) - bounded) / lineCap + atShippedCap := perLine * maxMtreeLines + t.Logf("unbounded allocated %d bytes, bounded allocated %d bytes (%d per line, ~%d at the shipped cap)", + unbounded, bounded, perLine, atShippedCap) - // measured at ~238MB against ~16.5GB uncapped, so the cap is doing its job, but note what it - // actually buys: 300k entries still cost a few hundred MB, and that is per concurrent cataloger. - // The budget is set to catch the cap being removed or raised by an order of magnitude, not to pin - // the exact figure. - assert.Less(t, bounded, uint64(512*intFile.MB), + assert.Less(t, bounded, unbounded/10, "the line cap has to stop the parse before it builds an entry per line") + + // measured at ~238MB for the shipped 300k lines, so the cap is doing its job, but note what it + // actually buys: that is still a few hundred MB, per concurrent cataloger. The budget is set to + // catch the cap being raised by an order of magnitude, not to pin the exact figure. + assert.Less(t, atShippedCap, uint64(512*intFile.MB), + "the shipped line cap allows more heap than budgeted") } // Test_parseMtree_sizeCapBoundsHeap is the byte-cap counterpart to the line-cap budget above. @@ -532,9 +546,12 @@ func Test_parseMtree_lineCapBoundsHeap(t *testing.T) { // be ordinary for the byte cap to be the thing under test, and there have to be few enough of them // that the line cap does not fire first. func Test_parseMtree_sizeCapBoundsHeap(t *testing.T) { + // same reasoning as the line cap: measured at a small byte cap, since the shipped 64MB cap means + // gzipping and parsing ~200MB of fixture, which is slow under -race and proves nothing more. const ( + sizeCap = intFile.MB lineLen = 1024 - numLines = 96 * 1024 // ~96MB across ~98k lines: past the 64MB cap, well under the 300k line cap + numLines = 2 * sizeCap / lineLen // twice the cap, and far under the line cap ) var raw bytes.Buffer @@ -542,7 +559,7 @@ func Test_parseMtree_sizeCapBoundsHeap(t *testing.T) { for range numLines { raw.Write(line) } - require.Greater(t, int64(raw.Len()), int64(maxMtreeSize), "fixture must exceed the byte cap") + require.Greater(t, int64(raw.Len()), int64(sizeCap), "fixture must exceed the byte cap") require.Less(t, numLines, maxMtreeLines, "and must not trip the line cap first") unbounded := testutils.MeasureAlloc(t, func() { @@ -554,11 +571,13 @@ func Test_parseMtree_sizeCapBoundsHeap(t *testing.T) { "fixture did not actually cost more than its own size; it is no longer a bomb") bounded := testutils.MeasureAlloc(t, func() { - _, err := parseMtreeWithLimits(gzipOf(t, raw.Bytes()), maxMtreeSize, maxMtreeLines) + _, err := parseMtreeWithLimits(gzipOf(t, raw.Bytes()), sizeCap, maxMtreeLines) require.ErrorIs(t, err, errMtreeTooLarge) }) t.Logf("unbounded allocated %d bytes, bounded allocated %d bytes", unbounded, bounded) - assert.Less(t, bounded, uint64(512*intFile.MB), + + // the old budget was 512MB against the 64MB cap, so 8x the cap + assert.Less(t, bounded, uint64(8*sizeCap), "the size cap has to stop the read at the cap, not after the member is drained") } diff --git a/syft/pkg/cataloger/debian/parse_deb_archive_test.go b/syft/pkg/cataloger/debian/parse_deb_archive_test.go index ad8c00ddf..8b916a476 100644 --- a/syft/pkg/cataloger/debian/parse_deb_archive_test.go +++ b/syft/pkg/cataloger/debian/parse_deb_archive_test.go @@ -373,9 +373,11 @@ func Test_processDataTar_capsCopyrightFiles(t *testing.T) { // than after the fact. The existing bounds test uses a payload small enough that buffering the whole // thing would pass anyway; this one is large enough that the difference shows up as bytes. func Test_decompressionStream_failsEarlyInsteadOfBuffering(t *testing.T) { - const expanded = 512 * intFile.MB + // 4x the 8MB budget below is plenty to tell bounded from unbounded. Going bigger only makes the + // fixture slower to build and inflate, which is most of this test's time under -race. + const expanded = 32 * intFile.MB - // well under 1% of input expanding to 512MB, so heap is what separates a streaming bound from a + // well under 1% of input expanding to 32MB, so heap is what separates a streaming bound from a // bound checked after the read completes var gz bytes.Buffer gw := gzip.NewWriter(&gz) diff --git a/syft/pkg/cataloger/golang/scan_binary_bomb_test.go b/syft/pkg/cataloger/golang/scan_binary_bomb_test.go index 6369bbda7..d11ffe7fa 100644 --- a/syft/pkg/cataloger/golang/scan_binary_bomb_test.go +++ b/syft/pkg/cataloger/golang/scan_binary_bomb_test.go @@ -20,19 +20,25 @@ import ( // Test_getBuildInfo_compressedSectionBomb covers the reason readBuildInfo exists: debug/buildinfo opens // ELF files with debug/elf itself, and elf.NewFile expands the section-name string table as it parses, // so an oversized compression header there is an unbounded allocation on a path elfutil.NewFile never -// sees. The fixture is a real zlib stream, so it delivers every byte its header promises. +// sees. The fixture is a real zlib stream, but it stops short of what its header promises: delivering +// the full 256MB only made it slower to build and inflate, mostly under -race. func Test_getBuildInfo_compressedSectionBomb(t *testing.T) { - const declared = 256 << 20 // comfortably over elfutil's bound, small enough to allocate in a test + const ( + declared = 256 << 20 // comfortably over elfutil's bound, which only reads the header + // debug/elf allocates the declared size up front today, so delivery barely matters. Twice the + // guarded budget keeps the fixture a bomb if it ever switches to reading in chunks. + delivered = 64 << 20 + ) - bomb := elfWithCompressedNameTable(t, declared) - t.Logf("%d byte fixture declares a %d byte section name table", len(bomb), declared) + bomb := elfWithCompressedNameTable(t, declared, delivered) + t.Logf("%d byte fixture declares a %d byte section name table and delivers %d", len(bomb), declared, delivered) // the unguarded path is the thing being defended against: prove the fixture really is a bomb unguarded := testutils.MeasureAlloc(t, func() { _, err := buildinfo.Read(bytes.NewReader(bomb)) t.Logf("buildinfo.Read err: %v", err) }) - assert.Greater(t, unguarded, uint64(declared), "fixture did not actually deliver the declared bytes") + assert.Greater(t, unguarded, uint64(delivered), "fixture did not actually deliver its bytes") guarded := testutils.MeasureAlloc(t, func() { _, err := getBuildInfo(bytes.NewReader(bomb)) @@ -47,17 +53,22 @@ func Test_getBuildInfo_compressedSectionBomb(t *testing.T) { // goversion opens the file itself and reads .symtab plus the string table it links, and those are // expanded lazily, so the section-name table bound getBuildInfo applies never reaches them. func Test_getCryptoInformation_compressedSymtabBomb(t *testing.T) { - const declared = 256 << 20 // comfortably over elfutil's bound, small enough to allocate in a test + const ( + declared = 256 << 20 // comfortably over elfutil's bound, which only reads the header + // debug/elf allocates the declared size up front today, so delivery barely matters. Twice the + // guarded budget keeps the fixture a bomb if it ever switches to reading in chunks. + delivered = 64 << 20 + ) - bomb := elfWithCompressedSymtab(t, declared) - t.Logf("%d byte fixture declares a %d byte symbol table", len(bomb), declared) + bomb := elfWithCompressedSymtab(t, declared, delivered) + t.Logf("%d byte fixture declares a %d byte symbol table and delivers %d", len(bomb), declared, delivered) // the unguarded path is the thing being defended against: prove the fixture really is a bomb unguarded := testutils.MeasureAlloc(t, func() { _, err := version.ReadExeFromReader(bytes.NewReader(bomb)) t.Logf("goversion err: %v", err) }) - assert.Greater(t, unguarded, uint64(declared), "fixture did not actually deliver the declared bytes") + assert.Greater(t, unguarded, uint64(delivered), "fixture did not actually deliver its bytes") guarded := testutils.MeasureAlloc(t, func() { _, err := getCryptoInformation(bytes.NewReader(bomb)) @@ -85,12 +96,12 @@ func Test_getCryptoInformation_passesThroughNonELF(t *testing.T) { } // elfWithCompressedSymtab builds a minimal ELF64 whose .symtab is SHF_COMPRESSED, declaring `declared` -// decompressed bytes and genuinely delivering them. The name table is left uncompressed so the file gets +// decompressed bytes and genuinely delivering `delivered` of them. The name table is left uncompressed so the file gets // past the bound getBuildInfo already applies, which is the point: this is the section that bound misses. -func elfWithCompressedSymtab(t *testing.T, declared uint64) []byte { +func elfWithCompressedSymtab(t *testing.T, declared, delivered uint64) []byte { t.Helper() - payload := make([]byte, declared) + payload := make([]byte, delivered) var compressed bytes.Buffer zw := zlib.NewWriter(&compressed) _, err := zw.Write(payload) @@ -150,12 +161,12 @@ func elfWithCompressedSymtab(t *testing.T, declared uint64) []byte { } // elfWithCompressedNameTable builds a minimal ELF64 whose only real section is a SHF_COMPRESSED .shstrtab -// declaring `declared` decompressed bytes and genuinely delivering them. Only the test that runs the +// declaring `declared` decompressed bytes and genuinely delivering `delivered` of them. Only the test that runs the // unguarded path needs delivery; use elfDeclaringNameTable everywhere else, since building this one costs -// `declared` bytes of allocation plus a zlib compress of them. -func elfWithCompressedNameTable(t *testing.T, declared uint64) []byte { +// `delivered` bytes of allocation plus a zlib compress of them. +func elfWithCompressedNameTable(t *testing.T, declared, delivered uint64) []byte { t.Helper() - return elfNameTableFixture(t, declared, true) + return elfNameTableFixture(t, declared, delivered) } // elfDeclaringNameTable declares `declared` bytes without delivering them. CheckSectionNameTable reads the @@ -163,19 +174,15 @@ func elfWithCompressedNameTable(t *testing.T, declared uint64) []byte { // bound itself never needs the stream to be real. func elfDeclaringNameTable(t *testing.T, declared uint64) []byte { t.Helper() - return elfNameTableFixture(t, declared, false) + return elfNameTableFixture(t, declared, 64) } -func elfNameTableFixture(t *testing.T, declared uint64, deliver bool) []byte { +func elfNameTableFixture(t *testing.T, declared, delivered uint64) []byte { t.Helper() // the decompressed name table only has to start with the section names; the rest is padding that // exists purely to make the declared size real - size := declared - if !deliver { - size = 64 - } - payload := make([]byte, size) + payload := make([]byte, delivered) copy(payload, "\x00.shstrtab\x00") var compressed bytes.Buffer diff --git a/syft/pkg/cataloger/snap/parse_kernel_changelog_test.go b/syft/pkg/cataloger/snap/parse_kernel_changelog_test.go index a1806a822..0e3c17c1a 100644 --- a/syft/pkg/cataloger/snap/parse_kernel_changelog_test.go +++ b/syft/pkg/cataloger/snap/parse_kernel_changelog_test.go @@ -318,7 +318,9 @@ func TestParseKernelChangelog(t *testing.T) { // thing holding this path, and it is incidental: switching to io.ReadAll or a bufio.Reader to pick up // long lines would remove it silently, which is what this test is here to catch. func TestParseKernelChangelog_gzipBombStaysBounded(t *testing.T) { - const expanded = 512 * 1024 * 1024 + // 4x the 8MB budget below is plenty to tell bounded from unbounded. Going bigger only makes the + // fixture slower to build and inflate, which is most of this test's time under -race. + const expanded = 32 * 1024 * 1024 // one enormous line with no newline, the worst case for a line scanner var gz bytes.Buffer From c511d509ef5428a396e5374fe3981c69744c95ac Mon Sep 17 00:00:00 2001 From: Will Murphy Date: Wed, 30 Sep 2026 11:07:12 -0400 Subject: [PATCH 37/40] fix: match opensource.org license URLs across SPDX URL forms (#5361) SPDX license list 3.29.0 rewrote every opensource.org seeAlso URL from opensource.org/licenses/ to opensource.org/license/ (and e.g. opensource.org/license/mit/ to opensource.org/license/MIT). Package metadata in the wild (Maven POM , npm, etc.) still uses the older forms, so regenerating the list would stop LicenseByURL from resolving them. Fall back to a normalized index of the opensource.org entries that ignores the www. prefix, /licenses/ vs /license/, case, and a trailing slash. The index is built from the generated urlToLicense map, so it works with both the current and the updated license list without changes to the generator. URL forms that would normalize to different licenses are left out rather than guessed. Signed-off-by: Will Murphy --- internal/spdxlicense/license.go | 43 +++++++++++ internal/spdxlicense/license_url_test.go | 90 ++++++++++++++++++++++++ 2 files changed, 133 insertions(+) diff --git a/internal/spdxlicense/license.go b/internal/spdxlicense/license.go index ff8bd1265..d23491e5c 100644 --- a/internal/spdxlicense/license.go +++ b/internal/spdxlicense/license.go @@ -44,6 +44,7 @@ type LicenseInfo struct { // LicenseByURL returns the license ID and name for a given URL from the SPDX license list. // The URL should match one of the URLs in the seeAlso field of an SPDX license. // The scheme (http:// or https://) is stripped before lookup, so both schemes match. +// opensource.org URLs additionally match regardless of the form used (see normalizeOpenSourceOrgURL). func LicenseByURL(url string) (LicenseInfo, bool) { url = strings.TrimSpace(url) url = stripScheme(url) @@ -52,9 +53,51 @@ func LicenseByURL(url string) (LicenseInfo, bool) { ID: id, }, true } + if normalized, ok := normalizeOpenSourceOrgURL(url); ok { + if id, exists := openSourceOrgURLToLicense[normalized]; exists { + return LicenseInfo{ + ID: id, + }, true + } + } return LicenseInfo{}, false } +// openSourceOrgURLToLicense indexes the opensource.org entries of urlToLicense by normalized URL. +var openSourceOrgURLToLicense = buildOpenSourceOrgURLToLicense(urlToLicense) + +func buildOpenSourceOrgURLToLicense(urls map[string]string) map[string]string { + index := make(map[string]string) + ambiguous := make(map[string]bool) + for url, id := range urls { + normalized, ok := normalizeOpenSourceOrgURL(url) + if !ok || ambiguous[normalized] { + continue + } + if existing, exists := index[normalized]; exists && existing != id { + // don't guess between licenses that only differ by URL form + delete(index, normalized) + ambiguous[normalized] = true + continue + } + index[normalized] = id + } + return index +} + +// normalizeOpenSourceOrgURL reduces a scheme-less opensource.org URL to a canonical form. +// SPDX has changed these URLs over time (e.g. 3.29.0 replaced opensource.org/licenses/MIT and +// opensource.org/license/mit/ with opensource.org/license/MIT), while package metadata in the wild +// still uses the older forms. The www. prefix, /licenses/ vs /license/, case, and trailing slash are ignored. +func normalizeOpenSourceOrgURL(url string) (string, bool) { + url = strings.TrimPrefix(strings.ToLower(url), "www.") + if !strings.HasPrefix(url, "opensource.org/") { + return "", false + } + url = strings.Replace(url, "opensource.org/licenses/", "opensource.org/license/", 1) + return strings.TrimSuffix(url, "/"), true +} + // stripScheme removes http:// or https:// prefix from a URL. // This allows a single map entry to match both schemes. func stripScheme(url string) string { diff --git a/internal/spdxlicense/license_url_test.go b/internal/spdxlicense/license_url_test.go index 2ae646308..490976de3 100644 --- a/internal/spdxlicense/license_url_test.go +++ b/internal/spdxlicense/license_url_test.go @@ -123,6 +123,96 @@ func TestLicenseByURL_AlternateScheme(t *testing.T) { } } +func TestLicenseByURL_OpenSourceOrgVariants(t *testing.T) { + // SPDX has changed the form of opensource.org URLs across list versions (e.g. 3.29.0 moved + // opensource.org/licenses/MIT to opensource.org/license/MIT); every form should keep resolving. + tests := []struct { + name string + url string + wantID string + }{ + {name: "old plural path", url: "http://opensource.org/licenses/MIT", wantID: "MIT"}, + {name: "new singular path", url: "https://opensource.org/license/MIT", wantID: "MIT"}, + {name: "lowercase with trailing slash", url: "https://opensource.org/license/mit/", wantID: "MIT"}, + {name: "plural path with trailing slash", url: "https://opensource.org/licenses/MIT/", wantID: "MIT"}, + {name: "www prefix", url: "https://www.opensource.org/licenses/MIT", wantID: "MIT"}, + {name: "uppercase host", url: "https://WWW.OpenSource.org/licenses/MIT", wantID: "MIT"}, + {name: "old plural path BSD-3-Clause", url: "https://opensource.org/licenses/BSD-3-Clause", wantID: "BSD-3-Clause"}, + {name: "new singular path BSD-3-Clause", url: "https://opensource.org/license/BSD-3-Clause", wantID: "BSD-3-Clause"}, + {name: "lowercase BSD-3-Clause", url: "https://opensource.org/licenses/bsd-3-clause", wantID: "BSD-3-Clause"}, + {name: "old plural path Apache-2.0", url: "http://opensource.org/licenses/Apache-2.0", wantID: "Apache-2.0"}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + info, found := LicenseByURL(tt.url) + if !found { + t.Fatalf("LicenseByURL(%q) found = false, want true", tt.url) + } + if info.ID != tt.wantID { + t.Errorf("LicenseByURL(%q) ID = %v, want %v", tt.url, info.ID, tt.wantID) + } + }) + } +} + +func TestLicenseByURL_NonOpenSourceOrgStaysExact(t *testing.T) { + // the relaxed matching is scoped to opensource.org; other hosts may have case-sensitive paths + if _, found := LicenseByURL("https://www.apache.org/licenses/license-2.0"); found { + t.Errorf("LicenseByURL() found = true for case-altered non-opensource.org URL, want false") + } + if _, found := LicenseByURL("https://opensource.org/licenses/not-a-real-license"); found { + t.Errorf("LicenseByURL() found = true for unknown opensource.org URL, want false") + } +} + +func TestNormalizeOpenSourceOrgURL(t *testing.T) { + tests := []struct { + url string + want string + wantOK bool + }{ + {url: "opensource.org/licenses/MIT", want: "opensource.org/license/mit", wantOK: true}, + {url: "opensource.org/license/mit/", want: "opensource.org/license/mit", wantOK: true}, + {url: "www.opensource.org/licenses/EPL-2.0", want: "opensource.org/license/epl-2.0", wantOK: true}, + {url: "lists.opensource.org/pipermail/license-discuss", wantOK: false}, + {url: "www.apache.org/licenses/LICENSE-2.0", wantOK: false}, + } + + for _, tt := range tests { + t.Run(tt.url, func(t *testing.T) { + got, ok := normalizeOpenSourceOrgURL(tt.url) + if ok != tt.wantOK { + t.Fatalf("normalizeOpenSourceOrgURL(%q) ok = %v, want %v", tt.url, ok, tt.wantOK) + } + if got != tt.want { + t.Errorf("normalizeOpenSourceOrgURL(%q) = %q, want %q", tt.url, got, tt.want) + } + }) + } +} + +func TestBuildOpenSourceOrgURLToLicense(t *testing.T) { + index := buildOpenSourceOrgURLToLicense(map[string]string{ + "opensource.org/licenses/MIT": "MIT", + "opensource.org/license/mit/": "MIT", + "opensource.org/licenses/Foo": "Foo-1.0", + "opensource.org/license/foo": "Foo-2.0", + "opensource.org/license/FOO/": "Foo-1.0", + "www.apache.org/licenses/Bar": "Bar", + }) + + if got := index["opensource.org/license/mit"]; got != "MIT" { + t.Errorf("index[mit] = %q, want MIT", got) + } + if got, exists := index["opensource.org/license/foo"]; exists { + t.Errorf("index[foo] = %q, want no entry for URL forms mapping to different licenses", got) + } + if len(index) != 1 { + t.Errorf("len(index) = %d, want 1 (non-opensource.org URLs excluded): %v", len(index), index) + } +} + func TestStripScheme(t *testing.T) { tests := []struct { name string From f21e439a636d00fa5f06360a1423321715b165c7 Mon Sep 17 00:00:00 2001 From: anchore-oss-update-bot Date: Wed, 30 Sep 2026 11:50:47 -0400 Subject: [PATCH 38/40] chore(deps): update SPDX license list (#5315) Signed-off-by: anchore-oss-update-bot Co-authored-by: anchore-oss-update-bot --- internal/spdxlicense/license_list.go | 453 +++++++++++++++------------ 1 file changed, 254 insertions(+), 199 deletions(-) diff --git a/internal/spdxlicense/license_list.go b/internal/spdxlicense/license_list.go index 78e330244..1a4357378 100644 --- a/internal/spdxlicense/license_list.go +++ b/internal/spdxlicense/license_list.go @@ -2,7 +2,7 @@ // This file was generated using data from https://spdx.org/licenses/licenses.json package spdxlicense -const Version = "3.28.0" +const Version = "3.29.0" var licenseIDs = map[string]string{ "0bsd": "0BSD", @@ -116,6 +116,7 @@ var licenseIDs = map[string]string{ "aswfdigitalassets1.0.0": "ASWF-Digital-Assets-1.0", "aswfdigitalassets1.1": "ASWF-Digital-Assets-1.1", "aswfdigitalassets1.1.0": "ASWF-Digital-Assets-1.1", + "atcgame": "atc-game", "baekmuk": "Baekmuk", "bahyph": "Bahyph", "barr": "Barr", @@ -142,8 +143,11 @@ var licenseIDs = map[string]string{ "briangladman2.0clause": "Brian-Gladman-2-Clause", "briangladman2clause": "Brian-Gladman-2-Clause", "briangladman3.0.0clause": "Brian-Gladman-3-Clause", + "briangladman3.0.0clausenoconversion": "Brian-Gladman-3-Clause-no-conversion", "briangladman3.0clause": "Brian-Gladman-3-Clause", + "briangladman3.0clausenoconversion": "Brian-Gladman-3-Clause-no-conversion", "briangladman3clause": "Brian-Gladman-3-Clause", + "briangladman3clausenoconversion": "Brian-Gladman-3-Clause-no-conversion", "bsd1.0.0clause": "BSD-1-Clause", "bsd1.0clause": "BSD-1-Clause", "bsd1clause": "BSD-1-Clause", @@ -154,6 +158,7 @@ var licenseIDs = map[string]string{ "bsd2.0.0clausenetbsd": "BSD-2-Clause", "bsd2.0.0clausepatent": "BSD-2-Clause-Patent", "bsd2.0.0clausepkgconfdisclaimer": "BSD-2-Clause-pkgconf-disclaimer", + "bsd2.0.0clauseposunchanged": "BSD-2-Clause-pos-unchanged", "bsd2.0.0clauseviews": "BSD-2-Clause-Views", "bsd2.0clause": "BSD-2-Clause", "bsd2.0clausedarwin": "BSD-2-Clause-Darwin", @@ -162,6 +167,7 @@ var licenseIDs = map[string]string{ "bsd2.0clausenetbsd": "BSD-2-Clause", "bsd2.0clausepatent": "BSD-2-Clause-Patent", "bsd2.0clausepkgconfdisclaimer": "BSD-2-Clause-pkgconf-disclaimer", + "bsd2.0clauseposunchanged": "BSD-2-Clause-pos-unchanged", "bsd2.0clauseviews": "BSD-2-Clause-Views", "bsd2clause": "BSD-2-Clause", "bsd2clausedarwin": "BSD-2-Clause-Darwin", @@ -170,6 +176,7 @@ var licenseIDs = map[string]string{ "bsd2clausenetbsd": "BSD-2-Clause", "bsd2clausepatent": "BSD-2-Clause-Patent", "bsd2clausepkgconfdisclaimer": "BSD-2-Clause-pkgconf-disclaimer", + "bsd2clauseposunchanged": "BSD-2-Clause-pos-unchanged", "bsd2clauseviews": "BSD-2-Clause-Views", "bsd3.0.0clause": "BSD-3-Clause", "bsd3.0.0clauseacpica": "BSD-3-Clause-acpica", @@ -184,6 +191,7 @@ var licenseIDs = map[string]string{ "bsd3.0.0clausenonuclearlicense2014": "BSD-3-Clause-No-Nuclear-License-2014", "bsd3.0.0clausenonuclearwarranty": "BSD-3-Clause-No-Nuclear-Warranty", "bsd3.0.0clauseopenmpi": "BSD-3-Clause-Open-MPI", + "bsd3.0.0clauseopenwebui": "BSD-3-Clause-OpenWebUI", "bsd3.0.0clausesun": "BSD-3-Clause-Sun", "bsd3.0.0clausetso": "BSD-3-Clause-Tso", "bsd3.0clause": "BSD-3-Clause", @@ -199,6 +207,7 @@ var licenseIDs = map[string]string{ "bsd3.0clausenonuclearlicense2014": "BSD-3-Clause-No-Nuclear-License-2014", "bsd3.0clausenonuclearwarranty": "BSD-3-Clause-No-Nuclear-Warranty", "bsd3.0clauseopenmpi": "BSD-3-Clause-Open-MPI", + "bsd3.0clauseopenwebui": "BSD-3-Clause-OpenWebUI", "bsd3.0clausesun": "BSD-3-Clause-Sun", "bsd3.0clausetso": "BSD-3-Clause-Tso", "bsd3clause": "BSD-3-Clause", @@ -214,6 +223,7 @@ var licenseIDs = map[string]string{ "bsd3clausenonuclearlicense2014": "BSD-3-Clause-No-Nuclear-License-2014", "bsd3clausenonuclearwarranty": "BSD-3-Clause-No-Nuclear-Warranty", "bsd3clauseopenmpi": "BSD-3-Clause-Open-MPI", + "bsd3clauseopenwebui": "BSD-3-Clause-OpenWebUI", "bsd3clausesun": "BSD-3-Clause-Sun", "bsd3clausetso": "BSD-3-Clause-Tso", "bsd4.0.0clause": "BSD-4-Clause", @@ -232,12 +242,15 @@ var licenseIDs = map[string]string{ "bsd4reno": "BSD-4.3RENO", "bsd4tahoe": "BSD-4.3TAHOE", "bsdadvertisingacknowledgement": "BSD-Advertising-Acknowledgement", + "bsdasktoendorse": "BSD-ask-to-endorse", "bsdattributionhpnddisclaimer": "BSD-Attribution-HPND-disclaimer", "bsdinfernonettverk": "BSD-Inferno-Nettverk", "bsdmarkmodifications": "BSD-Mark-Modifications", "bsdprotection": "BSD-Protection", + "bsdsourcealtgpl": "BSD-Source-alt-GPL", "bsdsourcebeginningfile": "BSD-Source-beginning-file", "bsdsourcecode": "BSD-Source-Code", + "bsdsourcecodenodisclaimer": "BSD-Source-Code-no-disclaimer", "bsdsystemics": "BSD-Systemics", "bsdsystemicsw3.0.0works": "BSD-Systemics-W3Works", "bsdsystemicsw3.0works": "BSD-Systemics-W3Works", @@ -246,6 +259,7 @@ var licenseIDs = map[string]string{ "bsl1.0": "BSL-1.0", "bsl1.0.0": "BSL-1.0", "buddy": "Buddy", + "bugroff": "Bugroff", "busl1": "BUSL-1.1", "busl1.1": "BUSL-1.1", "busl1.1.0": "BUSL-1.1", @@ -316,8 +330,11 @@ var licenseIDs = map[string]string{ "ccbync3.0": "CC-BY-NC-3.0", "ccbync3.0.0": "CC-BY-NC-3.0", "ccbync3.0.0de": "CC-BY-NC-3.0-DE", + "ccbync3.0.0igo": "CC-BY-NC-3.0-IGO", "ccbync3.0de": "CC-BY-NC-3.0-DE", + "ccbync3.0igo": "CC-BY-NC-3.0-IGO", "ccbync3de": "CC-BY-NC-3.0-DE", + "ccbync3igo": "CC-BY-NC-3.0-IGO", "ccbync4": "CC-BY-NC-4.0", "ccbync4.0": "CC-BY-NC-4.0", "ccbync4.0.0": "CC-BY-NC-4.0", @@ -592,6 +609,7 @@ var licenseIDs = map[string]string{ "fair": "Fair", "fbm": "FBM", "fdkaac": "FDK-AAC", + "fdkmpegh": "FDK-MPEG-H", "fergusontwofish": "Ferguson-Twofish", "frameworx1": "Frameworx-1.0", "frameworx1.0": "Frameworx-1.0", @@ -767,6 +785,9 @@ var licenseIDs = map[string]string{ "hippocratic2": "Hippocratic-2.1", "hippocratic2.1": "Hippocratic-2.1", "hippocratic2.1.0": "Hippocratic-2.1", + "hippocratic3.0.0core": "Hippocratic-3.0-core", + "hippocratic3.0core": "Hippocratic-3.0-core", + "hippocratic3core": "Hippocratic-3.0-core", "hp1986": "HP-1986", "hp1986.0": "HP-1986", "hp1986.0.0": "HP-1986", @@ -813,6 +834,7 @@ var licenseIDs = map[string]string{ "imlib2": "Imlib2", "imlib2.0": "Imlib2", "imlib2.0.0": "Imlib2", + "informatica": "Informatica", "infozip": "Info-ZIP", "innernet2": "Inner-Net-2.0", "innernet2.0": "Inner-Net-2.0", @@ -1011,6 +1033,9 @@ var licenseIDs = map[string]string{ "mulanpsl2.0.0": "MulanPSL-2.0", "multics": "Multics", "mup": "Mup", + "mvt1": "MVT-1.1", + "mvt1.1": "MVT-1.1", + "mvt1.1.0": "MVT-1.1", "naist2003": "NAIST-2003", "naist2003.0": "NAIST-2003", "naist2003.0.0": "NAIST-2003", @@ -1489,6 +1514,8 @@ var urlToLicense = map[string]string{ "cdla.io/permissive-1-0": "CDLA-Permissive-1.0", "cdla.io/sharing-1-0": "CDLA-Sharing-1.0", "celestrak.org/publications/AIAA/2006-6753/faq.php": "SGP4", + "cgit.freebsd.org/src/plain/sys/dev/aic7xxx/aic7xxx_osm.c": "BSD-Source-alt-GPL", + "cgit.freebsd.org/src/tree/usr.bin/sockstat/sockstat.c": "BSD-2-Clause-pos-unchanged", "cgit.freedesktop.org/libbsd/tree/man/setproctitle.3bsd": "libutil-David-Nugent", "coil.apotheon.org/plaintext/01.0.txt": "COIL-1.0", "core.tcl-lang.org/tk/file?name=compat/unistd.h": "HPND-UC", @@ -1512,6 +1539,7 @@ var urlToLicense = map[string]string{ "creativecommons.org/licenses/by-nc/2.0/legalcode": "CC-BY-NC-2.0", "creativecommons.org/licenses/by-nc/2.5/legalcode": "CC-BY-NC-2.5", "creativecommons.org/licenses/by-nc/3.0/de/legalcode": "CC-BY-NC-3.0-DE", + "creativecommons.org/licenses/by-nc/3.0/igo/legalcode": "CC-BY-NC-3.0-IGO", "creativecommons.org/licenses/by-nc/3.0/legalcode": "CC-BY-NC-3.0", "creativecommons.org/licenses/by-nc/4.0/legalcode": "CC-BY-NC-4.0", "creativecommons.org/licenses/by-nd-nc/1.0/legalcode": "CC-BY-NC-ND-1.0", @@ -1564,6 +1592,7 @@ var urlToLicense = map[string]string{ "directory.fsf.org/wiki/License:Fdk": "FDK-AAC", "doc.qt.io/archives/3.3/license.html": "QPL-1.0", "doc.qt.nokia.com/3.3/license.html": "QPL-1.0", + "docs.mvt.re/en/latest/license/": "MVT-1.1", "docs.omniverse.nvidia.com/install-guide/latest/common/licenses.html": "AML-glslang", "docs.oracle.com/cd/E81115_01/html/E81116/licenses.html": "man2html", "docs.python.org/3/license.html": "Python-2.0.1", @@ -1669,6 +1698,7 @@ var urlToLicense = map[string]string{ "fedoraproject.org/wiki/Licensing:Mplus?rd=Licensing/mplus": "mplus", "fedoraproject.org/wiki/Licensing:ThorPublicLicense": "TPL-1.0", "firstdonoharm.dev/version/2/1/license.html": "Hippocratic-2.1", + "firstdonoharm.dev/version/3/0/core.txt": "Hippocratic-3.0-core", "forge.gouv.qc.ca/licence/fr/liliq-v1-1/": "LiLiQ-P-1.1", "forge.gouv.qc.ca/licence/liliq-p": "LiLiQ-R-1.1", "forge.gouv.qc.ca/licence/liliq-p/": "LiLiQ-P-1.1", @@ -1686,68 +1716,70 @@ var urlToLicense = map[string]string{ "ftp://ftp.tin.org/pub/news/utils/newsx/newsx-1.6.tar.gz": "Zeeff", "gianluca.dellavedova.org/2011/01/03/clarified-artistic-license/": "ClArtistic", "git.denx.de/?p=u-boot.git;a=blob;f=arch/powerpc/cpu/ppc4xx/miiphy.c;h=297155fdafa064b955e53e9832de93bfb0cfb85b;hb=9fab4bf4cc077c21e43941866f3f2c196f28670d": "IBM-pibs", - "git.enlightenment.org/legacy/imlib2.git/tree/COPYING": "Imlib2", - "git.kernel.org/pub/scm/docs/man-pages/man-pages.git/tree/man2/getcpu.2#n4": "Linux-man-pages-1-para", - "git.kernel.org/pub/scm/docs/man-pages/man-pages.git/tree/man2/migrate_pages.2#n8": "Linux-man-pages-copyleft-2-para", - "git.kernel.org/pub/scm/docs/man-pages/man-pages.git/tree/man2/move_pages.2#n5": "Linux-man-pages-copyleft-2-para", - "git.kernel.org/pub/scm/docs/man-pages/man-pages.git/tree/man2/set_mempolicy.2#n5": "Linux-man-pages-copyleft-var", - "git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/tree/drivers/infiniband/core/sa.h": "Linux-OpenIB", - "git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/tree/net/sunrpc/auth_gss/gss_generic_token.c?h=v4.19": "HPND-sell-variant", - "git.openldap.org/openldap/openldap/-/blob/master/COPYRIGHT#L55-63": "BSD-4.3RENO", - "git.openldap.org/openldap/openldap/-/blob/master/COPYRIGHT?ref_type=heads#L39-51": "Furuseth", - "git.openldap.org/openldap/openldap/-/blob/master/libraries/libldap/os-local.c?ref_type=heads#L19-23": "PADL", - "git.savannah.gnu.org/cgit/freetype/freetype2.git/tree/docs/FTL.TXT": "FTL", - "git.savannah.gnu.org/cgit/gcl.git/tree/gcl/lsp/gcl_loop.lsp?h=Version_2_6_13pre": "LOOP", - "git.savannah.gnu.org/cgit/gnulib.git/tree/modules/COPYING?id=7b08932179d0d6b017f7df01a2ddf6e096b038e3": "FSFULLRSD", - "git.savannah.gnu.org/cgit/indent.git/tree/doc/indent.texi?id=a74c6b4ee49397cf330b333da1042bffa60ed14f#n1788": "BSD-4.3TAHOE", - "git.savannah.gnu.org/cgit/indent.git/tree/doc/indent.texi?id=a74c6b4ee49397cf330b333da1042bffa60ed14f#n74": "Latex2e-translated-notice", - "git.savannah.gnu.org/cgit/kazlib.git/tree/except.c?id=0062df360c2d17d57f6af19b0e444c51feb99036": "Kazlib", - "git.savannah.gnu.org/cgit/wget.git/tree/util/trunc.c?h=v1.21.3&id=40747a11e44ced5a8ac628a41f879ced3e2ebce9#n6": "FSFAP-no-warranty-disclaimer", - "git.sr.ht/~nabijaczleweli/ossp-var": "OSSP", - "github.com/389ds/389-ds-base/blob/main/ldap/include/sysexits-compat.h#L15": "BSD-4.3TAHOE", - "github.com/AcademySoftwareFoundation/foundation/blob/main/digital_assets/aswf_digital_assets_license_v1.0.txt": "ASWF-Digital-Assets-1.0", - "github.com/AcademySoftwareFoundation/foundation/blob/main/digital_assets/aswf_digital_assets_license_v1.1.txt": "ASWF-Digital-Assets-1.1", - "github.com/AdaCore/gnatcoll-core/blob/master/docs/index.rst": "AdaCore-doc", - "github.com/AdaCore/gnatcoll-db/blob/master/docs/index.rst": "AdaCore-doc", - "github.com/AdaCore/xmlada/blob/master/docs/index.rst": "AdaCore-doc", - "github.com/BuildCraft/BuildCraft/blob/623d323b1868712f29f4a8b0979a02e8d1835131/LICENSE": "MMPL-1.0.1", - "github.com/CommunitySpecification/1.0/blob/master/1._Community_Specification_License-v1.md": "Community-Spec-1.0", - "github.com/CorsixTH/deps/blob/fd339a9f526d1d9c9f01ccf39e438a015da50035/licences/libgsm.txt": "TU-Berlin-2.0", - "github.com/Cyan4973/xxHash/blob/dev/doc/xxhash_spec.md": "LPD-document", - "github.com/DISIC/politique-de-contribution-open-source/blob/master/LICENSE.pdf": "etalab-2.0", - "github.com/Distrotech/hdparm/blob/4517550db29a91420fb2b020349523b1b4512df2/LICENSE.TXT": "hdparm", - "github.com/Distrotech/transfig/blob/master/transfig/transfig.c": "Xfig", - "github.com/DrItanium/maya/blob/master/LICENSE.CLIPS": "Clips", - "github.com/Dual-Life/mime-base64/blob/master/Base64.xs#L12": "metamail", - "github.com/EthicalSource/hippocratic-license/blob/58c0e646d64ff6fbee275bfe2b9492f914e3ab2a/LICENSE.txt": "Hippocratic-2.1", - "github.com/FreeRADIUS/freeradius-client/blob/master/COPYRIGHT#L32": "HPND-Fenneberg-Livingston", - "github.com/GNOME/gcr/blob/master/docs/COPYING": "GCR-docs", - "github.com/GNOME/libxml2/blob/master/dict.c": "ISC-Veillard", - "github.com/HDFGroup/hdf5/?tab=License-1-ov-file#readme": "HDF5", - "github.com/KhronosGroup/SPIRV-Cross/blob/main/LICENSES/LicenseRef-KhronosFreeUse.txt": "MIT-Khronos-old", - "github.com/KhronosGroup/glslang/blob/main/LICENSE.txt#L949": "AML-glslang", - "github.com/MariaDB/server/blob/11.6/libmysqld/lib_sql.cc": "Boehm-GC-without-fee", - "github.com/Neo23x0/sigma/blob/master/LICENSE.Detection.Rules.md": "DRL-1.0", - "github.com/OGRECave/ogre/blob/master/OgreMain/include/OgreSingleton.h#L28C3-L35C46": "Game-Programming-Gems", - "github.com/Perl/perl5/blob/blead/util.c#L6136": "Martin-Birgmeier", - "github.com/PixarAnimationStudios/OpenSubdiv/blob/v3_5_0/opensubdiv/version.cpp#L2-L22": "Pixar", - "github.com/PixarAnimationStudios/OpenSubdiv/raw/v3_5_0/LICENSE.txt": "Pixar", - "github.com/RTimothyEdwards/magic/blob/master/LICENSE": "HPND-UC-export-US", - "github.com/SELinuxProject/selinux/blob/master/libselinux/LICENSE": "libselinux-1.0", - "github.com/SWI-Prolog/packages-clib/blob/master/sha1/brg_endian.h": "Brian-Gladman-3-Clause", - "github.com/SWI-Prolog/packages-xpce/blob/161a40cd82004f731ba48024f9d30af388a7edf5/src/img/gifwrite.c#L21-L26": "FBM", - "github.com/SWI-Prolog/swipl-devel/blob/master/src/os/dtoa.c": "dtoa", - "github.com/SigmaHQ/Detection-Rule-License/blob/6ec7fbde6101d101b5b5d1fcb8f9b69fbc76c04a/LICENSE.Detection.Rules.md": "DRL-1.1", - "github.com/Slicer/Slicer/blob/main/License.txt": "3D-Slicer-1.0", - "github.com/Unidata/UDUNITS-2/blob/master/COPYRIGHT": "UCAR", - "github.com/acpica/acpica/blob/master/source/common/acfileio.c#L119": "BSD-3-Clause-acpica", - "github.com/adaptivecomputing/torque/blob/master/PBS_License.txt": "OpenPBS-2.3", - "github.com/adversary-org/wtfnmf/raw/3f2cd8235a64350a57a51b9739715edaea63ec1a/COPYING.WTFNMFPL-utf8": "WTFNMFPL", - "github.com/adversary-org/wtfnmf/raw/refs/tags/1.0/COPYING.WTFNMFPL": "WTFNMFPL", - "github.com/ap/Text-Tabs/blob/master/lib.modern/Text/Tabs.pm#L148": "TTWL", - "github.com/apache/apr/blob/trunk/LICENSE#L298C6-L298C29": "Caldera-no-preamble", - "github.com/audacious-media-player/audacious/blob/master/COPYING": "BSD-2-Clause-pkgconf-disclaimer", - "github.com/audacious-media-player/audacious/blob/master/src/audacious/main.cc": "BSD-2-Clause-pkgconf-disclaimer", + "git.enlightenment.org/legacy/imlib2.git/tree/COPYING": "Imlib2", + "git.kernel.org/pub/scm/docs/man-pages/man-pages.git/tree/man2/getcpu.2#n4": "Linux-man-pages-1-para", + "git.kernel.org/pub/scm/docs/man-pages/man-pages.git/tree/man2/migrate_pages.2#n8": "Linux-man-pages-copyleft-2-para", + "git.kernel.org/pub/scm/docs/man-pages/man-pages.git/tree/man2/move_pages.2#n5": "Linux-man-pages-copyleft-2-para", + "git.kernel.org/pub/scm/docs/man-pages/man-pages.git/tree/man2/set_mempolicy.2#n5": "Linux-man-pages-copyleft-var", + "git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/tree/drivers/infiniband/core/sa.h": "Linux-OpenIB", + "git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/tree/net/sunrpc/auth_gss/gss_generic_token.c?h=v4.19": "HPND-sell-variant", + "git.openldap.org/openldap/openldap/-/blob/master/COPYRIGHT#L55-63": "BSD-4.3RENO", + "git.openldap.org/openldap/openldap/-/blob/master/COPYRIGHT?ref_type=heads#L39-51": "Furuseth", + "git.openldap.org/openldap/openldap/-/blob/master/libraries/libldap/os-local.c?ref_type=heads#L19-23": "PADL", + "git.savannah.gnu.org/cgit/freetype/freetype2.git/tree/docs/FTL.TXT": "FTL", + "git.savannah.gnu.org/cgit/gcl.git/tree/gcl/lsp/gcl_loop.lsp?h=Version_2_6_13pre": "LOOP", + "git.savannah.gnu.org/cgit/gnulib.git/tree/modules/COPYING?id=7b08932179d0d6b017f7df01a2ddf6e096b038e3": "FSFULLRSD", + "git.savannah.gnu.org/cgit/indent.git/tree/doc/indent.texi?id=a74c6b4ee49397cf330b333da1042bffa60ed14f#n1788": "BSD-4.3TAHOE", + "git.savannah.gnu.org/cgit/indent.git/tree/doc/indent.texi?id=a74c6b4ee49397cf330b333da1042bffa60ed14f#n74": "Latex2e-translated-notice", + "git.savannah.gnu.org/cgit/kazlib.git/tree/except.c?id=0062df360c2d17d57f6af19b0e444c51feb99036": "Kazlib", + "git.savannah.gnu.org/cgit/wget.git/tree/util/trunc.c?h=v1.21.3&id=40747a11e44ced5a8ac628a41f879ced3e2ebce9#n6": "FSFAP-no-warranty-disclaimer", + "git.sr.ht/~nabijaczleweli/ossp-var": "OSSP", + "github.com/389ds/389-ds-base/blob/main/ldap/include/sysexits-compat.h#L15": "BSD-4.3TAHOE", + "github.com/AcademySoftwareFoundation/foundation/blob/main/digital_assets/aswf_digital_assets_license_v1.0.txt": "ASWF-Digital-Assets-1.0", + "github.com/AcademySoftwareFoundation/foundation/blob/main/digital_assets/aswf_digital_assets_license_v1.1.txt": "ASWF-Digital-Assets-1.1", + "github.com/AdaCore/gnatcoll-core/blob/master/docs/index.rst": "AdaCore-doc", + "github.com/AdaCore/gnatcoll-db/blob/master/docs/index.rst": "AdaCore-doc", + "github.com/AdaCore/xmlada/blob/master/docs/index.rst": "AdaCore-doc", + "github.com/BuildCraft/BuildCraft/blob/623d323b1868712f29f4a8b0979a02e8d1835131/LICENSE": "MMPL-1.0.1", + "github.com/CommunitySpecification/1.0/blob/master/1._Community_Specification_License-v1.md": "Community-Spec-1.0", + "github.com/CorsixTH/deps/blob/fd339a9f526d1d9c9f01ccf39e438a015da50035/licences/libgsm.txt": "TU-Berlin-2.0", + "github.com/Cyan4973/xxHash/blob/dev/doc/xxhash_spec.md": "LPD-document", + "github.com/DISIC/politique-de-contribution-open-source/blob/master/LICENSE.pdf": "etalab-2.0", + "github.com/Distrotech/hdparm/blob/4517550db29a91420fb2b020349523b1b4512df2/LICENSE.TXT": "hdparm", + "github.com/Distrotech/transfig/blob/master/transfig/transfig.c": "Xfig", + "github.com/DrItanium/maya/blob/master/LICENSE.CLIPS": "Clips", + "github.com/Dual-Life/mime-base64/blob/master/Base64.xs#L12": "metamail", + "github.com/EthicalSource/hippocratic-license/blob/58c0e646d64ff6fbee275bfe2b9492f914e3ab2a/LICENSE.txt": "Hippocratic-2.1", + "github.com/Fraunhofer-IIS/mpeghdec/blob/main/LICENSE.txt": "FDK-MPEG-H", + "github.com/FreeRADIUS/freeradius-client/blob/master/COPYRIGHT#L32": "HPND-Fenneberg-Livingston", + "github.com/GNOME/gcr/blob/master/docs/COPYING": "GCR-docs", + "github.com/GNOME/libxml2/blob/master/dict.c": "ISC-Veillard", + "github.com/HDFGroup/hdf5/?tab=License-1-ov-file#readme": "HDF5", + "github.com/KhronosGroup/SPIRV-Cross/blob/main/LICENSES/LicenseRef-KhronosFreeUse.txt": "MIT-Khronos-old", + "github.com/KhronosGroup/glslang/blob/main/LICENSE.txt#L949": "AML-glslang", + "github.com/MariaDB/server/blob/11.6/libmysqld/lib_sql.cc": "Boehm-GC-without-fee", + "github.com/Neo23x0/sigma/blob/master/LICENSE.Detection.Rules.md": "DRL-1.0", + "github.com/OGRECave/ogre/blob/master/OgreMain/include/OgreSingleton.h#L28C3-L35C46": "Game-Programming-Gems", + "github.com/Perl/perl5/blob/blead/util.c#L6136": "Martin-Birgmeier", + "github.com/PicoEFI/PicoEFI/blob/2263ac1a2bc98e5534d64a4896d3cfbbae00745a/LICENSES/LicenseRef-scancode-bsd-no-disclaimer-unmodified.txt": "BSD-Source-Code-no-disclaimer", + "github.com/PixarAnimationStudios/OpenSubdiv/blob/v3_5_0/opensubdiv/version.cpp#L2-L22": "Pixar", + "github.com/PixarAnimationStudios/OpenSubdiv/raw/v3_5_0/LICENSE.txt": "Pixar", + "github.com/RTimothyEdwards/magic/blob/master/LICENSE": "HPND-UC-export-US", + "github.com/SELinuxProject/selinux/blob/master/libselinux/LICENSE": "libselinux-1.0", + "github.com/SWI-Prolog/packages-clib/blob/master/sha1/brg_endian.h": "Brian-Gladman-3-Clause", + "github.com/SWI-Prolog/packages-xpce/blob/161a40cd82004f731ba48024f9d30af388a7edf5/src/img/gifwrite.c#L21-L26": "FBM", + "github.com/SWI-Prolog/swipl-devel/blob/master/src/os/dtoa.c": "dtoa", + "github.com/SigmaHQ/Detection-Rule-License/blob/6ec7fbde6101d101b5b5d1fcb8f9b69fbc76c04a/LICENSE.Detection.Rules.md": "DRL-1.1", + "github.com/Slicer/Slicer/blob/main/License.txt": "3D-Slicer-1.0", + "github.com/Unidata/UDUNITS-2/blob/master/COPYRIGHT": "UCAR", + "github.com/acpica/acpica/blob/master/source/common/acfileio.c#L119": "BSD-3-Clause-acpica", + "github.com/adaptivecomputing/torque/blob/master/PBS_License.txt": "OpenPBS-2.3", + "github.com/adversary-org/wtfnmf/raw/3f2cd8235a64350a57a51b9739715edaea63ec1a/COPYING.WTFNMFPL-utf8": "WTFNMFPL", + "github.com/adversary-org/wtfnmf/raw/refs/tags/1.0/COPYING.WTFNMFPL": "WTFNMFPL", + "github.com/ap/Text-Tabs/blob/master/lib.modern/Text/Tabs.pm#L148": "TTWL", + "github.com/apache/apr/blob/trunk/LICENSE#L298C6-L298C29": "Caldera-no-preamble", + "github.com/audacious-media-player/audacious/blob/master/COPYING": "BSD-2-Clause-pkgconf-disclaimer", + "github.com/audacious-media-player/audacious/blob/master/src/audacious/main.cc": "BSD-2-Clause-pkgconf-disclaimer", "github.com/aws/mit-0": "MIT-0", "github.com/awsdocs/aws-cloud9-user-guide/blob/master/LICENSE-SAMPLECODE": "MIT-0", "github.com/bagder/curl/blob/master/COPYING": "curl", @@ -1772,6 +1804,7 @@ var urlToLicense = map[string]string{ "github.com/file/file/blob/master/COPYING": "BSD-2-Clause-Darwin", "github.com/freebsd/freebsd-src/blob/main/lib/msun/src/e_acosh.c": "SunPro", "github.com/freebsd/freebsd-src/blob/main/lib/msun/src/e_lgammal.c": "SunPro", + "github.com/freebsd/pkg/blob/main/src/add.c": "BSD-2-Clause-pos-unchanged", "github.com/gabegundy/atlserver/blob/master/License.txt": "MS-LPL", "github.com/greymass/swift-eosio/blob/master/LICENSE": "BSD-3-Clause-No-Military-License", "github.com/hamano/man2html/blob/master/man2html.c": "man2html", @@ -1784,6 +1817,7 @@ var urlToLicense = map[string]string{ "github.com/jsommers/pytricia/blob/master/patricia.c#L33-L67": "BSD-4-Clause", "github.com/kfish/xsel/blob/master/COPYING": "HPND-sell-variant", "github.com/kohler/t1utils/blob/master/LICENSE": "MIT-Click", + "github.com/krb5/krb5/blob/kfw-4.0.1-beta1/src/lib/crypto/builtin/aes/aesopt.h": "Brian-Gladman-3-Clause-no-conversion", "github.com/krb5/krb5/blob/krb5-1.21.2-final/NOTICE#L111-L133": "HPND-export2-US", "github.com/krb5/krb5/blob/krb5-1.21.2-final/NOTICE#L1157-L1182": "HPND-export-US-modify", "github.com/krb5/krb5/blob/krb5-1.21.2-final/NOTICE#L140-L156": "Brian-Gladman-2-Clause", @@ -1793,6 +1827,7 @@ var urlToLicense = map[string]string{ "github.com/krb5/krb5/blob/krb5-1.21.2-final/NOTICE#L831-L852": "HPND-export-US-acknowledgement", "github.com/krzyzanowskim/CryptoSwift/blob/main/LICENSE": "CryptoSwift", "github.com/lattera/freebsd/blob/master/sys/cam/cam.c#L4": "BSD-Source-beginning-file", + "github.com/libarchive/libarchive/blob/master/libarchive_fe/line_reader.c": "BSD-2-Clause-pos-unchanged", "github.com/libcheck/check/blob/master/checkmk/checkmk.in": "checkmk", "github.com/libjpeg-turbo/libjpeg-turbo/blob/main/README.ijg#L117-L161": "IJG", "github.com/lsof-org/lsof/blob/master/COPYING": "lsof", @@ -1805,6 +1840,7 @@ var urlToLicense = map[string]string{ "github.com/mm2/Little-CMS/blob/master/src/cmssm.c#L207": "softSurfer", "github.com/mruby/mruby/blob/83d12f8d52522cdb7c8cc46fad34821359f453e6/mrbgems/mruby-dir/src/Win/dirent.c#L127-L140": "HPND-Kevlin-Henney", "github.com/mtoyoda/sl/blob/master/LICENSE": "SL", + "github.com/mvt-project/mvt/blob/main/LICENSE": "MVT-1.1", "github.com/n8n-io/n8n/blob/master/LICENSE.md": "SUL-1.0", "github.com/ncbi/datasets/blob/0ea4cd16b61e5b799d9cc55aecfa016d6c9bd2bf/LICENSE.md": "NCBI-PD", "github.com/ncbi/datasets/blob/master/LICENSE.md": "NCBI-PD", @@ -1815,6 +1851,7 @@ var urlToLicense = map[string]string{ "github.com/novnc/noVNC/blob/master/core/crypto/des.js#L24": "Widget-Workshop", "github.com/oetiker/rrdtool-1.x/blob/master/src/plbasename.c#L8-L11": "gtkbook", "github.com/open-quantum-safe/liboqs/blob/40b01fdbb270f8614fde30e65d30e9da18c02393/src/common/rand/rand_nist.c#L1-L15": "NIST-Software", + "github.com/open-webui/open-webui/blob/main/LICENSE": "BSD-3-Clause-OpenWebUI", "github.com/openjdk/jdk/blob/6162e2c5213c5dd7c1127fd9616b543efa898962/LICENSE": "GPL-2.0-or-later", "github.com/openssh/openssh-portable/blob/1b11ea7c58cd5c59838b5fa574cd456d6047b2d4/LICENCE#L10": "SSH-OpenSSH", "github.com/openssh/openssh-portable/blob/1b11ea7c58cd5c59838b5fa574cd456d6047b2d4/pathnames.h": "SSH-short", @@ -1846,6 +1883,7 @@ var urlToLicense = map[string]string{ "github.com/sigmavirus24/x11-ssh-askpass/blob/master/dynlist.c": "HPND-sell-variant-MIT-disclaimer-rev", "github.com/signal11/hidapi/blob/master/LICENSE-orig.txt": "HIDAPI", "github.com/slogan621/gtkbook": "gtkbook", + "github.com/sudo-project/sudo/blob/c1307ea9ff340ce0538779f8e456501461fc44b7/plugins/sudoers/redblack.c#L24-L43": "BSD-ask-to-endorse", "github.com/swh/ladspa/blob/7bf6f3799fdba70fda297c2d8fd9f526803d9680/gsm/COPYRIGHT": "TU-Berlin-1.0", "github.com/tats/w3m/blob/master/COPYING": "w3m", "github.com/tcheneau/Routing/blob/f09f46fcfe636107f22f2c98348188a65a135d98/README.md": "NIST-PD", @@ -1856,6 +1894,7 @@ var urlToLicense = map[string]string{ "github.com/uim/libgcroots/blob/master/COPYING": "Boehm-GC", "github.com/usnistgov/fipy/blob/86aaa5c2ba2c6f1be19593c5986071cf6568cc34/LICENSE.rst": "NIST-PD-fallback", "github.com/usnistgov/jsip/blob/59700e6926cbe96c5cdae897d9a7d2656b42abe3/LICENSE": "NIST-PD-fallback", + "github.com/vattam/BSDGames/blob/master/atc/extern.c#L36": "atc-game", "github.com/vixie/cron/tree/545b3f5246824a9cda5905eeb7cf019c95e66995": "Vixie-Cron", "github.com/vstroebel/jpeg-encoder/blob/main/src/fdct.rs#L1-L72": "IJG", "github.com/wernerd/ZRTPCPP/blob/6b3cd8e6783642292bad0c21e3e5e5ce45ff3e03/cryptcommon/twofish.c#L113C3-L127": "Ferguson-Twofish", @@ -1869,6 +1908,7 @@ var urlToLicense = map[string]string{ "gitlab.com/embeddable-common-lisp/ecl/-/blob/develop/src/lsp/loop.lsp": "LOOP", "gitlab.com/libeigen/eigen/-/blob/master/COPYING.MINPACK": "Minpack", "gitlab.common-lisp.net/cmucl/cmucl/-/blob/master/src/code/loop.lisp": "LOOP", + "gitlab.freedesktop.org/dbus/dbus/-/blob/789d97ad53044f94351a958e785c2923643b346a/COPYING#L11-207": "AFL-2.1", "gitlab.freedesktop.org/freetype/freetype/-/raw/master/docs/FTL.TXT": "FTL", "gitlab.freedesktop.org/libopenraw/libopenraw/blob/master/lib/ljpegdecompressor.cpp#L32": "Cornell-Lossless-JPEG", "gitlab.freedesktop.org/mesa/glw/-/blob/master/README?ref_type=heads": "SGI-OpenGL", @@ -1892,6 +1932,7 @@ var urlToLicense = map[string]string{ "gitlab.freedesktop.org/xorg/lib/libxinerama/-/blob/cc22c2f60c3862482562955116d5455263b443dc/COPYING#L44-66": "X11-no-permit-persons", "gitlab.freedesktop.org/xorg/lib/libxtst/-/blob/master/COPYING?ref_type=heads#L108-117": "HPND-doc-sell", "gitlab.freedesktop.org/xorg/lib/libxtst/-/blob/master/COPYING?ref_type=heads#L70-77": "HPND-doc", + "gitlab.freedesktop.org/xorg/xserver/-/blob/dd5c2595a42d3ff0c4f18d9b53d1f6c3fd934fd4/COPYING#L365-389": "MIT", "gitlab.freedesktop.org/xorg/xserver/-/blob/master/COPYING?ref_type=heads#L1781": "HPND-sell-MIT-disclaimer-xserver", "gitlab.freedesktop.org/xorg/xserver/-/blob/master/COPYING?ref_type=heads#L239": "DEC-3-Clause", "gitlab.freedesktop.org/xorg/xserver/-/blob/master/COPYING?ref_type=heads#L752": "Adobe-Display-PostScript", @@ -1906,6 +1947,7 @@ var urlToLicense = map[string]string{ "heasarc.gsfc.nasa.gov/docs/software/ftools/fv/doc/license.html": "CFITSIO", "helixcommunity.org/content/rpsl": "RPSL-1.0", "hstechdocs.helpsystems.com/manuals/globalscape/archive/cuteftp6/open_ssl_license_agreement.htm": "OpenSSL-standalone", + "interoperable-europe.ec.europa.eu/sites/default/files/inline-files/EUPL%20v1_2%20EN(1).txt": "EUPL-1.2", "java.net/projects/javaeetutorial/pages/BerkeleyLicense": "BSD-3-Clause-No-Nuclear-License-2014", "javaee.github.io/glassfish/LICENSE": "CDDL-1.1", "jogamp.org/git/?p=gluegen.git;a=blob_plain;f=LICENSE.txt": "BSD-3-Clause-No-Nuclear-Warranty", @@ -1924,6 +1966,7 @@ var urlToLicense = map[string]string{ "library.netapp.com/ecm/ecm_download_file/ECMP1196395": "OpenSSL-standalone", "license.coscl.org.cn/MulanPSL/": "MulanPSL-1.0", "license.coscl.org.cn/MulanPSL2": "MulanPSL-2.0", + "license.mvt.re/1.1/": "MVT-1.1", "lists.gnu.org/archive/html/autoconf/2012-04/msg00061.html": "FSFULLRWD", "lists.opensource.org/pipermail/license-discuss_lists.opensource.org/2002-November/006304.html": "HPND", "mariadb.com/bsl11/": "BUSL-1.1", @@ -1965,99 +2008,105 @@ var urlToLicense = map[string]string{ "opensource.linux-mirror.org/licenses/afl-1.1.txt": "AFL-1.1", "opensource.linux-mirror.org/licenses/afl-1.2.txt": "AFL-1.2", "opensource.linux-mirror.org/licenses/afl-2.1.txt": "AFL-2.1", - "opensource.org/license/apache-2-0": "Apache-2.0", - "opensource.org/license/mit/": "MIT", - "opensource.org/license/olfl-1-3/": "OLFL-1.3", - "opensource.org/license/osc-license-1-0": "OSC-1.0", - "opensource.org/licenses/0BSD": "0BSD", - "opensource.org/licenses/AGPL-3.0": "AGPL-3.0-or-later", - "opensource.org/licenses/APL-1.0": "APL-1.0", - "opensource.org/licenses/Apache-1.1": "Apache-1.1", - "opensource.org/licenses/Apache-2.0": "Apache-2.0", - "opensource.org/licenses/Artistic-1.0": "Artistic-1.0-cl8", - "opensource.org/licenses/BSD-2-Clause": "BSD-2-Clause", - "opensource.org/licenses/BSD-3-Clause": "BSD-3-Clause", - "opensource.org/licenses/BSDplusPatent": "BSD-2-Clause-Patent", - "opensource.org/licenses/BSL-1.0": "BSL-1.0", - "opensource.org/licenses/CAL-1.0": "CAL-1.0-Combined-Work-Exception", - "opensource.org/licenses/CATOSL-1.1": "CATOSL-1.1", - "opensource.org/licenses/CNRI-Python": "CNRI-Python", - "opensource.org/licenses/CPAL-1.0": "CPAL-1.0", - "opensource.org/licenses/CPL-1.0": "CPL-1.0", - "opensource.org/licenses/CUA-OPL-1.0": "CUA-OPL-1.0", - "opensource.org/licenses/ECL-1.0": "ECL-1.0", - "opensource.org/licenses/ECL-2.0": "ECL-2.0", - "opensource.org/licenses/EFL-1.0": "EFL-1.0", - "opensource.org/licenses/EFL-2.0": "EFL-2.0", - "opensource.org/licenses/EPL-1.0": "EPL-1.0", - "opensource.org/licenses/EUDatagrid": "EUDatagrid", - "opensource.org/licenses/EUPL-1.1": "EUPL-1.1", - "opensource.org/licenses/EUPL-1.2": "EUPL-1.2", - "opensource.org/licenses/Entessa": "Entessa", - "opensource.org/licenses/Fair": "Fair", - "opensource.org/licenses/Frameworx-1.0": "Frameworx-1.0", - "opensource.org/licenses/GPL-2.0": "GPL-2.0-or-later", - "opensource.org/licenses/GPL-3.0": "GPL-3.0-or-later", - "opensource.org/licenses/HPND": "HPND", - "opensource.org/licenses/IPA": "IPA", - "opensource.org/licenses/IPL-1.0": "IPL-1.0", - "opensource.org/licenses/ISC": "ISC", - "opensource.org/licenses/Intel": "Intel", - "opensource.org/licenses/LGPL-2.1": "LGPL-2.1-or-later", - "opensource.org/licenses/LGPL-3.0": "LGPL-3.0-or-later", - "opensource.org/licenses/LPL-1.0": "LPL-1.0", - "opensource.org/licenses/LPL-1.02": "LPL-1.02", - "opensource.org/licenses/LPPL-1.3c": "LPPL-1.3c", - "opensource.org/licenses/LiLiQ-P-1.1": "LiLiQ-P-1.1", - "opensource.org/licenses/LiLiQ-R-1.1": "LiLiQ-R-1.1", - "opensource.org/licenses/LiLiQ-Rplus-1.1": "LiLiQ-Rplus-1.1", - "opensource.org/licenses/MIT": "MIT", - "opensource.org/licenses/MPL-1.0": "MPL-1.0", - "opensource.org/licenses/MPL-1.1": "MPL-1.1", - "opensource.org/licenses/MPL-2.0": "MPL-2.0-no-copyleft-exception", - "opensource.org/licenses/MS-PL": "MS-PL", - "opensource.org/licenses/MS-RL": "MS-RL", - "opensource.org/licenses/MirOS": "MirOS", - "opensource.org/licenses/Motosoto": "Motosoto", - "opensource.org/licenses/Multics": "Multics", - "opensource.org/licenses/NASA-1.3": "NASA-1.3", - "opensource.org/licenses/NCSA": "NCSA", - "opensource.org/licenses/NGPL": "NGPL", - "opensource.org/licenses/NOSL3.0": "NPOSL-3.0", - "opensource.org/licenses/NTP": "NTP", - "opensource.org/licenses/Naumen": "Naumen", - "opensource.org/licenses/OCLC-2.0": "OCLC-2.0", - "opensource.org/licenses/OFL-1.1": "OFL-1.1-no-RFN", - "opensource.org/licenses/OGTSL": "OGTSL", - "opensource.org/licenses/OPL-2.1": "OSET-PL-2.1", - "opensource.org/licenses/OSL-1.0": "OSL-1.0", - "opensource.org/licenses/OSL-2.1": "OSL-2.1", - "opensource.org/licenses/OSL-3.0": "OSL-3.0", - "opensource.org/licenses/PHP-3.0": "PHP-3.0", - "opensource.org/licenses/PostgreSQL": "PostgreSQL", - "opensource.org/licenses/Python-2.0": "Python-2.0", - "opensource.org/licenses/QPL-1.0": "QPL-1.0", - "opensource.org/licenses/RPL-1.1": "RPL-1.1", - "opensource.org/licenses/RPL-1.5": "RPL-1.5", - "opensource.org/licenses/RPSL-1.0": "RPSL-1.0", - "opensource.org/licenses/RSCPL": "RSCPL", - "opensource.org/licenses/SISSL": "SISSL", - "opensource.org/licenses/SPL-1.0": "SPL-1.0", - "opensource.org/licenses/SimPL-2.0": "SimPL-2.0", - "opensource.org/licenses/Sleepycat": "Sleepycat", - "opensource.org/licenses/UCL-1.0": "UCL-1.0", - "opensource.org/licenses/UPL": "UPL-1.0", - "opensource.org/licenses/VSL-1.0": "VSL-1.0", - "opensource.org/licenses/W3C": "W3C", - "opensource.org/licenses/Watcom-1.0": "Watcom-1.0", - "opensource.org/licenses/Xnet": "Xnet", - "opensource.org/licenses/ZPL-2.0": "ZPL-2.0", - "opensource.org/licenses/Zlib": "Zlib", - "opensource.org/licenses/afl-3.0": "AFL-3.0", - "opensource.org/licenses/artistic-license-2.0": "Artistic-2.0", - "opensource.org/licenses/attribution": "AAL", - "opensource.org/licenses/cddl1": "CDDL-1.0", - "opensource.org/licenses/nokia": "Nokia", + "opensource.org/license/0BSD": "0BSD", + "opensource.org/license/AAL": "AAL", + "opensource.org/license/AFL-3.0": "AFL-3.0", + "opensource.org/license/AGPL-3.0": "AGPL-3.0-or-later", + "opensource.org/license/AGPL-3.0-only": "AGPL-3.0-only", + "opensource.org/license/APL-1.0": "APL-1.0", + "opensource.org/license/Artistic-1.0": "Artistic-1.0-cl8", + "opensource.org/license/Artistic-2.0": "Artistic-2.0", + "opensource.org/license/BSD-2-Clause": "BSD-2-Clause", + "opensource.org/license/BSD-2-Clause-Patent": "BSD-2-Clause-Patent", + "opensource.org/license/BSD-3-Clause": "BSD-3-Clause", + "opensource.org/license/BSL-1.0": "BSL-1.0", + "opensource.org/license/CAL-1.0": "CAL-1.0-Combined-Work-Exception", + "opensource.org/license/CATOSL-1.1": "CATOSL-1.1", + "opensource.org/license/CDDL-1.0": "CDDL-1.0", + "opensource.org/license/CNRI-Python": "CNRI-Python", + "opensource.org/license/CNRI-Python-GPL-Compatible": "CNRI-Python-GPL-Compatible", + "opensource.org/license/CPAL-1.0": "CPAL-1.0", + "opensource.org/license/CPL-1.0": "CPL-1.0", + "opensource.org/license/CUA-OPL-1.0": "CUA-OPL-1.0", + "opensource.org/license/ECL-1.0": "ECL-1.0", + "opensource.org/license/ECL-2.0": "ECL-2.0", + "opensource.org/license/EFL-1.0": "EFL-1.0", + "opensource.org/license/EFL-2.0": "EFL-2.0", + "opensource.org/license/EPL-1.0": "EPL-1.0", + "opensource.org/license/EUDatagrid": "EUDatagrid", + "opensource.org/license/EUPL-1.1": "EUPL-1.1", + "opensource.org/license/EUPL-1.2": "EUPL-1.2", + "opensource.org/license/Entessa": "Entessa", + "opensource.org/license/Fair": "Fair", + "opensource.org/license/Frameworx-1.0": "Frameworx-1.0", + "opensource.org/license/GPL-2.0": "GPL-2.0-or-later", + "opensource.org/license/GPL-2.0-only": "GPL-2.0-only", + "opensource.org/license/GPL-3.0": "GPL-3.0-or-later", + "opensource.org/license/GPL-3.0-only": "GPL-3.0-only", + "opensource.org/license/HPND": "HPND", + "opensource.org/license/IPA": "IPA", + "opensource.org/license/IPL-1.0": "IPL-1.0", + "opensource.org/license/ISC": "ISC", + "opensource.org/license/Intel": "Intel", + "opensource.org/license/LGPL-2.1": "LGPL-2.1-or-later", + "opensource.org/license/LGPL-2.1-only": "LGPL-2.1-only", + "opensource.org/license/LGPL-3.0": "LGPL-3.0-or-later", + "opensource.org/license/LPL-1.0": "LPL-1.0", + "opensource.org/license/LPL-1.02": "LPL-1.02", + "opensource.org/license/LPPL-1.3c": "LPPL-1.3c", + "opensource.org/license/LiLiQ-P-1.1": "LiLiQ-P-1.1", + "opensource.org/license/LiLiQ-R-1.1": "LiLiQ-R-1.1", + "opensource.org/license/LiLiQ-Rplus-1.1": "LiLiQ-Rplus-1.1", + "opensource.org/license/MIT": "MIT", + "opensource.org/license/MPL-1.0": "MPL-1.0", + "opensource.org/license/MPL-1.1": "MPL-1.1", + "opensource.org/license/MPL-2.0": "MPL-2.0-no-copyleft-exception", + "opensource.org/license/MS-PL": "MS-PL", + "opensource.org/license/MS-RL": "MS-RL", + "opensource.org/license/MirOS": "MirOS", + "opensource.org/license/Motosoto": "Motosoto", + "opensource.org/license/Multics": "Multics", + "opensource.org/license/NASA-1.3": "NASA-1.3", + "opensource.org/license/NCSA": "NCSA", + "opensource.org/license/NGPL": "NGPL", + "opensource.org/license/NPOSL-3.0": "NPOSL-3.0", + "opensource.org/license/NTP": "NTP", + "opensource.org/license/Naumen": "Naumen", + "opensource.org/license/Nokia": "Nokia", + "opensource.org/license/OCLC-2.0": "OCLC-2.0", + "opensource.org/license/OFL-1.1": "OFL-1.1-no-RFN", + "opensource.org/license/OGTSL": "OGTSL", + "opensource.org/license/OLFL-1.3": "OLFL-1.3", + "opensource.org/license/OSC-1.0": "OSC-1.0", + "opensource.org/license/OSET-PL-2.1": "OSET-PL-2.1", + "opensource.org/license/OSL-1.0": "OSL-1.0", + "opensource.org/license/OSL-2.1": "OSL-2.1", + "opensource.org/license/OSL-3.0": "OSL-3.0", + "opensource.org/license/PHP-3.0": "PHP-3.0", + "opensource.org/license/PSF-2-0": "PSF-2.0", + "opensource.org/license/PostgreSQL": "PostgreSQL", + "opensource.org/license/Python-2.0": "Python-2.0", + "opensource.org/license/Python-2.0.1": "Python-2.0.1", + "opensource.org/license/QPL-1.0": "QPL-1.0", + "opensource.org/license/RPL-1.1": "RPL-1.1", + "opensource.org/license/RPL-1.5": "RPL-1.5", + "opensource.org/license/RPSL-1.0": "RPSL-1.0", + "opensource.org/license/RSCPL": "RSCPL", + "opensource.org/license/SISSL": "SISSL", + "opensource.org/license/SPL-1.0": "SPL-1.0", + "opensource.org/license/SiMPL-2.0": "SimPL-2.0", + "opensource.org/license/Sleepycat": "Sleepycat", + "opensource.org/license/UCL-1.0": "UCL-1.0", + "opensource.org/license/UPL-1.0": "UPL-1.0", + "opensource.org/license/VSL-1.0": "VSL-1.0", + "opensource.org/license/W3C": "W3C", + "opensource.org/license/Watcom-1.0": "Watcom-1.0", + "opensource.org/license/Xnet": "Xnet", + "opensource.org/license/ZPL-2.0": "ZPL-2.0", + "opensource.org/license/Zlib": "Zlib", + "opensource.org/license/apache-1.1": "Apache-1.1", + "opensource.org/license/apache-2.0": "Apache-2.0", + "opensource.org/license/curl": "curl", "oss.sgi.com/projects/FreeB/": "SGI-B-1.1", "oss.sgi.com/projects/FreeB/SGIFreeSWLicB.1.0.html": "SGI-B-1.0", "oss.sgi.com/projects/FreeB/SGIFreeSWLicB.2.0.pdf": "SGI-B-2.0", @@ -2082,37 +2131,40 @@ var urlToLicense = map[string]string{ "raw.githubusercontent.com/NTIA/scos-sensor/refs/heads/master/LICENSE.md": "NTIA-PD", "raw.githubusercontent.com/OpenMDW/OpenMDW/refs/heads/main/1.0/LICENSE.openmdw": "OpenMDW-1.0", "raw.githubusercontent.com/blackhole89/notekit/master/data/fonts/Charter%20license.txt": "Bitstream-Charter", - "rc.quest.com/topics/openssh/license.php#tcpwrappers": "TCP-wrappers", - "research.cs.wisc.edu/condor/license.html#condor": "Condor-1.1", - "research.scea.com/scea_shared_source_license.html": "SCEA", - "romanrm.net/mit-zero": "MIT-0", - "sam.zoy.org/wtfpl/COPYING": "WTFPL", - "scripts.sil.org/cms/scripts/page.php?item_id=OFL10_web": "OFL-1.0-no-RFN", - "scripts.sil.org/cms/scripts/page.php?item_id=OFL_web": "OFL-1.1-no-RFN", - "slicer.org/LICENSE": "3D-Slicer-1.0", - "solderpad.org/licenses/SHL-0.5/": "SHL-0.5", - "solderpad.org/licenses/SHL-0.51/": "SHL-0.51", - "source.icu-project.org/repos/icu/icu/trunk/license.html": "ICU", - "sourceforge.net/p/buddy/gitcode/ci/master/tree/README": "Buddy", - "sourceforge.net/p/ctrio/git/ci/master/tree/README": "ISC-Veillard", - "sourceforge.net/p/mcj/xfig/ci/master/tree/src/Makefile.am": "Xfig", - "sourceforge.net/p/netpbm/code/HEAD/tree/super_stable/converter/other/pnmtorle.c": "URT-RLE", - "sourceforge.net/p/netpbm/code/HEAD/tree/super_stable/converter/other/rletopnm.c": "URT-RLE", - "sourceforge.net/p/netpbm/code/HEAD/tree/super_stable/converter/ppm/ppmtompeg/jrevdct.c#l1189": "MPEG-SSG", - "sourceforge.net/p/netpbm/code/HEAD/tree/super_stable/editor/pnmstitch.c#l2": "pnmstitch", - "sourceforge.net/p/netpbm/code/HEAD/tree/super_stable/netpbm.c#l8": "HPND-Pbmplus", - "sourceforge.net/p/sbcl/sbcl/ci/master/tree/src/code/loop.lisp": "LOOP", - "sourceforge.net/p/trustedqsl/tqsl/ci/master/tree/LICENSE.txt": "TrustedQSL", - "sourceforge.net/p/xmedcon/code/ci/master/tree/libs/dicom/README": "OFFIS", - "sourceforge.net/p/xmedcon/code/ci/master/tree/libs/ljpg/": "IJG-short", - "sourceforge.net/projects/sevenzip/files/LZMA%20SDK/": "LZMA-SDK-9.22", - "sources.debian.org/copyright/license/debianutils/4.11.2/": "SMAIL-GPL", - "sources.debian.org/src/openmpi/4.1.0-10/ompi/debuggers/msgq_interface.h/?hl=19#L19": "mpi-permissive", - "sources.gentoo.org/cgi-bin/viewvc.cgi/gentoo-x86/licenses/BitTorrent?r1=1.1&r2=1.1.1.1&diff_format=s": "BitTorrent-1.0", - "sourceware.org/bzip2/1.0.5/bzip2-manual-1.0.5.html": "bzip2-1.0.6", - "sourceware.org/cgit/binutils-gdb/tree/include/coff/sym.h#n11": "MIPS", - "sourceware.org/cgit/binutils-gdb/tree/include/coff/symconst.h#n11": "ThirdEye", - "sourceware.org/cgit/valgrind/tree/mpi/libmpiwrap.c": "bzip2-1.0.6", + "raw.githubusercontent.com/landtuna/open-mtools/6c04c2882eca7c4313ec267766ea53ac9bfe129b/mdump.c": "Informatica", + "raw.githubusercontent.com/landtuna/open-mtools/6c04c2882eca7c4313ec267766ea53ac9bfe129b/msend.c": "Informatica", + "rc.quest.com/topics/openssh/license.php#tcpwrappers": "TCP-wrappers", + "research.cs.wisc.edu/condor/license.html#condor": "Condor-1.1", + "research.scea.com/scea_shared_source_license.html": "SCEA", + "romanrm.net/mit-zero": "MIT-0", + "sam.zoy.org/wtfpl/COPYING": "WTFPL", + "scancode-licensedb.aboutcode.org/bsd-unchanged": "BSD-2-Clause-pos-unchanged", + "scripts.sil.org/cms/scripts/page.php?item_id=OFL10_web": "OFL-1.0-no-RFN", + "scripts.sil.org/cms/scripts/page.php?item_id=OFL_web": "OFL-1.1-no-RFN", + "slicer.org/LICENSE": "3D-Slicer-1.0", + "solderpad.org/licenses/SHL-0.5/": "SHL-0.5", + "solderpad.org/licenses/SHL-0.51/": "SHL-0.51", + "source.icu-project.org/repos/icu/icu/trunk/license.html": "ICU", + "sourceforge.net/p/buddy/gitcode/ci/master/tree/README": "Buddy", + "sourceforge.net/p/ctrio/git/ci/master/tree/README": "ISC-Veillard", + "sourceforge.net/p/mcj/xfig/ci/master/tree/src/Makefile.am": "Xfig", + "sourceforge.net/p/netpbm/code/HEAD/tree/super_stable/converter/other/pnmtorle.c": "URT-RLE", + "sourceforge.net/p/netpbm/code/HEAD/tree/super_stable/converter/other/rletopnm.c": "URT-RLE", + "sourceforge.net/p/netpbm/code/HEAD/tree/super_stable/converter/ppm/ppmtompeg/jrevdct.c#l1189": "MPEG-SSG", + "sourceforge.net/p/netpbm/code/HEAD/tree/super_stable/editor/pnmstitch.c#l2": "pnmstitch", + "sourceforge.net/p/netpbm/code/HEAD/tree/super_stable/netpbm.c#l8": "HPND-Pbmplus", + "sourceforge.net/p/sbcl/sbcl/ci/master/tree/src/code/loop.lisp": "LOOP", + "sourceforge.net/p/trustedqsl/tqsl/ci/master/tree/LICENSE.txt": "TrustedQSL", + "sourceforge.net/p/xmedcon/code/ci/master/tree/libs/dicom/README": "OFFIS", + "sourceforge.net/p/xmedcon/code/ci/master/tree/libs/ljpg/": "IJG-short", + "sourceforge.net/projects/sevenzip/files/LZMA%20SDK/": "LZMA-SDK-9.22", + "sources.debian.org/copyright/license/debianutils/4.11.2/": "SMAIL-GPL", + "sources.debian.org/src/openmpi/4.1.0-10/ompi/debuggers/msgq_interface.h/?hl=19#L19": "mpi-permissive", + "sources.gentoo.org/cgi-bin/viewvc.cgi/gentoo-x86/licenses/BitTorrent?r1=1.1&r2=1.1.1.1&diff_format=s": "BitTorrent-1.0", + "sourceware.org/bzip2/1.0.5/bzip2-manual-1.0.5.html": "bzip2-1.0.6", + "sourceware.org/cgit/binutils-gdb/tree/include/coff/sym.h#n11": "MIPS", + "sourceware.org/cgit/binutils-gdb/tree/include/coff/symconst.h#n11": "ThirdEye", + "sourceware.org/cgit/valgrind/tree/mpi/libmpiwrap.c": "bzip2-1.0.6", "sourceware.org/git/?p=binutils-gdb.git;a=blob;f=libiberty/strcasecmp.c;h=131d81c2ce7881fa48c363dc5bf5fb302c61ce0b;hb=HEAD": "BSD-4.3RENO", "sourceware.org/git/?p=binutils-gdb.git;a=blob;f=readline/readline/support/wcwidth.c;h=0f5ec995796f4813abbcf4972aec0378ab74722a;hb=HEAD#l55": "HPND-Markus-Kuhn", "sourceware.org/git/?p=bzip2.git;a=blob;f=LICENSE;hb=bzip2-1.0.6": "bzip2-1.0.6", @@ -2129,6 +2181,7 @@ var urlToLicense = map[string]string{ "tahoe-lafs.org/trac/tahoe-lafs/browser/trunk/COPYING.TGPPL.rst": "TGPPL-1.0", "ti.arc.nasa.gov/opensource/nosa/": "NASA-1.3", "trac.enlightenment.org/e/browser/trunk/imlib2/COPYING": "Imlib2", + "tunes.org/legalese/bugroff.html": "Bugroff", "ubuntu.com/legal/font-licence": "Ubuntu-font-1.0", "unlicense.org/": "Unlicense", "vimdoc.sourceforge.net/htmldoc/uganda.html": "Vim", @@ -2139,6 +2192,7 @@ var urlToLicense = map[string]string{ "web.archive.org/web/20041020171434/http://www.rosenlaw.com/osl2.0.html": "OSL-2.0", "web.archive.org/web/20050212003940/http://www.rosenlaw.com/osl21.htm": "OSL-2.1", "web.archive.org/web/20060319014854/http://info.borland.com/devsupport/interbase/opensource/IPL.html": "Interbase-1.0", + "web.archive.org/web/20061013013632/http://opensource.linux-mirror.org/licenses/afl-2.1.txt": "AFL-2.1", "web.archive.org/web/20080207013128/http://www.gnome.org/fonts/": "Bitstream-Vera", "web.archive.org/web/20100302225219/http://www.zimbra.com/license/zimbra-public-license-1-3.html": "Zimbra-1.3", "web.archive.org/web/20111123062036/http://research.cs.wisc.edu/condor/license.html#condor": "Condor-1.1", @@ -2243,11 +2297,11 @@ var urlToLicense = map[string]string{ "www.jython.org/license.html": "CNRI-Jython", "www.kermitproject.org/ck90.html#source": "HPND-export-US", "www.kernel.org/doc/man-pages/licenses.html": "Linux-man-pages-copyleft", + "www.latex-project.org/lppl/LPPL-1.3c.txt": "LPPL-1.3c", "www.latex-project.org/lppl/lppl-1-0.txt": "LPPL-1.0", "www.latex-project.org/lppl/lppl-1-1.txt": "LPPL-1.1", "www.latex-project.org/lppl/lppl-1-2.txt": "LPPL-1.2", "www.latex-project.org/lppl/lppl-1-3a.txt": "LPPL-1.3a", - "www.latex-project.org/lppl/lppl-1-3c.txt": "LPPL-1.3c", "www.lemis.com/grog/UNIX/ancient-source-all.pdf": "Caldera", "www.libpng.org/pub/png/src/libpng-LICENSE.txt": "libpng-2.0", "www.mail-archive.com/debian-bugs-rc@lists.debian.org/msg11494.html": "Symlinks", @@ -2269,16 +2323,17 @@ var urlToLicense = map[string]string{ "www.netlib.org/lapack/LICENSE.txt": "BSD-3-Clause-Open-MPI", "www.netlib.org/minpack/disclaimer": "Minpack", "www.networkcultures.org/_uploads/%233notebook_telekommunist.pdf": "PPL", - "www.oclc.org/research/activities/software/license/v2final.htm": "OCLC-2.0", - "www.ogc.org/ogc/software/1.0": "OGC-1.0", - "www.ohwr.org/project/cernohl/wikis/Documents/CERN-OHL-version-2": "CERN-OHL-W-2.0", - "www.ohwr.org/project/licenses/wikis/cern-ohl-v1.1": "CERN-OHL-1.1", - "www.ohwr.org/project/licenses/wikis/cern-ohl-v1.2": "CERN-OHL-1.2", - "www.open-mpi.org/community/license.php": "BSD-3-Clause-Open-MPI", - "www.opencascade.com/content/occt-public-license": "OCCT-PL", - "www.opendatacommons.org/licenses/odbl/1.0/": "ODbL-1.0", - "www.opengroup.org/testing/downloads/The_Open_Group_TSL.txt": "OGTSL", - "www.openhub.net/licenses/mslpl": "MS-LPL", + "www.nist.gov/open/copyright-fair-use-and-licensing-statements-srd-data-software-and-technical-series-publications": "NIST-Software", + "www.oclc.org/research/activities/software/license/v2final.htm": "OCLC-2.0", + "www.ogc.org/ogc/software/1.0": "OGC-1.0", + "www.ohwr.org/project/cernohl/wikis/Documents/CERN-OHL-version-2": "CERN-OHL-W-2.0", + "www.ohwr.org/project/licenses/wikis/cern-ohl-v1.1": "CERN-OHL-1.1", + "www.ohwr.org/project/licenses/wikis/cern-ohl-v1.2": "CERN-OHL-1.2", + "www.open-mpi.org/community/license.php": "BSD-3-Clause-Open-MPI", + "www.opencascade.com/content/occt-public-license": "OCCT-PL", + "www.opendatacommons.org/licenses/odbl/1.0/": "ODbL-1.0", + "www.opengroup.org/testing/downloads/The_Open_Group_TSL.txt": "OGTSL", + "www.openhub.net/licenses/mslpl": "MS-LPL", "www.openldap.org/devel/gitweb.cgi?p=openldap.git;a=blob;f=LICENSE;hb=1cae062821881f41b73012ba816434897abf4205": "OLDAP-2.6", "www.openldap.org/devel/gitweb.cgi?p=openldap.git;a=blob;f=LICENSE;hb=37b4b3f6cc4bf34e1d3dec61e69914b9819d8894": "NBPL-1.0", "www.openldap.org/devel/gitweb.cgi?p=openldap.git;a=blob;f=LICENSE;hb=42b0383c50c299977b5893ee695cf4e486fb0dc7": "OLDAP-1.2", @@ -2301,7 +2356,7 @@ var urlToLicense = map[string]string{ "www.opensource.apple.com/license/apsl/": "APSL-2.0", "www.opensource.apple.com/source/IOSerialFamily/IOSerialFamily-7/APPLE_LICENSE": "APSL-1.1", "www.opensource.apple.com/source/tcl/tcl-5/tcl/generic/regfronts.c": "Spencer-99", - "www.opensource.org/licenses/EPL-2.0": "EPL-2.0", + "www.opensource.org/license/EPL-2.0": "EPL-2.0", "www.openssl.org/source/license.html": "OpenSSL", "www.osetfoundation.org/public-license": "OSET-PL-2.1", "www.parliament.uk/site-information/copyright-parliament/open-parliament-licence/": "OPL-UK-3.0", From 0bbcff1ae3c4fadbad4701a9e35898a576055a42 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ismael=20D=C3=B6nmez?= Date: Wed, 30 Sep 2026 17:54:36 +0200 Subject: [PATCH 39/40] Fix hang when scanning a Java resource adapter (.rar) file (#5348) * fix: scan a Java resource adapter (.rar) given as a file source The file source picks an extractor for the scan target from its file name alone. A Java resource adapter archive (JCA) uses the .rar extension but is a zip file, so it was handed to the RAR extractor. With rardecode v2.2.0 the RAR extractor does not return on non-RAR input (an infinite loop in its signature search), so `syft scan --from file adapter.rar` never finished. The same bytes named .jar scan in under a second. When the name says RAR but the content is a zip archive, leave the file unextracted, the same way .jar, .war and .ear scan targets are handled. The java cataloger already reads .rar files (anchore/syft#4136) and now finds the resource adapter and its nested jars. Real RAR archives are still extracted. Signed-off-by: Ismael Donmez Co-Authored-By: Claude Opus 5.5 (1M context) * chore(deps): bump github.com/nwaples/rardecode/v2 from 2.2.0 to 2.4.1 rardecode v2.2.0 loops forever in its RAR signature search when the input does not start with a RAR signature (fixed upstream in nwaples/rardecode f501ac8, "fix infinite loop in findSig", released in v2.2.2). syft reaches that search for any file source whose name contains ".rar", so a scan of a self-extracting RAR, or of any other file with such a name, never finished. With v2.4.1 a self-extracting RAR is extracted, and a file that is not a RAR archive fails fast with "rardecode: RAR signature not found", the same way a file named .zip that is not a zip archive fails today. github.com/mholt/archives v0.1.5 (the latest release) still requires v2.2.0, so the newer version is selected here. Signed-off-by: Ismael Donmez Co-Authored-By: Claude Opus 5.5 (1M context) * fix: clarify the .rar zip log line and cover self-extracting RAR sources - the debug line no longer calls every `.rar`-named zip a resource adapter - a self-extracting RAR (stub before the signature) given as a file source is still extracted Signed-off-by: Alex Goodman --------- Signed-off-by: Ismael Donmez Signed-off-by: Alex Goodman Co-authored-by: Claude Opus 5.5 (1M context) Co-authored-by: Alex Goodman --- go.mod | 2 +- go.sum | 4 +- syft/source/filesource/file_source.go | 24 ++++ syft/source/filesource/file_source_test.go | 119 +++++++++++++++++++ syft/source/testdata/rar-archive/example.rar | Bin 0 -> 86 bytes test/cli/archive_test.go | 55 +++++++++ 6 files changed, 201 insertions(+), 3 deletions(-) create mode 100644 syft/source/testdata/rar-archive/example.rar diff --git a/go.mod b/go.mod index b7573e132..09b8346f3 100644 --- a/go.mod +++ b/go.mod @@ -248,7 +248,7 @@ require ( github.com/muesli/cancelreader v0.2.2 // indirect github.com/muesli/termenv v0.16.0 // indirect github.com/ncruces/go-strftime v1.0.0 // indirect - github.com/nwaples/rardecode/v2 v2.2.0 // indirect + github.com/nwaples/rardecode/v2 v2.4.1 // indirect github.com/olekukonko/cat v0.0.0-20250911104152-50322a0618f6 // indirect github.com/olekukonko/errors v1.2.0 // indirect github.com/olekukonko/ll v0.1.6 // indirect diff --git a/go.sum b/go.sum index f8bbcca93..b66b9b51a 100644 --- a/go.sum +++ b/go.sum @@ -758,8 +758,8 @@ github.com/nix-community/go-nix v0.0.0-20250101154619-4bdde671e0a1 h1:kpt9ZfKcm+ github.com/nix-community/go-nix v0.0.0-20250101154619-4bdde671e0a1/go.mod h1:qgCw4bBKZX8qMgGeEZzGFVT3notl42dBjNqO2jut0M0= github.com/nsf/jsondiff v0.0.0-20210926074059-1e845ec5d249 h1:NHrXEjTNQY7P0Zfx1aMrNhpgxHmow66XQtm0aQLY0AE= github.com/nsf/jsondiff v0.0.0-20210926074059-1e845ec5d249/go.mod h1:mpRZBD8SJ55OIICQ3iWH0Yz3cjzA61JdqMLoWXeB2+8= -github.com/nwaples/rardecode/v2 v2.2.0 h1:4ufPGHiNe1rYJxYfehALLjup4Ls3ck42CWwjKiOqu0A= -github.com/nwaples/rardecode/v2 v2.2.0/go.mod h1:7uz379lSxPe6j9nvzxUZ+n7mnJNgjsRNb6IbvGVHRmw= +github.com/nwaples/rardecode/v2 v2.4.1 h1:F7zNW2LdAuuBThHWXQaiFUGVD/sef299NfWSB1nHAl4= +github.com/nwaples/rardecode/v2 v2.4.1/go.mod h1:7uz379lSxPe6j9nvzxUZ+n7mnJNgjsRNb6IbvGVHRmw= github.com/olekukonko/cat v0.0.0-20250911104152-50322a0618f6 h1:zrbMGy9YXpIeTnGj4EljqMiZsIcE09mmF8XsD5AYOJc= github.com/olekukonko/cat v0.0.0-20250911104152-50322a0618f6/go.mod h1:rEKTHC9roVVicUIfZK7DYrdIoM0EOr8mK1Hj5s3JjH0= github.com/olekukonko/errors v1.2.0 h1:10Zcn4GeV59t/EGqJc8fUjtFT/FuUh5bTMzZ1XwmCRo= diff --git a/syft/source/filesource/file_source.go b/syft/source/filesource/file_source.go index 1c49e4905..a443d2baa 100644 --- a/syft/source/filesource/file_source.go +++ b/syft/source/filesource/file_source.go @@ -214,6 +214,11 @@ func fileAnalysisPath(path string, skipExtractArchive bool) (string, func() erro envelopedUnarchiver, _, err := intFile.IdentifyArchive(context.Background(), path, nil) if unarchiver, ok := envelopedUnarchiver.(archives.Extractor); err == nil && ok { + if isZipWithRarName(path, envelopedUnarchiver) { + log.Debugf("source path has a .rar name but zip content (e.g. a Java resource adapter archive); analyzing it as a single file") + return analysisPath, cleanupFn, nil + } + analysisPath, cleanupFn, err = unarchiveToTmp(path, unarchiver) if err != nil { return "", cleanupFn, fmt.Errorf("unable to unarchive source file: %w", err) @@ -225,6 +230,25 @@ func fileAnalysisPath(path string, skipExtractArchive bool) (string, func() erro return analysisPath, cleanupFn, nil } +// isZipWithRarName reports whether the archive format was identified as RAR from the file name while the file +// content is a zip archive. Archive identification above uses the file name only, and ".rar" is also the extension +// of Java resource adapter archives (JCA), which are zip files. Such a file is left for the java cataloger to read, +// the same way .jar, .war and .ear files are, instead of being handed to the RAR extractor, which cannot read it. +func isZipWithRarName(path string, format archives.Format) bool { + if _, ok := format.(archives.Rar); !ok { + return false + } + + f, err := os.Open(path) + if err != nil { + return false + } + defer f.Close() + + match, err := archives.Zip{}.Match(context.Background(), "", f) + return err == nil && match.ByStream +} + func digestOfFileContents(path string) string { f, err := os.Open(path) if err != nil { diff --git a/syft/source/filesource/file_source_test.go b/syft/source/filesource/file_source_test.go index bba5c61e2..1866dd668 100644 --- a/syft/source/filesource/file_source_test.go +++ b/syft/source/filesource/file_source_test.go @@ -1,6 +1,8 @@ package filesource import ( + "archive/zip" + "bytes" "io" "os" "os/exec" @@ -173,6 +175,123 @@ func TestNewFromFile_WithArchive(t *testing.T) { } } +func TestNewFromFile_RarExtension(t *testing.T) { + testutil.Chdir(t, "..") // run with source/testdata + + testCases := []struct { + desc string + input string + wantExtracted bool + wantPaths []string + wantNoPaths []string + }{ + { + // a Java resource adapter archive is a zip file with a .rar extension: it must not be handed to the RAR + // extractor, but left for the java cataloger to read as a single file (like a .jar) + desc: "zip archive with a .rar name is analyzed as a single file", + input: createResourceAdapterArchive(t, t.TempDir()), + wantExtracted: false, + wantPaths: []string{"example-ra-1.0.0.rar"}, + wantNoPaths: []string{"META-INF/ra.xml", "example-lib-1.0.0.jar"}, + }, + { + // a self-extracting RAR has an executable stub before the RAR signature. The stub holds no "R" bytes, + // which is the input that made rardecode v2.2.0 loop forever in its signature search + desc: "self-extracting RAR archive is extracted", + input: writeSfxRar(t, t.TempDir()), + wantExtracted: true, + wantPaths: []string{"inside.txt"}, + wantNoPaths: []string{"example-sfx.rar"}, + }, + { + // a RAR 5.0 archive holding one stored file (inside.txt) + desc: "RAR archive is extracted", + input: "testdata/rar-archive/example.rar", + wantExtracted: true, + wantPaths: []string{"inside.txt"}, + wantNoPaths: []string{"example.rar"}, + }, + } + for _, test := range testCases { + t.Run(test.desc, func(t *testing.T) { + src, err := New(Config{ + Path: test.input, + }) + require.NoError(t, err) + t.Cleanup(func() { + require.NoError(t, src.Close()) + }) + + assert.Equal(t, test.wantExtracted, src.(*fileSource).analysisPath != test.input) + + res, err := src.FileResolver(source.SquashedScope) + require.NoError(t, err) + + for _, p := range test.wantPaths { + refs, err := res.FilesByPath(p) + require.NoError(t, err) + assert.Len(t, refs, 1, "expected to find %q", p) + } + + for _, p := range test.wantNoPaths { + refs, err := res.FilesByPath(p) + require.NoError(t, err) + assert.Empty(t, refs, "expected not to find %q", p) + } + }) + } +} + +// createResourceAdapterArchive writes a minimal Java resource adapter archive (a zip file with a .rar extension that +// holds a deployment descriptor and a nested jar) into dir and returns its path. +func createResourceAdapterArchive(t testing.TB, dir string) string { + t.Helper() + + lib := zipBytes(t, [][2]string{ + {"META-INF/MANIFEST.MF", "Manifest-Version: 1.0\n"}, + {"META-INF/maven/org.example/example-lib/pom.properties", "groupId=org.example\nartifactId=example-lib\nversion=1.0.0\n"}, + }) + + ra := zipBytes(t, [][2]string{ + {"META-INF/MANIFEST.MF", "Manifest-Version: 1.0\n"}, + {"META-INF/ra.xml", "\n"}, + {"example-lib-1.0.0.jar", string(lib)}, + {"META-INF/maven/org.example/example-ra/pom.properties", "groupId=org.example\nartifactId=example-ra\nversion=1.0.0\n"}, + }) + + archivePath := filepath.Join(dir, "example-ra-1.0.0.rar") + require.NoError(t, os.WriteFile(archivePath, ra, 0o600)) + return archivePath +} + +// writeSfxRar writes testdata/rar-archive/example.rar behind a stand-in executable stub into dir and returns its path. +func writeSfxRar(t testing.TB, dir string) string { + t.Helper() + + rar, err := os.ReadFile("testdata/rar-archive/example.rar") + require.NoError(t, err) + + sfxPath := filepath.Join(dir, "example-sfx.rar") + require.NoError(t, os.WriteFile(sfxPath, append(bytes.Repeat([]byte("MZ stub "), 1024), rar...), 0o600)) + return sfxPath +} + +// zipBytes returns a zip archive holding the given (name, contents) entries, in order. +func zipBytes(t testing.TB, entries [][2]string) []byte { + t.Helper() + + var buf bytes.Buffer + w := zip.NewWriter(&buf) + for _, e := range entries { + f, err := w.Create(e[0]) + require.NoError(t, err) + _, err = f.Write([]byte(e[1])) + require.NoError(t, err) + } + require.NoError(t, w.Close()) + return buf.Bytes() +} + // setupArchiveTest encapsulates common test setup work for tar file tests. It returns a cleanup function, // which should be called (typically deferred) by the caller, the path of the created tar archive, and an error, // which should trigger a fatal test failure in the consuming test. The returned cleanup function will never be nil diff --git a/syft/source/testdata/rar-archive/example.rar b/syft/source/testdata/rar-archive/example.rar new file mode 100644 index 0000000000000000000000000000000000000000..f2641fd4c8bfbe75b73a48d15632b8b40f508c5c GIT binary patch literal 86 zcmWGaEK-zWXJj}kWo*RE$iToq_sIhZCMHD|#U;#|3zr!)Fmh$)6=$ZT>XlTKBr2q3 m=A1QZe#ic%AE6oMRs6cUS)GcwCkxg\n"}, + {"example-lib-1.0.0.jar", string(lib)}, + {"META-INF/maven/org.example/example-ra/pom.properties", "groupId=org.example\nartifactId=example-ra\nversion=1.0.0\n"}, + }) + + archivePath := filepath.Join(destDir, "example-ra-1.0.0.rar") + require.NoError(t, os.WriteFile(archivePath, ra, 0o600)) + return archivePath +} + +// zipBytes returns a zip archive holding the given (name, contents) entries, in order. +func zipBytes(t *testing.T, entries [][2]string) []byte { + var buf bytes.Buffer + w := zip.NewWriter(&buf) + for _, e := range entries { + f, err := w.Create(e[0]) + require.NoError(t, err) + _, err = f.Write([]byte(e[1])) + require.NoError(t, err) + } + require.NoError(t, w.Close()) + return buf.Bytes() +} From d61c2cada3e4b21ce44b6820bcf37ba3e861c3b8 Mon Sep 17 00:00:00 2001 From: Mayank Sekhar Date: Wed, 30 Sep 2026 22:21:03 +0530 Subject: [PATCH 40/40] fix(binary): correct traefik version detection on big-endian arches (#5281) The traefik-binary classifier regex required a NUL byte or replacement character immediately before the version string. On some architectures (e.g. s390x), the real version string in the binary has no such prefix, so the classifier instead matched an earlier, unrelated NUL-prefixed numeric string elsewhere in the binary (e.g. '2.7.0' from an unrelated printf-style format constant), producing an incorrect version. This adds an additional pattern, tried first via MatchAny, that matches version strings terminated by two NUL bytes without requiring a NUL prefix. The original pattern remains as a fallback for older traefik binaries (e.g. v1.x) that use single-NUL termination. Fixes #4980 Signed-off-by: Mayank Sekhar --- .../binary/classifier_cataloger_test.go | 24 ++++++++++++++++++ syft/pkg/cataloger/binary/classifiers.go | 11 ++++++-- .../snippets/traefik/2.11.1/linux-arm/traefik | Bin 0 -> 279 bytes .../traefik/3.6.8/linux-s390x/traefik | Bin 0 -> 280 bytes 4 files changed, 33 insertions(+), 2 deletions(-) create mode 100644 syft/pkg/cataloger/binary/testdata/classifiers/snippets/traefik/2.11.1/linux-arm/traefik create mode 100644 syft/pkg/cataloger/binary/testdata/classifiers/snippets/traefik/3.6.8/linux-s390x/traefik diff --git a/syft/pkg/cataloger/binary/classifier_cataloger_test.go b/syft/pkg/cataloger/binary/classifier_cataloger_test.go index 4b4e2ed7d..54cd04f3b 100644 --- a/syft/pkg/cataloger/binary/classifier_cataloger_test.go +++ b/syft/pkg/cataloger/binary/classifier_cataloger_test.go @@ -407,6 +407,30 @@ func Test_Cataloger_PositiveCases(t *testing.T) { Metadata: metadata("traefik-binary"), }, }, + { + // regression test: big-endian architectures (e.g. s390x) previously matched an unrelated + // single-NUL-terminated numeric string elsewhere in the binary instead of the real version + logicalFixture: "traefik/3.6.8/linux-s390x", + expected: pkg.Package{ + Name: "traefik", + Version: "3.6.8", + Type: "binary", + PURL: "pkg:generic/traefik@3.6.8", + Locations: locations("traefik"), + Metadata: metadata("traefik-binary"), + }, + }, + { + logicalFixture: "traefik/2.11.1/linux-arm", + expected: pkg.Package{ + Name: "traefik", + Version: "2.11.1", + Type: "binary", + PURL: "pkg:generic/traefik@2.11.1", + Locations: locations("traefik"), + Metadata: metadata("traefik-binary"), + }, + }, { logicalFixture: "memcached/1.6.18/linux-amd64", expected: pkg.Package{ diff --git a/syft/pkg/cataloger/binary/classifiers.go b/syft/pkg/cataloger/binary/classifiers.go index 17834ab0c..26929ae2a 100644 --- a/syft/pkg/cataloger/binary/classifiers.go +++ b/syft/pkg/cataloger/binary/classifiers.go @@ -284,11 +284,18 @@ func DefaultClassifiers() []binutils.Classifier { { Class: "traefik-binary", FileGlob: "**/traefik", - EvidenceMatcher: m.FileContentsVersionMatcher( + EvidenceMatcher: binutils.MatchAny( + // on some architectures (e.g. s390x) the real version string isn't NUL-prefixed + // and is instead terminated by two NUL bytes; checking this pattern first avoids + // matching an unrelated single-NUL-terminated numeric string elsewhere in the binary + m.FileContentsVersionMatcher( + `(?m)(?P[0-9]+\.[0-9]+\.[0-9]+(-alpha[0-9]|-beta[0-9]|-rc[0-9])?)\x00\x00`), // [NUL]v1.7.34[NUL] // [NUL]2.9.6[NUL] // 3.0.4[NUL] - `(?m)(\x00v?|\x{FFFD}.?)(?P[0-9]+\.[0-9]+\.[0-9]+(-alpha[0-9]|-beta[0-9]|-rc[0-9])?)\x00`), + m.FileContentsVersionMatcher( + `(?m)(\x00v?|\x{FFFD}.?)(?P[0-9]+\.[0-9]+\.[0-9]+(-alpha[0-9]|-beta[0-9]|-rc[0-9])?)\x00`), + ), Package: "traefik", PURL: mustPURL("pkg:generic/traefik@version"), CPEs: singleCPE("cpe:2.3:a:traefik:traefik:*:*:*:*:*:*:*:*", cpe.NVDDictionaryLookupSource), diff --git a/syft/pkg/cataloger/binary/testdata/classifiers/snippets/traefik/2.11.1/linux-arm/traefik b/syft/pkg/cataloger/binary/testdata/classifiers/snippets/traefik/2.11.1/linux-arm/traefik new file mode 100644 index 0000000000000000000000000000000000000000..f0be458f085acc558ee8c1f5de24b738de5318e4 GIT binary patch literal 279 zcmZvWJx;_h5JtTn@+mB(U&Z6!NdugLj&3|2vk@VQSWAdg&~Ok=z?o?2*aAAf@;&M4 zTj@zA)1OJ&{HV9q&eA7?E+YoWdXe>^pC;q8KCkn>OTRl2ggBW1&c#}ME+b2E&AJ#` zbV(`@S%gAVIJm;>gN-2oCw3JDxFkTdy3LFH#j4NMVk8QsCh(l0I8T*u%#t(2F}4ef z9KE+bp=4y2ykKERjF_|@hQXBAF6Q^9>6>ZWvTQHrqO^HG+$go*e!qQuDs_JaM{AE( Kse}4&U)Mi9Hc8e1 literal 0 HcmV?d00001 diff --git a/syft/pkg/cataloger/binary/testdata/classifiers/snippets/traefik/3.6.8/linux-s390x/traefik b/syft/pkg/cataloger/binary/testdata/classifiers/snippets/traefik/3.6.8/linux-s390x/traefik new file mode 100644 index 0000000000000000000000000000000000000000..0141469f67050af9c17f554df4a88366d320ece8 GIT binary patch literal 280 zcmX|5J5Izf6m;ztN(xI`BCY)V{7V`xK*tGeKR-4iBrDbuqN3pjT!5Q!C5r8Xk$_?} zKWQ{edaR>rPpR_sAXny7ZKDD-V8Plb=epdt!>GDEEz|K>+ub1<=SP(-6iulFj~o#k z7Ba*c$YM$qL!s&^`fM_oFz7gt4N)gk9cI?iNS@~UHx>c%K%_%~l5%#8-gC7WDi{V# zh#~Yb4Fe72Oki&*nP4$G>Y