oss/syft
2
0
Fork 0
mirror of https://github.com/anchore/syft.git synced 2026-11-05 17:06:32 +01:00
Code Issues Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'upstream/main' into add-nested-archive-cataloging

Browse Source 
Signed-off-by: Keith Zantow <keith.zantow@anchore.com>
This commit is contained in:
Keith Zantow
2026-09-30 14:55:38 -04:00
parent 8e2cdf2429 d61c2cada3
commit 12d3097741
188 changed files with 5325 additions and 1234 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.binny.yaml
+1 -1
View File
@@ -16,7 +16,7 @@ tools:
# used to upload test fixture cache
- name: oras
version:
want: v1.3.3
want: v1.3.4
method: github-release
with:
repo: oras-project/oras
.gitattributes
+7
View File
@@ -0,0 +1,7 @@
# check out text files with lf on every platform (regardless of core.autocrlf), letting git detect binary files
* text=auto eol=lf
# fixtures that must keep their crlf line endings byte for byte
syft/linux/testdata/unprintable -text
syft/pkg/cataloger/bitnami/testdata/no-rel/opt/bitnami/redis/.spdx-redis.spdx -text
syft/pkg/cataloger/java/testdata/jar-metadata/**/META-INF/MANIFEST.MF -text
.github/workflows/release.yaml
+1 -1
View File
@@ -86,7 +86,7 @@ jobs:
# dockers_v2 builds multi-platform manifests in a single buildx invocation, which requires a
# docker-container driver builder (the default builder on the runner cannot build multi-platform).
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@37fe631027851001ddb9b187196cc803df7f5f0e # v4.3.0
uses: docker/setup-buildx-action@f87e5991a6d7451dcb8d9637bfbc97413f497069 # v4.4.1
- name: Login to Docker Hub
uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f #v4.6.0
.github/workflows/validations.yaml
+29 -1
View File
@@ -27,7 +27,7 @@ jobs:
# enough; a 4 vCPU instance can't pull much more than that anyway.
# note: s3-cache intentionally omitted -- PR runs are untrusted and must not write to the
# shared cache backend that the trusted release workflow reads from (cache poisoning).
runs-on: &test-runner "runs-on=${{ github.run_id }}/cpu=4+8/ram=16+32/family=m5+m6+m7+m8+c5+c6+c7+c8/spot=price-capacity-optimized/volume=100gb"
runs-on: &test-runner "runs-on=${{ github.run_id }}/cpu=4+8/ram=16+32/family=m6+m7+m8+c6+c7+c8/spot=price-capacity-optimized/volume=100gb"
permissions:
contents: read
steps:
@@ -49,6 +49,13 @@ jobs:
Unit-Test:
# Note: changing this job name requires making the same update in the .github/workflows/release.yaml pipeline
name: "Unit tests"
runs-on: ubuntu-latest
needs: [Unit-Test-Linux, Unit-Test-Windows]
steps:
- run: echo All unit tests passed!
Unit-Test-Linux:
name: "Unit tests (Linux)"
runs-on: *test-runner
permissions:
contents: read
@@ -68,6 +75,27 @@ jobs:
- name: Check for capability drift
run: make check-capability-drift
Unit-Test-Windows:
name: "Unit tests (Windows)"
runs-on: windows-latest
permissions:
contents: read
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 #v7.0.1
with:
persist-credentials: false
# go-make's setup only (not ./.github/actions/bootstrap): no apt, binny tools, or fixture cache are needed
- name: Setup go + go-make tooling
uses: anchore/go-make/.github/actions/setup@fa5421b3bf24c9d18f11a736c5eb4172f2b7d683 # v0.8.1
with:
# keep in sync with the go-version default in ./.github/actions/bootstrap
go-version: "1.26.2"
# the full unit suite leans on linux docker fixtures, so this runs the windows-relevant subset
- name: Run unit tests
run: make unit:windows
Integration-Test:
# Note: changing this job name requires making the same update in the .github/workflows/release.yaml pipeline
name: "Integration tests"
.golangci.yaml
+8 -1
View File
@@ -7,6 +7,7 @@ linters:
- asciicheck
- bodyclose
- copyloopvar
- depguard
- dogsled
- dupl
- errcheck
@@ -29,6 +30,13 @@ linters:
- unused
- whitespace
settings:
depguard:
rules:
yaml:
deny:
# gopkg.in/yaml.v3 is unmaintained and panics on some malformed input (see #5327)
- pkg: gopkg.in/yaml.v3
desc: use go.yaml.in/yaml/v3 instead
funlen:
lines: 70
statements: 50
@@ -71,7 +79,6 @@ linters:
# do not enable...
# - deadcode # The owner seems to have abandoned the linter. Replaced by "unused".
# - depguard # We don't have a configuration for this yet
# - goprintffuncname # does not catch all cases and there are exceptions
# - nakedret # does not catch all cases and should not fail a build
# - gochecknoglobals
.make/main.go
+19 -1
View File
@@ -1,6 +1,7 @@
package main
import (
"os"
"path/filepath"
"runtime"
"strconv"
@@ -52,7 +53,7 @@ func main() {
raceFlag = " -race"
}
Run(
"go test -count=1 -timeout=30m -v"+raceFlag+" ./cmd/syft/internal/test/integration/...",
"go test -count=1 -timeout=30m"+raceFlag+" ./cmd/syft/internal/test/integration/...",
run.Env("GODEBUG", "dontfreezetheworld=1"),
)
},
@@ -96,6 +97,23 @@ func main() {
},
},
// windows unit tests: deliberately not hooked into "test" (no RunsOn). The full unit suite leans on
// linux docker fixture images and posix-only assumptions, so this covers the packages that own
// host path handling (where windows bugs live) and skips the tests that need docker image fixtures.
// ponytail: curated package list, widen it as more of the suite is made windows-safe.
Task{
Name: "unit:windows",
Description: "run the windows-relevant subset of unit tests",
Run: func() {
Run("go test -count=1"+
` -skip "Image|Squash|AllLayers|MixFileTypes|GivenCoordinates"`+
" ./syft/internal/windows/... ./syft/internal/fileresolver/... ./syft/source/directorysource/... ./syft/source/filesource/..."+
" ./syft/file ./syft/file/cataloger/filedigest/... ./syft/file/cataloger/filecontent/...",
run.Stdout(os.Stderr),
)
},
},
// default validation pipeline (replaces Taskfile `default`/`pr-validations`/`validations`).
Task{
Name: "default",
cmd/syft/cli/ui/handle_attestation.go
+5
View File
@@ -158,6 +158,11 @@ func (l *backgroundLineReader) read(reader io.Reader, stage *progress.Stage) {
l.lock.Unlock()
}
if err := s.Err(); err != nil {
log.WithFields("error", err).Debug("unable to read attestation output")
// keep draining so the writer never blocks on a full pipe
_, _ = io.Copy(io.Discard, reader)
}
}
func (l backgroundLineReader) Lines() []string {
cmd/syft/internal/test/integration/multiplatform_oci_test.go
+11 -13
View File
@@ -16,19 +16,21 @@ import (
"github.com/anchore/syft/syft/source"
)
// multiplatformImage is busybox:1.38.0 pinned by index digest, since docker hub periodically rebuilds and
// re-pushes official image tags, which changes the per-platform digests asserted below.
const multiplatformImage = "docker.io/library/busybox@sha256:fd7dc98638c8e305f4dc34e979f1c0fdfdcaeb0fbf8fcff77ae834b6da3d7e6e"
// TestMultiPlatformOCIImageSelection verifies that syft can load a single platform out of a
// multi-platform OCI image on disk (both --from oci-dir and --from oci-archive) by selecting the
// image that matches the requested platform.
func TestMultiPlatformOCIImageSelection(t *testing.T) {
remoteImage := "docker.io/library/busybox:1.38.0"
// Per-platform image config digests within the multi-platform index, obtained from the OCI layout
// (see anchore/stereoscope integration tests: TestPlatformSelectionWithOciLocalSources).
expectedDigest := map[string]string{
"arm64": "sha256:e0e8b3cbfed68a90084781e2962f9c0deead51c5a3f11a488eef0283a4284bc2",
"s390x": "sha256:0cf160e720a8e4f20883b72276a6eaded83b79539f3f1d39e35a3336154b9960",
"amd64": "sha256:c6348fa86ba0fb2108c9334f5fe913ddc6d853313e655891f133a0127c30099f",
"ppc64le": "sha256:b144fc0e06537d07956cde340b878a81a717e394edb736d3110858a12a6635cb",
"arm64": "sha256:d2482869a6b838d3b4c9cad0c8085c06277909482771eb525d298fc3a7d07927",
"s390x": "sha256:e55a0fa55032cceac2d983f4b4a25b28e42303b46a2f1915fc0fa45051e6ca70",
"amd64": "sha256:aaef90e065235eb0b2d9938be85d1b81add2902cf89affe2cdfa19be33476ac0",
"ppc64le": "sha256:8158829b015d1c5285d63d54d37e95998ef0af4177e3f416678decbcaff3170c",
}
// syft --from source tag -> stereoscope OCI source used to prepare the local fixture
@@ -39,7 +41,7 @@ func TestMultiPlatformOCIImageSelection(t *testing.T) {
for from, imageSource := range sources {
t.Run(from, func(t *testing.T) {
localPath := imagetest.PrepareMultiplatformFixtureImage(t, imageSource, remoteImage)
localPath := imagetest.PrepareMultiplatformFixtureImage(t, imageSource, multiplatformImage)
for _, arch := range []string{"amd64", "arm64", "s390x", "ppc64le"} {
t.Run(fmt.Sprintf("linux/%s", arch), func(t *testing.T) {
platform, err := image.NewPlatform("linux/" + arch)
@@ -73,8 +75,6 @@ func TestMultiPlatformOCIImageSelection(t *testing.T) {
// TestMultiPlatformOCIImageSelection_UnavailablePlatform verifies that requesting a platform not present
// in the multi-platform OCI image results in an error rather than silently selecting the wrong image.
func TestMultiPlatformOCIImageSelection_UnavailablePlatform(t *testing.T) {
remoteImage := "docker.io/library/busybox:1.38.0"
// windows/amd64 is not present in this linux-only multi-platform image
platform, err := image.NewPlatform("windows/amd64")
require.NoError(t, err)
@@ -86,7 +86,7 @@ func TestMultiPlatformOCIImageSelection_UnavailablePlatform(t *testing.T) {
for from, imageSource := range sources {
t.Run(from, func(t *testing.T) {
localPath := imagetest.PrepareMultiplatformFixtureImage(t, imageSource, remoteImage)
localPath := imagetest.PrepareMultiplatformFixtureImage(t, imageSource, multiplatformImage)
_, err := syft.GetSource(
context.Background(),
@@ -101,8 +101,6 @@ func TestMultiPlatformOCIImageSelection_UnavailablePlatform(t *testing.T) {
// TestMultiPlatformOCIImageSelection_DefaultPlatform verifies that not specifying a platform results
// in the current platform being selected.
func TestMultiPlatformOCIImageSelection_DefaultPlatform(t *testing.T) {
remoteImage := "docker.io/library/busybox:1.38.0"
sources := map[string]image.Source{
"oci-dir": image.OciDirectorySource,
"oci-archive": image.OciTarballSource,
@@ -110,7 +108,7 @@ func TestMultiPlatformOCIImageSelection_DefaultPlatform(t *testing.T) {
for from, imageSource := range sources {
t.Run(from, func(t *testing.T) {
localPath := imagetest.PrepareMultiplatformFixtureImage(t, imageSource, remoteImage)
localPath := imagetest.PrepareMultiplatformFixtureImage(t, imageSource, multiplatformImage)
src, err := syft.GetSource(
context.Background(),
cmd/syft/internal/ui/ui.go
+45 -1
View File
@@ -1,6 +1,7 @@
package ui
import (
"bytes"
"fmt"
"io"
"os"
@@ -9,6 +10,7 @@ import (
"time"
tea "github.com/charmbracelet/bubbletea"
"github.com/charmbracelet/lipgloss"
"github.com/wagoodman/go-partybus"
"github.com/anchore/bubbly"
@@ -53,7 +55,7 @@ func New(out io.Writer, quiet bool, handlers ...bubbly.EventHandler) *UI {
func (m *UI) Setup(subscription partybus.Unsubscribable) error {
// we still want to collect log messages, however, we also the logger shouldn't write to the screen directly
if logWrapper, ok := log.Get().(logger.Controller); ok {
logWrapper.SetOutput(m.frame.(*frame.Frame).Footer())
logWrapper.SetOutput(&logFooter{w: m.frame.(*frame.Frame).Footer()})
}
m.subscription = subscription
@@ -193,6 +195,48 @@ func (m UI) View() string {
return v
}
var (
logTitleStyle = lipgloss.NewStyle().Foreground(lipgloss.Color("8")) // dark grey
// the first log line gets a branch pointing at it from the title, the rest are indented to match
logBranch = logTitleStyle.Render("└─▶") + " "
logIndent = " "
)
// logFooter decorates log entries drawn below the TUI with a one-time title (written when the first entry arrives)
// and a branch from the title to the first log line. Writes are serialized by the logger.
type logFooter struct {
w io.Writer
started bool
}
func (l *logFooter) Write(p []byte) (int, error) {
var buf bytes.Buffer
first := !l.started
if first {
buf.WriteString(logTitleStyle.Render("Logs (non-fatal, see -v for more):") + "\n")
l.started = true
}
for i, line := range bytes.SplitAfter(p, []byte("\n")) {
if len(line) == 0 {
continue
}
if first && i == 0 {
buf.WriteString(logBranch)
} else {
buf.WriteString(logIndent)
}
buf.Write(line)
}
if _, err := l.w.Write(buf.Bytes()); err != nil {
return 0, err
}
return len(p), nil
}
// IsTerminal marks the footer as terminal-bound (it is drawn by the TUI) so the logger keeps colors. NO_COLOR is
// still honored by the formatter.
func (*logFooter) IsTerminal() bool { return true }
func runWithTimeout(timeout time.Duration, fn func() error) (err error) {
c := make(chan struct{}, 1)
go func() {
go.mod
+13 -13
View File
@@ -1,6 +1,6 @@
module github.com/anchore/syft
go 1.26.3
go 1.26.8
require (
github.com/BurntSushi/toml v1.6.0
@@ -11,12 +11,12 @@ require (
github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d
github.com/acobaugh/osrelease v0.1.0
github.com/adrg/xdg v0.5.3
github.com/anchore/bubbly v0.2.1
github.com/anchore/bubbly v0.2.2
github.com/anchore/clio v0.1.1
github.com/anchore/fangs v0.1.1
github.com/anchore/go-collections v0.1.1
github.com/anchore/go-homedir v0.1.1
github.com/anchore/go-logger v0.1.1
github.com/anchore/go-logger v0.2.0
github.com/anchore/go-macholibre v0.1.1
github.com/anchore/go-rpmdb v0.2.0
github.com/anchore/go-sync v0.1.2
@@ -36,7 +36,7 @@ require (
github.com/deitch/magic v0.0.0-20230404182410-1ff89d7342da
github.com/diskfs/go-diskfs v1.9.4
github.com/distribution/reference v0.6.0
github.com/dustin/go-humanize v1.0.1
github.com/dustin/go-humanize v1.1.0
github.com/elliotchance/phpserialize v1.4.0
github.com/facebookincubator/nvdtools v0.1.5
github.com/github/go-spdx/v2 v2.7.0
@@ -56,18 +56,18 @@ require (
github.com/hashicorp/go-cleanhttp v0.5.2
github.com/hashicorp/go-getter v1.8.9
github.com/hashicorp/go-multierror v1.1.1
github.com/hashicorp/hcl/v2 v2.24.0
github.com/hashicorp/hcl/v2 v2.25.0
github.com/iancoleman/strcase v0.3.0
github.com/invopop/jsonschema v0.14.0
github.com/jedib0t/go-pretty/v6 v6.8.3
github.com/jinzhu/copier v0.4.0
github.com/kastenhq/goversion v0.0.0-20230811215019-93b2f8823953
github.com/klauspost/compress v1.20.0
github.com/magiconair/properties v1.18.11
github.com/magiconair/properties v1.18.12
github.com/mholt/archives v0.1.5
github.com/moby/sys/mountinfo v0.7.2
github.com/nix-community/go-nix v0.0.0-20250101154619-4bdde671e0a1
github.com/olekukonko/tablewriter v1.1.4
github.com/olekukonko/tablewriter v1.1.5
github.com/opencontainers/go-digest v1.0.0
github.com/pelletier/go-toml v1.9.5
github.com/quasilyte/go-ruleguard/dsl v0.3.23
@@ -97,8 +97,7 @@ require (
golang.org/x/net v0.59.0
golang.org/x/time v0.16.0
golang.org/x/tools v0.50.0
gopkg.in/yaml.v3 v3.0.1
modernc.org/sqlite v1.58.0
modernc.org/sqlite v1.59.0
)
require (
@@ -131,6 +130,7 @@ require (
github.com/anchore/go-struct-converter v0.2.0-rc2 // indirect
github.com/andybalholm/brotli v1.2.0 // indirect
github.com/apparentlymart/go-textseg/v15 v15.0.0 // indirect
github.com/apparentlymart/go-textseg/v17 v17.0.1 // indirect
github.com/aquasecurity/go-version v0.0.1 // indirect
github.com/atotto/clipboard v0.1.4 // indirect
github.com/aws/aws-sdk-go-v2 v1.44.0 // indirect
@@ -170,7 +170,7 @@ require (
github.com/cncf/xds/go v0.0.0-20260202195803-dba9d589def2 // indirect
github.com/containerd/cgroups/v3 v3.1.3 // indirect
github.com/containerd/containerd/api v1.11.1 // indirect
github.com/containerd/containerd/v2 v2.3.5 // indirect
github.com/containerd/containerd/v2 v2.3.6 // indirect
github.com/containerd/continuity v0.5.0 // indirect
github.com/containerd/errdefs v1.0.0 // indirect
github.com/containerd/errdefs/pkg v0.3.0 // indirect
@@ -248,7 +248,7 @@ require (
github.com/muesli/cancelreader v0.2.2 // indirect
github.com/muesli/termenv v0.16.0 // indirect
github.com/ncruces/go-strftime v1.0.0 // indirect
github.com/nwaples/rardecode/v2 v2.2.0 // indirect
github.com/nwaples/rardecode/v2 v2.4.1 // indirect
github.com/olekukonko/cat v0.0.0-20250911104152-50322a0618f6 // indirect
github.com/olekukonko/errors v1.2.0 // indirect
github.com/olekukonko/ll v0.1.6 // indirect
@@ -293,7 +293,7 @@ require (
github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 // indirect
github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8 // indirect
github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e // indirect
github.com/zclconf/go-cty v1.16.3 // indirect
github.com/zclconf/go-cty v1.19.0 // indirect
go.opencensus.io v0.24.0 // indirect
go.opentelemetry.io/auto/sdk v1.2.1 // indirect
go.opentelemetry.io/contrib/detectors/gcp v1.44.0 // indirect
@@ -321,7 +321,7 @@ require (
google.golang.org/grpc v1.83.2 // indirect
google.golang.org/protobuf v1.36.12 // indirect
gopkg.in/warnings.v0 v0.1.2 // indirect
modernc.org/libc v1.75.6 // indirect
modernc.org/libc v1.75.7 // indirect
modernc.org/mathutil v1.7.1 // indirect
modernc.org/memory v1.12.1 // indirect
)
go.sum
+24 -22
View File
@@ -120,8 +120,8 @@ github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuy
github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
github.com/anchore/bubbly v0.2.1 h1:b3ZNBefY8ks5DI4SQ7BXSbBIsN6eoaTD6s1su7NuCBo=
github.com/anchore/bubbly v0.2.1/go.mod h1:o6a9aH3/fCn+mb5g1T1tBbQZ4hGR0yO5/uD1d2Vtago=
github.com/anchore/bubbly v0.2.2 h1:53JHRlpaNZjMcUYdz7n1U7+kOhy0zsJv9gpGEXvTcz8=
github.com/anchore/bubbly v0.2.2/go.mod h1:M4twoFkczlfuNdU8QV7oQjVJwPbxdUWRMfKydcLba0U=
github.com/anchore/clio v0.1.1 h1:9bkYnaFbpT0J3G0qSHyQqa/lpK5zV0C9Gtgh82H/jj0=
github.com/anchore/clio v0.1.1/go.mod h1:CFPIopJlakJbBpKtAL/EPpj4W9hgBO7PfFc8N2FAtlE=
github.com/anchore/fangs v0.1.1 h1:NBTHocmWM2/0gNhebpoE/wEB0Z0nxh2lESww2o/Uf5M=
@@ -130,8 +130,8 @@ github.com/anchore/go-collections v0.1.1 h1:kGlmSehs+ctTBZr+98F2+taHeolHy+XDx1Nc
github.com/anchore/go-collections v0.1.1/go.mod h1:zgDbPSNDpfU47tVo7aW+2suG7+vXB1Sq1qtDhRJknQ0=
github.com/anchore/go-homedir v0.1.1 h1:KpaO5BPrVT7cVMcEZ9KfFfEi0iuo3DtagPGl0C744X0=
github.com/anchore/go-homedir v0.1.1/go.mod h1:9B0DGhbmMAJVGGEbrlb+PkORM5eDFCEOtZ3xQ22qaLA=
github.com/anchore/go-logger v0.1.1 h1:HPBNmQVBdYliwaFVUL7oJ3UvSlaoXAMj/ZagAIYaCpA=
github.com/anchore/go-logger v0.1.1/go.mod h1:ekWuh5BkZVwyXnyEd4fwL29N4pNLssgRsqh2+kFN/b8=
github.com/anchore/go-logger v0.2.0 h1:V8Gazz+F8xwWEWz+whqt8/5163IkO4XrAie+1qbK10k=
github.com/anchore/go-logger v0.2.0/go.mod h1:omqLWBwT/B5/aTwIH6pl5WBylBzeH3BGHUTS6OFeFvY=
github.com/anchore/go-lzo v0.1.1 h1:IwL/fvkdtlIrYIXck6WxZ3nb8WjjHziYYmGxlooyOnM=
github.com/anchore/go-lzo v0.1.1/go.mod h1:3kLx0bve2oN1iDwgM1U5zGku1Tfbdb0No5qp1eL1fIk=
github.com/anchore/go-macholibre v0.1.1 h1:sTBHi3S1WVIOCamc6NyT8Sn9mBmD30DCTfjfVm4k6/M=
@@ -156,6 +156,8 @@ github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be/go.mod h1:ySMOLuW
github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY=
github.com/apparentlymart/go-textseg/v15 v15.0.0 h1:uYvfpb3DyLSCGWnctWKGj857c6ew1u1fNQOlOtuGxQY=
github.com/apparentlymart/go-textseg/v15 v15.0.0/go.mod h1:K8XmNZdhEBkdlyDdvbmmsvpAG721bKi0joRfFdHIWJ4=
github.com/apparentlymart/go-textseg/v17 v17.0.1 h1:bpMXRgQ5cEoRNuQke1a80/Nl6w3G5eoIbWo9f3gXkAs=
github.com/apparentlymart/go-textseg/v17 v17.0.1/go.mod h1:fa8X4jgGeevslICIY6LcdjkSecWnXmYd9Lk34z/VxZs=
github.com/aquasecurity/go-pep440-version v0.0.1 h1:8VKKQtH2aV61+0hovZS3T//rUF+6GDn18paFTVS0h0M=
github.com/aquasecurity/go-pep440-version v0.0.1/go.mod h1:3naPe+Bp6wi3n4l5iBFCZgS0JG8vY6FT0H4NGhFJ+i4=
github.com/aquasecurity/go-version v0.0.1 h1:4cNl516agK0TCn5F7mmYN+xVs1E3S45LkgZk3cbaW2E=
@@ -294,8 +296,8 @@ github.com/containerd/cgroups/v3 v3.1.3 h1:eUNflyMddm18+yrDmZPn3jI7C5hJ9ahABE5q6
github.com/containerd/cgroups/v3 v3.1.3/go.mod h1:PKZ2AcWmSBsY/tJUVhtS/rluX0b1uq1GmPO1ElCmbOw=
github.com/containerd/containerd/api v1.11.1 h1:h8nfoDW9+fNsC/9TwiAHj8B1GzXKtR4eFtkhi/X5RLU=
github.com/containerd/containerd/api v1.11.1/go.mod h1:CaQFRu+N1MtbgL6JDOJLUB1hCKESU1lD6MuTJhgtdlw=
github.com/containerd/containerd/v2 v2.3.5 h1:9MYlI81gUcOZ0WsCkSMtvOU7rTR3hqAoa2eCzhoLlkA=
github.com/containerd/containerd/v2 v2.3.5/go.mod h1:RXDyLPaI3zoO7dFdAW9/54W4cix+z3A6larieufC9mg=
github.com/containerd/containerd/v2 v2.3.6 h1:huoqZXaDW1x1kgPvFC4rMwxv92vUJnqgq7I5b1rSddM=
github.com/containerd/containerd/v2 v2.3.6/go.mod h1:qaUSWKk9EchqbudSbhv1MxI8dXKGYHLpZtMQdkEkVZ4=
github.com/containerd/continuity v0.5.0 h1:7a85HZpCSs+1Zps0Ee3DPSuAWY+0SJM1JNM51nlEVDg=
github.com/containerd/continuity v0.5.0/go.mod h1:/lNJvtJKUQStBzpVQ1+rasXO1LAWtUQssk28EZvJ3nE=
github.com/containerd/errdefs v1.0.0 h1:tg5yIfIlQIrxYtu9ajqY42W3lpS19XqdxRQeEwYG8PI=
@@ -348,8 +350,8 @@ github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDD
github.com/dsnet/compress v0.0.2-0.20230904184137-39efe44ab707 h1:2tV76y6Q9BB+NEBasnqvs7e49aEBFI8ejC89PSnWH+4=
github.com/dsnet/compress v0.0.2-0.20230904184137-39efe44ab707/go.mod h1:qssHWj60/X5sZFNxpG4HBPDHVqxNm4DfnCKgrbZOT+s=
github.com/dsnet/golib v0.0.0-20171103203638-1ea166775780/go.mod h1:Lj+Z9rebOhdfkVLjJ8T6VcRQv3SXugXy999NBtR9aFY=
github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY=
github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
github.com/dustin/go-humanize v1.1.0 h1:dbKTrvD0klcbBV/h4AWJdMuZogJACoMlvWIWZ5b2xWg=
github.com/dustin/go-humanize v1.1.0/go.mod h1:hc1CvRkJMsgxqjmjMQF3QNRAZBwY8AXBAzKYoSX9sFI=
github.com/elazarl/goproxy v1.7.2 h1:Y2o6urb7Eule09PjlhQRGNsqRfPmYI3KKQLFpCAV3+o=
github.com/elazarl/goproxy v1.7.2/go.mod h1:82vkLNir0ALaW14Rc399OTTjyNREgmdL2cVoIbS6XaE=
github.com/elliotchance/phpserialize v1.4.0 h1:cAp/9+KSnEbUC8oYCE32n2n84BeW8HOY3HMDI8hG2OY=
@@ -595,8 +597,8 @@ github.com/hashicorp/golang-lru v0.5.4/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uG
github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k=
github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM=
github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ=
github.com/hashicorp/hcl/v2 v2.24.0 h1:2QJdZ454DSsYGoaE6QheQZjtKZSUs9Nh2izTWiwQxvE=
github.com/hashicorp/hcl/v2 v2.24.0/go.mod h1:oGoO1FIQYfn/AgyOhlg9qLC6/nOJPX3qGbkZpYAcqfM=
github.com/hashicorp/hcl/v2 v2.25.0 h1:HmmQVYRny4MaBo4b20TjmL46wyuUxpnMWkPZ4+NTbWk=
github.com/hashicorp/hcl/v2 v2.25.0/go.mod h1:vR+FKETxoZAmRlHgFfKmuqivj+C4Izm/c66XkmZ3r7M=
github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO+LraFDTW64=
github.com/hashicorp/mdns v1.0.1/go.mod h1:4gW7WsVCke5TE7EPeYliwHlRUyBtfCwuFwuMg2DmyNY=
github.com/hashicorp/mdns v1.0.4/go.mod h1:mtBihi+LeNXGtG8L9dX59gAEa12BDtBQSp4v/YAJqrc=
@@ -668,8 +670,8 @@ github.com/lucasb-eyer/go-colorful v1.3.0 h1:2/yBRLdWBZKrf7gB40FoiKfAWYQ0lqNcbuQ
github.com/lucasb-eyer/go-colorful v1.3.0/go.mod h1:R4dSotOR9KMtayYi1e77YzuveK+i7ruzyGqttikkLy0=
github.com/lyft/protoc-gen-star v0.5.3/go.mod h1:V0xaHgaf5oCCqmcxYcWiDfTiKsZsRc87/1qhoTACD8w=
github.com/magiconair/properties v1.8.5/go.mod h1:y3VJvCyxH9uVvJTWEGAELF3aiYNyPKd5NZ3oSwXrF60=
github.com/magiconair/properties v1.18.11 h1:j5ozYZl0zCjG7ahMDH0GWIobOvvUzT0BdAguG0ViKy0=
github.com/magiconair/properties v1.18.11/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3vdS329zhj2hYo0=
github.com/magiconair/properties v1.18.12 h1:sT9zQpvTB3B4gzrX0tmZNTEaGyg8Zw55MFYRE32Mr9I=
github.com/magiconair/properties v1.18.12/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3vdS329zhj2hYo0=
github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc=
github.com/maruel/natural v1.3.0 h1:VsmCsBmEyrR46RomtgHs5hbKADGRVtliHTyCOLFBpsg=
github.com/maruel/natural v1.3.0/go.mod h1:v+Rfd79xlw1AgVBjbO0BEQmptqb5HvL/k9GRHB7ZKEg=
@@ -756,16 +758,16 @@ github.com/nix-community/go-nix v0.0.0-20250101154619-4bdde671e0a1 h1:kpt9ZfKcm+
github.com/nix-community/go-nix v0.0.0-20250101154619-4bdde671e0a1/go.mod h1:qgCw4bBKZX8qMgGeEZzGFVT3notl42dBjNqO2jut0M0=
github.com/nsf/jsondiff v0.0.0-20210926074059-1e845ec5d249 h1:NHrXEjTNQY7P0Zfx1aMrNhpgxHmow66XQtm0aQLY0AE=
github.com/nsf/jsondiff v0.0.0-20210926074059-1e845ec5d249/go.mod h1:mpRZBD8SJ55OIICQ3iWH0Yz3cjzA61JdqMLoWXeB2+8=
github.com/nwaples/rardecode/v2 v2.2.0 h1:4ufPGHiNe1rYJxYfehALLjup4Ls3ck42CWwjKiOqu0A=
github.com/nwaples/rardecode/v2 v2.2.0/go.mod h1:7uz379lSxPe6j9nvzxUZ+n7mnJNgjsRNb6IbvGVHRmw=
github.com/nwaples/rardecode/v2 v2.4.1 h1:F7zNW2LdAuuBThHWXQaiFUGVD/sef299NfWSB1nHAl4=
github.com/nwaples/rardecode/v2 v2.4.1/go.mod h1:7uz379lSxPe6j9nvzxUZ+n7mnJNgjsRNb6IbvGVHRmw=
github.com/olekukonko/cat v0.0.0-20250911104152-50322a0618f6 h1:zrbMGy9YXpIeTnGj4EljqMiZsIcE09mmF8XsD5AYOJc=
github.com/olekukonko/cat v0.0.0-20250911104152-50322a0618f6/go.mod h1:rEKTHC9roVVicUIfZK7DYrdIoM0EOr8mK1Hj5s3JjH0=
github.com/olekukonko/errors v1.2.0 h1:10Zcn4GeV59t/EGqJc8fUjtFT/FuUh5bTMzZ1XwmCRo=
github.com/olekukonko/errors v1.2.0/go.mod h1:ppzxA5jBKcO1vIpCXQ9ZqgDh8iwODz6OXIGKU8r5m4Y=
github.com/olekukonko/ll v0.1.6 h1:lGVTHO+Qc4Qm+fce/2h2m5y9LvqaW+DCN7xW9hsU3uA=
github.com/olekukonko/ll v0.1.6/go.mod h1:NVUmjBb/aCtUpjKk75BhWrOlARz3dqsM+OtszpY4o88=
github.com/olekukonko/tablewriter v1.1.4 h1:ORUMI3dXbMnRlRggJX3+q7OzQFDdvgbN9nVWj1drm6I=
github.com/olekukonko/tablewriter v1.1.4/go.mod h1:+kedxuyTtgoZLwif3P1Em4hARJs+mVnzKxmsCL/C5RY=
github.com/olekukonko/tablewriter v1.1.5 h1:4LoZSfMySpMQY3PT8RWJsJeuEuMIoo9xGRgvmqjg6IQ=
github.com/olekukonko/tablewriter v1.1.5/go.mod h1:+kedxuyTtgoZLwif3P1Em4hARJs+mVnzKxmsCL/C5RY=
github.com/onsi/gomega v1.34.1 h1:EUMJIKUjM8sKjYbtxQI9A4z2o+rruxnzNvpknOXie6k=
github.com/onsi/gomega v1.34.1/go.mod h1:kU1QgUvBDLXBJq618Xvm2LUX6rSAfRaFRTcdOeDLwwY=
github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U=
@@ -976,8 +978,8 @@ github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9de
github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
github.com/zclconf/go-cty v1.16.3 h1:osr++gw2T61A8KVYHoQiFbFd1Lh3JOCXc/jFLJXKTxk=
github.com/zclconf/go-cty v1.16.3/go.mod h1:VvMs5i0vgZdhYawQNq5kePSpLAoz8u1xvZgrPIxfnZE=
github.com/zclconf/go-cty v1.19.0 h1:IV8WdqYZc2c5rLX9bEoLNXKojBAp0MZPBHMIrCoa/s4=
github.com/zclconf/go-cty v1.19.0/go.mod h1:12W89jGn3JCOIQi7infWr9m80rOkb5RNYJqXMZcN4c8=
github.com/zclconf/go-cty-debug v0.0.0-20240509010212-0d6042c53940 h1:4r45xpDWB6ZMSMNJFMOjqrGHynW3DIBuR2H9j0ug+Mo=
github.com/zclconf/go-cty-debug v0.0.0-20240509010212-0d6042c53940/go.mod h1:CmBdvvj3nqzfzJ6nTCIwDTPZ56aVGvDrmztiO5g3qrM=
github.com/zyedidia/generic v1.2.2-0.20230320175451-4410d2372cb1 h1:V+UsotZpAVvfj3X/LMoEytoLzSiP6Lg0F7wdVyu9gGg=
@@ -1526,8 +1528,8 @@ modernc.org/gc/v3 v3.1.5 h1:21ldfPfRYE31Tb7B3mwAK8gy1AxP4+dKjrOQPfqakoc=
modernc.org/gc/v3 v3.1.5/go.mod h1:HFK/6AGESC7Ex+EZJhJ2Gni6cTaYpSMmU/cT9RmlfYY=
modernc.org/goabi0 v0.2.0 h1:HvEowk7LxcPd0eq6mVOAEMai46V+i7Jrj13t4AzuNks=
modernc.org/goabi0 v0.2.0/go.mod h1:CEFRnnJhKvWT1c1JTI3Avm+tgOWbkOu5oPA8eH8LnMI=
modernc.org/libc v1.75.6 h1:yKk8qo+Di4gkmvRboK8ocCqH22FiUCR6jRy2OwtCRus=
modernc.org/libc v1.75.6/go.mod h1:bO5o2ztHxBb2rjz0PgdHN0sSMw57CgxGFLZ3Qd/QpVQ=
modernc.org/libc v1.75.7 h1:o3DTP9/0p9pKmY2WCKQaySW6wIiZhNM7wc2lUoyhfew=
modernc.org/libc v1.75.7/go.mod h1:bO5o2ztHxBb2rjz0PgdHN0sSMw57CgxGFLZ3Qd/QpVQ=
modernc.org/mathutil v1.7.1 h1:GCZVGXdaN8gTqB1Mf/usp1Y/hSqgI2vAGGP4jZMCxOU=
modernc.org/mathutil v1.7.1/go.mod h1:4p5IwJITfppl0G4sUEDtCr4DthTaT47/N3aT6MhfgJg=
modernc.org/memory v1.12.1 h1:nFMiWrpStgZczNl6XI9GnIk/rWhYIyHGUaR04pGbp9g=
@@ -1536,8 +1538,8 @@ modernc.org/opt v0.2.0 h1:tGyef5ApycA7FSEOMraay9SaTk5zmbx7Tu+cJs4QKZg=
modernc.org/opt v0.2.0/go.mod h1:03fq9lsNfvkYSfxrfUhZCWPk1lm4cq4N+Bh//bEtgns=
modernc.org/sortutil v1.2.1 h1:+xyoGf15mM3NMlPDnFqrteY07klSFxLElE2PVuWIJ7w=
modernc.org/sortutil v1.2.1/go.mod h1:7ZI3a3REbai7gzCLcotuw9AC4VZVpYMjDzETGsSMqJE=
modernc.org/sqlite v1.58.0 h1:38u40/bwkfM7f0Myhosl+SEMltSDxnGdQf8o6Kjmys0=
modernc.org/sqlite v1.58.0/go.mod h1:rsD2CckafgObKC4DhBlGBf+RiHxkc3hINGt1Xw32tVY=
modernc.org/sqlite v1.59.0 h1:X1es1GpqBlS/5T+vbM4HLUdaa8OtQx468DF2vrx+38A=
modernc.org/sqlite v1.59.0/go.mod h1:+paeT2A3iPRHkQDwG7oA6Tk0zQd5woMEI8q7orfry8k=
modernc.org/strutil v1.2.1 h1:UneZBkQA+DX2Rp35KcM69cSsNES9ly8mQWD71HKlOA0=
modernc.org/strutil v1.2.1/go.mod h1:EHkiggD70koQxjVdSBM3JKM7k6L0FbGE5eymy9i3B9A=
modernc.org/token v1.1.0 h1:Xl7Ap9dKaEs5kLoOQeQmPWevfnk/DM5qcLcYlA8ys6Y=
internal/capabilities/capabilities.go
+1 -1
View File
@@ -9,7 +9,7 @@ import (
"sort"
"github.com/scylladb/go-set/strset"
"gopkg.in/yaml.v3"
"go.yaml.in/yaml/v3"
"github.com/anchore/syft/internal/task"
)
internal/capabilities/generate/io.go
+2 -2
View File
@@ -1,4 +1,4 @@
// this file handles YAML file reading and writing with comment preservation, using gopkg.in/yaml.v3's node tree to maintain all existing comments during regeneration.
// this file handles YAML file reading and writing with comment preservation, using go.yaml.in/yaml/v3's node tree to maintain all existing comments during regeneration.
package main
import (
@@ -6,7 +6,7 @@ import (
"os"
"regexp"
"gopkg.in/yaml.v3"
"go.yaml.in/yaml/v3"
"github.com/anchore/syft/internal/capabilities"
"github.com/anchore/syft/internal/capabilities/internal"
internal/capabilities/generate/io_test.go
+1 -1
View File
@@ -4,7 +4,7 @@ import (
"testing"
"github.com/stretchr/testify/require"
"gopkg.in/yaml.v3"
"go.yaml.in/yaml/v3"
)
func TestFindFieldValue(t *testing.T) {
internal/capabilities/internal/load_capabilities.go
+1 -1
View File
@@ -5,7 +5,7 @@ import (
"os"
"path/filepath"
"gopkg.in/yaml.v3"
"go.yaml.in/yaml/v3"
"github.com/anchore/syft/internal/capabilities"
)
internal/capabilities/model_test.go
+1 -1
View File
@@ -5,7 +5,7 @@ import (
"github.com/google/go-cmp/cmp"
"github.com/stretchr/testify/require"
"gopkg.in/yaml.v3"
"go.yaml.in/yaml/v3"
)
func TestConfigSchemaExtensions(t *testing.T) {
internal/line_scanner.go
+19
View File
@@ -0,0 +1,19 @@
package internal
import (
"bufio"
"io"
)
// maxScannedLineSize is the longest line NewLineScanner will return before Scan stops with bufio.ErrTooLong.
// The buffer only grows to this size when a line needs it.
const maxScannedLineSize = 1024 * 1024
// NewLineScanner returns a bufio.Scanner that accepts lines up to 1MB rather than the 64KB default, since
// package metadata files carry long single-line fields (descriptions, license text, dependency lists). Callers
// must still check Err() after scanning.
func NewLineScanner(r io.Reader) *bufio.Scanner {
s := bufio.NewScanner(r)
s.Buffer(nil, maxScannedLineSize)
return s
}
internal/line_scanner_test.go
+24
View File
@@ -0,0 +1,24 @@
package internal
import (
"bufio"
"strings"
"testing"
"github.com/stretchr/testify/require"
)
func TestNewLineScanner(t *testing.T) {
long := strings.Repeat("a", 100*1024)
s := NewLineScanner(strings.NewReader(long + "\nb\n"))
require.True(t, s.Scan())
require.Equal(t, long, s.Text())
require.True(t, s.Scan())
require.Equal(t, "b", s.Text())
require.False(t, s.Scan())
require.NoError(t, s.Err())
s = NewLineScanner(strings.NewReader(strings.Repeat("a", maxScannedLineSize+1)))
require.False(t, s.Scan())
require.ErrorIs(t, s.Err(), bufio.ErrTooLong)
}
internal/os/feature_detection.go
+4 -6
View File
@@ -6,6 +6,7 @@ import (
"errors"
"fmt"
"regexp"
"slices"
"github.com/anchore/syft/internal"
"github.com/anchore/syft/internal/log"
@@ -191,12 +192,9 @@ func hasInstalledESMPackage(packages *pkg.Collection) bool {
if packages == nil {
return false
}
for p := range packages.Enumerate(pkg.DebPkg) {
if esmVersionPattern.MatchString(p.Version) {
return true
}
}
return false
return slices.ContainsFunc(packages.Sorted(pkg.DebPkg), func(p pkg.Package) bool {
return esmVersionPattern.MatchString(p.Version)
})
}
func fileMatchesAny(resolver file.Resolver, location file.Location, patterns ...*regexp.Regexp) (bool, error) {
internal/os/feature_detection_test.go
+28
View File
@@ -8,6 +8,7 @@ import (
"path/filepath"
"slices"
"testing"
"time"
"github.com/stretchr/testify/require"
@@ -232,6 +233,33 @@ func Test_findUbuntuFeatures(t *testing.T) {
}
}
// returning early from Collection.Enumerate leaks its goroutine and read lock, so any later write blocks forever.
func Test_findUbuntuFeatures_releasesPackageCollectionLock(t *testing.T) {
resolver := fixtureResolverForDir(t, "testdata/ubuntu_plain")
s := sbom.SBOM{}
s.Artifacts.LinuxDistribution = &linux.Release{ID: "ubuntu"}
s.Artifacts.Packages = pkg.NewCollection(
pkg.Package{Name: "openssl", Version: "1.1.1f-1ubuntu2.19+esm1", Type: pkg.DebPkg},
pkg.Package{Name: "libcap2", Version: "1:2.32-1ubuntu0.1~esm1", Type: pkg.DebPkg},
)
require.NoError(t, os.DetectFeatures(context.Background(), resolver, sbomsync.NewBuilder(&s)))
require.True(t, s.Artifacts.LinuxDistribution.ExtendedSupport)
added := make(chan struct{})
go func() {
s.Artifacts.Packages.Add(pkg.Package{Name: "zlib1g", Version: "1:1.2.11.dfsg-2ubuntu1", Type: pkg.DebPkg})
close(added)
}()
select {
case <-added:
case <-time.After(5 * time.Second):
t.Fatal("package collection is still read-locked after feature detection")
}
}
// a read error in the apt-source signal must not suppress the ubuntu-advantage status signal; the three
// signals are meant to be independent fallbacks.
func Test_findUbuntuFeatures_signalErrorDoesNotSuppressOthers(t *testing.T) {
internal/packagemetadata/untyped_fields_test.go
+74
View File
@@ -0,0 +1,74 @@
package packagemetadata
import (
"reflect"
"strings"
"testing"
)
// untypedFieldAllowlist holds package metadata fields that are allowed to carry untyped (interface) values.
// Values in these fields come straight from whatever a parser decoded, so they are not shaped by syft
// and can hold anything an encoder or hasher chokes on. Every entry is part of existing SBOM output and
// cannot be retyped without breaking it; new entries need a reason and a review.
var untypedFieldAllowlist = map[string]string{
"GGUFFileHeader.RemainingKeyValues": "raw GGUF header key-values; retyping changes the header output and metadataHash",
}
// TestNoUntypedMetadataFields fails when a package metadata type gains a field typed any, []any,
// map[string]any (or any other interface) that is not on the allowlist.
func TestNoUntypedMetadataFields(t *testing.T) {
var hits []string
for _, ty := range AllTypes() {
rt := reflect.TypeOf(ty)
collectUntyped(rt, rt.Name(), map[reflect.Type]bool{}, &hits)
}
seen := map[string]bool{}
for _, h := range hits {
seen[h] = true
if _, ok := untypedFieldAllowlist[h]; !ok {
t.Errorf("untyped field %s: use a concrete type, or add it to untypedFieldAllowlist with a reason", h)
}
}
for k := range untypedFieldAllowlist {
if !seen[k] {
t.Errorf("stale untypedFieldAllowlist entry %s: field no longer exists or is now typed", k)
}
}
}
// collectUntyped records "Struct.Field" for every field whose type reaches an interface, descending
// through pointers, slices, arrays, maps and nested syft-owned structs.
func collectUntyped(rt reflect.Type, path string, visited map[reflect.Type]bool, hits *[]string) {
switch rt.Kind() {
case reflect.Interface:
*hits = append(*hits, path)
case reflect.Pointer, reflect.Slice, reflect.Array:
collectUntyped(rt.Elem(), path, visited, hits)
case reflect.Map:
collectUntyped(rt.Key(), path, visited, hits)
collectUntyped(rt.Elem(), path, visited, hits)
case reflect.Struct:
if visited[rt] {
return
}
if rt.Name() != "" && !strings.HasPrefix(rt.PkgPath(), "github.com/anchore/syft/") {
// a third-party type in the model carries whatever shape the library gives it (the same leak as an
// interface field), so it counts as untyped. The standard library is left alone.
if strings.Contains(strings.Split(rt.PkgPath(), "/")[0], ".") {
*hits = append(*hits, path)
}
return
}
visited[rt] = true
for i := 0; i < rt.NumField(); i++ {
f := rt.Field(i)
// unexported embedded structs still promote their exported fields to the encoders and the hasher. A
// field is only out of reach when it is neither encoded nor hashed by SetID.
if (!f.IsExported() && !f.Anonymous) || (f.Tag.Get("json") == "-" && f.Tag.Get("hash") == "ignore") {
continue
}
collectUntyped(f.Type, rt.Name()+"."+f.Name, visited, hits)
}
}
}
internal/safetensors.go
+7
View File
@@ -0,0 +1,7 @@
package internal
// MaxSafeTensorsHeaderSize caps a safetensors header's JSON (excluding its
// 8-byte length prefix). Shared by the ai cataloger and the OCI model source so
// dir and OCI scans accept the same headers. Real-world shard headers are well
// under 1 MB.
const MaxSafeTensorsHeaderSize = 8 * 1024 * 1024
internal/spdxlicense/license.go
+43
View File
@@ -44,6 +44,7 @@ type LicenseInfo struct {
// LicenseByURL returns the license ID and name for a given URL from the SPDX license list.
// The URL should match one of the URLs in the seeAlso field of an SPDX license.
// The scheme (http:// or https://) is stripped before lookup, so both schemes match.
// opensource.org URLs additionally match regardless of the form used (see normalizeOpenSourceOrgURL).
func LicenseByURL(url string) (LicenseInfo, bool) {
url = strings.TrimSpace(url)
url = stripScheme(url)
@@ -52,9 +53,51 @@ func LicenseByURL(url string) (LicenseInfo, bool) {
ID: id,
}, true
}
if normalized, ok := normalizeOpenSourceOrgURL(url); ok {
if id, exists := openSourceOrgURLToLicense[normalized]; exists {
return LicenseInfo{
ID: id,
}, true
}
}
return LicenseInfo{}, false
}
// openSourceOrgURLToLicense indexes the opensource.org entries of urlToLicense by normalized URL.
var openSourceOrgURLToLicense = buildOpenSourceOrgURLToLicense(urlToLicense)
func buildOpenSourceOrgURLToLicense(urls map[string]string) map[string]string {
index := make(map[string]string)
ambiguous := make(map[string]bool)
for url, id := range urls {
normalized, ok := normalizeOpenSourceOrgURL(url)
if !ok || ambiguous[normalized] {
continue
}
if existing, exists := index[normalized]; exists && existing != id {
// don't guess between licenses that only differ by URL form
delete(index, normalized)
ambiguous[normalized] = true
continue
}
index[normalized] = id
}
return index
}
// normalizeOpenSourceOrgURL reduces a scheme-less opensource.org URL to a canonical form.
// SPDX has changed these URLs over time (e.g. 3.29.0 replaced opensource.org/licenses/MIT and
// opensource.org/license/mit/ with opensource.org/license/MIT), while package metadata in the wild
// still uses the older forms. The www. prefix, /licenses/ vs /license/, case, and trailing slash are ignored.
func normalizeOpenSourceOrgURL(url string) (string, bool) {
url = strings.TrimPrefix(strings.ToLower(url), "www.")
if !strings.HasPrefix(url, "opensource.org/") {
return "", false
}
url = strings.Replace(url, "opensource.org/licenses/", "opensource.org/license/", 1)
return strings.TrimSuffix(url, "/"), true
}
// stripScheme removes http:// or https:// prefix from a URL.
// This allows a single map entry to match both schemes.
func stripScheme(url string) string {
internal/spdxlicense/license_list.go
+254 -199
View File
@@ -2,7 +2,7 @@
// This file was generated using data from https://spdx.org/licenses/licenses.json
package spdxlicense
const Version = "3.28.0"
const Version = "3.29.0"
var licenseIDs = map[string]string{
"0bsd": "0BSD",
@@ -116,6 +116,7 @@ var licenseIDs = map[string]string{
"aswfdigitalassets1.0.0": "ASWF-Digital-Assets-1.0",
"aswfdigitalassets1.1": "ASWF-Digital-Assets-1.1",
"aswfdigitalassets1.1.0": "ASWF-Digital-Assets-1.1",
"atcgame": "atc-game",
"baekmuk": "Baekmuk",
"bahyph": "Bahyph",
"barr": "Barr",
@@ -142,8 +143,11 @@ var licenseIDs = map[string]string{
"briangladman2.0clause": "Brian-Gladman-2-Clause",
"briangladman2clause": "Brian-Gladman-2-Clause",
"briangladman3.0.0clause": "Brian-Gladman-3-Clause",
"briangladman3.0.0clausenoconversion": "Brian-Gladman-3-Clause-no-conversion",
"briangladman3.0clause": "Brian-Gladman-3-Clause",
"briangladman3.0clausenoconversion": "Brian-Gladman-3-Clause-no-conversion",
"briangladman3clause": "Brian-Gladman-3-Clause",
"briangladman3clausenoconversion": "Brian-Gladman-3-Clause-no-conversion",
"bsd1.0.0clause": "BSD-1-Clause",
"bsd1.0clause": "BSD-1-Clause",
"bsd1clause": "BSD-1-Clause",
@@ -154,6 +158,7 @@ var licenseIDs = map[string]string{
"bsd2.0.0clausenetbsd": "BSD-2-Clause",
"bsd2.0.0clausepatent": "BSD-2-Clause-Patent",
"bsd2.0.0clausepkgconfdisclaimer": "BSD-2-Clause-pkgconf-disclaimer",
"bsd2.0.0clauseposunchanged": "BSD-2-Clause-pos-unchanged",
"bsd2.0.0clauseviews": "BSD-2-Clause-Views",
"bsd2.0clause": "BSD-2-Clause",
"bsd2.0clausedarwin": "BSD-2-Clause-Darwin",
@@ -162,6 +167,7 @@ var licenseIDs = map[string]string{
"bsd2.0clausenetbsd": "BSD-2-Clause",
"bsd2.0clausepatent": "BSD-2-Clause-Patent",
"bsd2.0clausepkgconfdisclaimer": "BSD-2-Clause-pkgconf-disclaimer",
"bsd2.0clauseposunchanged": "BSD-2-Clause-pos-unchanged",
"bsd2.0clauseviews": "BSD-2-Clause-Views",
"bsd2clause": "BSD-2-Clause",
"bsd2clausedarwin": "BSD-2-Clause-Darwin",
@@ -170,6 +176,7 @@ var licenseIDs = map[string]string{
"bsd2clausenetbsd": "BSD-2-Clause",
"bsd2clausepatent": "BSD-2-Clause-Patent",
"bsd2clausepkgconfdisclaimer": "BSD-2-Clause-pkgconf-disclaimer",
"bsd2clauseposunchanged": "BSD-2-Clause-pos-unchanged",
"bsd2clauseviews": "BSD-2-Clause-Views",
"bsd3.0.0clause": "BSD-3-Clause",
"bsd3.0.0clauseacpica": "BSD-3-Clause-acpica",
@@ -184,6 +191,7 @@ var licenseIDs = map[string]string{
"bsd3.0.0clausenonuclearlicense2014": "BSD-3-Clause-No-Nuclear-License-2014",
"bsd3.0.0clausenonuclearwarranty": "BSD-3-Clause-No-Nuclear-Warranty",
"bsd3.0.0clauseopenmpi": "BSD-3-Clause-Open-MPI",
"bsd3.0.0clauseopenwebui": "BSD-3-Clause-OpenWebUI",
"bsd3.0.0clausesun": "BSD-3-Clause-Sun",
"bsd3.0.0clausetso": "BSD-3-Clause-Tso",
"bsd3.0clause": "BSD-3-Clause",
@@ -199,6 +207,7 @@ var licenseIDs = map[string]string{
"bsd3.0clausenonuclearlicense2014": "BSD-3-Clause-No-Nuclear-License-2014",
"bsd3.0clausenonuclearwarranty": "BSD-3-Clause-No-Nuclear-Warranty",
"bsd3.0clauseopenmpi": "BSD-3-Clause-Open-MPI",
"bsd3.0clauseopenwebui": "BSD-3-Clause-OpenWebUI",
"bsd3.0clausesun": "BSD-3-Clause-Sun",
"bsd3.0clausetso": "BSD-3-Clause-Tso",
"bsd3clause": "BSD-3-Clause",
@@ -214,6 +223,7 @@ var licenseIDs = map[string]string{
"bsd3clausenonuclearlicense2014": "BSD-3-Clause-No-Nuclear-License-2014",
"bsd3clausenonuclearwarranty": "BSD-3-Clause-No-Nuclear-Warranty",
"bsd3clauseopenmpi": "BSD-3-Clause-Open-MPI",
"bsd3clauseopenwebui": "BSD-3-Clause-OpenWebUI",
"bsd3clausesun": "BSD-3-Clause-Sun",
"bsd3clausetso": "BSD-3-Clause-Tso",
"bsd4.0.0clause": "BSD-4-Clause",
@@ -232,12 +242,15 @@ var licenseIDs = map[string]string{
"bsd4reno": "BSD-4.3RENO",
"bsd4tahoe": "BSD-4.3TAHOE",
"bsdadvertisingacknowledgement": "BSD-Advertising-Acknowledgement",
"bsdasktoendorse": "BSD-ask-to-endorse",
"bsdattributionhpnddisclaimer": "BSD-Attribution-HPND-disclaimer",
"bsdinfernonettverk": "BSD-Inferno-Nettverk",
"bsdmarkmodifications": "BSD-Mark-Modifications",
"bsdprotection": "BSD-Protection",
"bsdsourcealtgpl": "BSD-Source-alt-GPL",
"bsdsourcebeginningfile": "BSD-Source-beginning-file",
"bsdsourcecode": "BSD-Source-Code",
"bsdsourcecodenodisclaimer": "BSD-Source-Code-no-disclaimer",
"bsdsystemics": "BSD-Systemics",
"bsdsystemicsw3.0.0works": "BSD-Systemics-W3Works",
"bsdsystemicsw3.0works": "BSD-Systemics-W3Works",
@@ -246,6 +259,7 @@ var licenseIDs = map[string]string{
"bsl1.0": "BSL-1.0",
"bsl1.0.0": "BSL-1.0",
"buddy": "Buddy",
"bugroff": "Bugroff",
"busl1": "BUSL-1.1",
"busl1.1": "BUSL-1.1",
"busl1.1.0": "BUSL-1.1",
@@ -316,8 +330,11 @@ var licenseIDs = map[string]string{
"ccbync3.0": "CC-BY-NC-3.0",
"ccbync3.0.0": "CC-BY-NC-3.0",
"ccbync3.0.0de": "CC-BY-NC-3.0-DE",
"ccbync3.0.0igo": "CC-BY-NC-3.0-IGO",
"ccbync3.0de": "CC-BY-NC-3.0-DE",
"ccbync3.0igo": "CC-BY-NC-3.0-IGO",
"ccbync3de": "CC-BY-NC-3.0-DE",
"ccbync3igo": "CC-BY-NC-3.0-IGO",
"ccbync4": "CC-BY-NC-4.0",
"ccbync4.0": "CC-BY-NC-4.0",
"ccbync4.0.0": "CC-BY-NC-4.0",
@@ -592,6 +609,7 @@ var licenseIDs = map[string]string{
"fair": "Fair",
"fbm": "FBM",
"fdkaac": "FDK-AAC",
"fdkmpegh": "FDK-MPEG-H",
"fergusontwofish": "Ferguson-Twofish",
"frameworx1": "Frameworx-1.0",
"frameworx1.0": "Frameworx-1.0",
@@ -767,6 +785,9 @@ var licenseIDs = map[string]string{
"hippocratic2": "Hippocratic-2.1",
"hippocratic2.1": "Hippocratic-2.1",
"hippocratic2.1.0": "Hippocratic-2.1",
"hippocratic3.0.0core": "Hippocratic-3.0-core",
"hippocratic3.0core": "Hippocratic-3.0-core",
"hippocratic3core": "Hippocratic-3.0-core",
"hp1986": "HP-1986",
"hp1986.0": "HP-1986",
"hp1986.0.0": "HP-1986",
@@ -813,6 +834,7 @@ var licenseIDs = map[string]string{
"imlib2": "Imlib2",
"imlib2.0": "Imlib2",
"imlib2.0.0": "Imlib2",
"informatica": "Informatica",
"infozip": "Info-ZIP",
"innernet2": "Inner-Net-2.0",
"innernet2.0": "Inner-Net-2.0",
@@ -1011,6 +1033,9 @@ var licenseIDs = map[string]string{
"mulanpsl2.0.0": "MulanPSL-2.0",
"multics": "Multics",
"mup": "Mup",
"mvt1": "MVT-1.1",
"mvt1.1": "MVT-1.1",
"mvt1.1.0": "MVT-1.1",
"naist2003": "NAIST-2003",
"naist2003.0": "NAIST-2003",
"naist2003.0.0": "NAIST-2003",
@@ -1489,6 +1514,8 @@ var urlToLicense = map[string]string{
"cdla.io/permissive-1-0": "CDLA-Permissive-1.0",
"cdla.io/sharing-1-0": "CDLA-Sharing-1.0",
"celestrak.org/publications/AIAA/2006-6753/faq.php": "SGP4",
"cgit.freebsd.org/src/plain/sys/dev/aic7xxx/aic7xxx_osm.c": "BSD-Source-alt-GPL",
"cgit.freebsd.org/src/tree/usr.bin/sockstat/sockstat.c": "BSD-2-Clause-pos-unchanged",
"cgit.freedesktop.org/libbsd/tree/man/setproctitle.3bsd": "libutil-David-Nugent",
"coil.apotheon.org/plaintext/01.0.txt": "COIL-1.0",
"core.tcl-lang.org/tk/file?name=compat/unistd.h": "HPND-UC",
@@ -1512,6 +1539,7 @@ var urlToLicense = map[string]string{
"creativecommons.org/licenses/by-nc/2.0/legalcode": "CC-BY-NC-2.0",
"creativecommons.org/licenses/by-nc/2.5/legalcode": "CC-BY-NC-2.5",
"creativecommons.org/licenses/by-nc/3.0/de/legalcode": "CC-BY-NC-3.0-DE",
"creativecommons.org/licenses/by-nc/3.0/igo/legalcode": "CC-BY-NC-3.0-IGO",
"creativecommons.org/licenses/by-nc/3.0/legalcode": "CC-BY-NC-3.0",
"creativecommons.org/licenses/by-nc/4.0/legalcode": "CC-BY-NC-4.0",
"creativecommons.org/licenses/by-nd-nc/1.0/legalcode": "CC-BY-NC-ND-1.0",
@@ -1564,6 +1592,7 @@ var urlToLicense = map[string]string{
"directory.fsf.org/wiki/License:Fdk": "FDK-AAC",
"doc.qt.io/archives/3.3/license.html": "QPL-1.0",
"doc.qt.nokia.com/3.3/license.html": "QPL-1.0",
"docs.mvt.re/en/latest/license/": "MVT-1.1",
"docs.omniverse.nvidia.com/install-guide/latest/common/licenses.html": "AML-glslang",
"docs.oracle.com/cd/E81115_01/html/E81116/licenses.html": "man2html",
"docs.python.org/3/license.html": "Python-2.0.1",
@@ -1669,6 +1698,7 @@ var urlToLicense = map[string]string{
"fedoraproject.org/wiki/Licensing:Mplus?rd=Licensing/mplus": "mplus",
"fedoraproject.org/wiki/Licensing:ThorPublicLicense": "TPL-1.0",
"firstdonoharm.dev/version/2/1/license.html": "Hippocratic-2.1",
"firstdonoharm.dev/version/3/0/core.txt": "Hippocratic-3.0-core",
"forge.gouv.qc.ca/licence/fr/liliq-v1-1/": "LiLiQ-P-1.1",
"forge.gouv.qc.ca/licence/liliq-p": "LiLiQ-R-1.1",
"forge.gouv.qc.ca/licence/liliq-p/": "LiLiQ-P-1.1",
@@ -1686,68 +1716,70 @@ var urlToLicense = map[string]string{
"ftp://ftp.tin.org/pub/news/utils/newsx/newsx-1.6.tar.gz": "Zeeff",
"gianluca.dellavedova.org/2011/01/03/clarified-artistic-license/": "ClArtistic",
"git.denx.de/?p=u-boot.git;a=blob;f=arch/powerpc/cpu/ppc4xx/miiphy.c;h=297155fdafa064b955e53e9832de93bfb0cfb85b;hb=9fab4bf4cc077c21e43941866f3f2c196f28670d": "IBM-pibs",
"git.enlightenment.org/legacy/imlib2.git/tree/COPYING": "Imlib2",
"git.kernel.org/pub/scm/docs/man-pages/man-pages.git/tree/man2/getcpu.2#n4": "Linux-man-pages-1-para",
"git.kernel.org/pub/scm/docs/man-pages/man-pages.git/tree/man2/migrate_pages.2#n8": "Linux-man-pages-copyleft-2-para",
"git.kernel.org/pub/scm/docs/man-pages/man-pages.git/tree/man2/move_pages.2#n5": "Linux-man-pages-copyleft-2-para",
"git.kernel.org/pub/scm/docs/man-pages/man-pages.git/tree/man2/set_mempolicy.2#n5": "Linux-man-pages-copyleft-var",
"git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/tree/drivers/infiniband/core/sa.h": "Linux-OpenIB",
"git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/tree/net/sunrpc/auth_gss/gss_generic_token.c?h=v4.19": "HPND-sell-variant",
"git.openldap.org/openldap/openldap/-/blob/master/COPYRIGHT#L55-63": "BSD-4.3RENO",
"git.openldap.org/openldap/openldap/-/blob/master/COPYRIGHT?ref_type=heads#L39-51": "Furuseth",
"git.openldap.org/openldap/openldap/-/blob/master/libraries/libldap/os-local.c?ref_type=heads#L19-23": "PADL",
"git.savannah.gnu.org/cgit/freetype/freetype2.git/tree/docs/FTL.TXT": "FTL",
"git.savannah.gnu.org/cgit/gcl.git/tree/gcl/lsp/gcl_loop.lsp?h=Version_2_6_13pre": "LOOP",
"git.savannah.gnu.org/cgit/gnulib.git/tree/modules/COPYING?id=7b08932179d0d6b017f7df01a2ddf6e096b038e3": "FSFULLRSD",
"git.savannah.gnu.org/cgit/indent.git/tree/doc/indent.texi?id=a74c6b4ee49397cf330b333da1042bffa60ed14f#n1788": "BSD-4.3TAHOE",
"git.savannah.gnu.org/cgit/indent.git/tree/doc/indent.texi?id=a74c6b4ee49397cf330b333da1042bffa60ed14f#n74": "Latex2e-translated-notice",
"git.savannah.gnu.org/cgit/kazlib.git/tree/except.c?id=0062df360c2d17d57f6af19b0e444c51feb99036": "Kazlib",
"git.savannah.gnu.org/cgit/wget.git/tree/util/trunc.c?h=v1.21.3&id=40747a11e44ced5a8ac628a41f879ced3e2ebce9#n6": "FSFAP-no-warranty-disclaimer",
"git.sr.ht/~nabijaczleweli/ossp-var": "OSSP",
"github.com/389ds/389-ds-base/blob/main/ldap/include/sysexits-compat.h#L15": "BSD-4.3TAHOE",
"github.com/AcademySoftwareFoundation/foundation/blob/main/digital_assets/aswf_digital_assets_license_v1.0.txt": "ASWF-Digital-Assets-1.0",
"github.com/AcademySoftwareFoundation/foundation/blob/main/digital_assets/aswf_digital_assets_license_v1.1.txt": "ASWF-Digital-Assets-1.1",
"github.com/AdaCore/gnatcoll-core/blob/master/docs/index.rst": "AdaCore-doc",
"github.com/AdaCore/gnatcoll-db/blob/master/docs/index.rst": "AdaCore-doc",
"github.com/AdaCore/xmlada/blob/master/docs/index.rst": "AdaCore-doc",
"github.com/BuildCraft/BuildCraft/blob/623d323b1868712f29f4a8b0979a02e8d1835131/LICENSE": "MMPL-1.0.1",
"github.com/CommunitySpecification/1.0/blob/master/1._Community_Specification_License-v1.md": "Community-Spec-1.0",
"github.com/CorsixTH/deps/blob/fd339a9f526d1d9c9f01ccf39e438a015da50035/licences/libgsm.txt": "TU-Berlin-2.0",
"github.com/Cyan4973/xxHash/blob/dev/doc/xxhash_spec.md": "LPD-document",
"github.com/DISIC/politique-de-contribution-open-source/blob/master/LICENSE.pdf": "etalab-2.0",
"github.com/Distrotech/hdparm/blob/4517550db29a91420fb2b020349523b1b4512df2/LICENSE.TXT": "hdparm",
"github.com/Distrotech/transfig/blob/master/transfig/transfig.c": "Xfig",
"github.com/DrItanium/maya/blob/master/LICENSE.CLIPS": "Clips",
"github.com/Dual-Life/mime-base64/blob/master/Base64.xs#L12": "metamail",
"github.com/EthicalSource/hippocratic-license/blob/58c0e646d64ff6fbee275bfe2b9492f914e3ab2a/LICENSE.txt": "Hippocratic-2.1",
"github.com/FreeRADIUS/freeradius-client/blob/master/COPYRIGHT#L32": "HPND-Fenneberg-Livingston",
"github.com/GNOME/gcr/blob/master/docs/COPYING": "GCR-docs",
"github.com/GNOME/libxml2/blob/master/dict.c": "ISC-Veillard",
"github.com/HDFGroup/hdf5/?tab=License-1-ov-file#readme": "HDF5",
"github.com/KhronosGroup/SPIRV-Cross/blob/main/LICENSES/LicenseRef-KhronosFreeUse.txt": "MIT-Khronos-old",
"github.com/KhronosGroup/glslang/blob/main/LICENSE.txt#L949": "AML-glslang",
"github.com/MariaDB/server/blob/11.6/libmysqld/lib_sql.cc": "Boehm-GC-without-fee",
"github.com/Neo23x0/sigma/blob/master/LICENSE.Detection.Rules.md": "DRL-1.0",
"github.com/OGRECave/ogre/blob/master/OgreMain/include/OgreSingleton.h#L28C3-L35C46": "Game-Programming-Gems",
"github.com/Perl/perl5/blob/blead/util.c#L6136": "Martin-Birgmeier",
"github.com/PixarAnimationStudios/OpenSubdiv/blob/v3_5_0/opensubdiv/version.cpp#L2-L22": "Pixar",
"github.com/PixarAnimationStudios/OpenSubdiv/raw/v3_5_0/LICENSE.txt": "Pixar",
"github.com/RTimothyEdwards/magic/blob/master/LICENSE": "HPND-UC-export-US",
"github.com/SELinuxProject/selinux/blob/master/libselinux/LICENSE": "libselinux-1.0",
"github.com/SWI-Prolog/packages-clib/blob/master/sha1/brg_endian.h": "Brian-Gladman-3-Clause",
"github.com/SWI-Prolog/packages-xpce/blob/161a40cd82004f731ba48024f9d30af388a7edf5/src/img/gifwrite.c#L21-L26": "FBM",
"github.com/SWI-Prolog/swipl-devel/blob/master/src/os/dtoa.c": "dtoa",
"github.com/SigmaHQ/Detection-Rule-License/blob/6ec7fbde6101d101b5b5d1fcb8f9b69fbc76c04a/LICENSE.Detection.Rules.md": "DRL-1.1",
"github.com/Slicer/Slicer/blob/main/License.txt": "3D-Slicer-1.0",
"github.com/Unidata/UDUNITS-2/blob/master/COPYRIGHT": "UCAR",
"github.com/acpica/acpica/blob/master/source/common/acfileio.c#L119": "BSD-3-Clause-acpica",
"github.com/adaptivecomputing/torque/blob/master/PBS_License.txt": "OpenPBS-2.3",
"github.com/adversary-org/wtfnmf/raw/3f2cd8235a64350a57a51b9739715edaea63ec1a/COPYING.WTFNMFPL-utf8": "WTFNMFPL",
"github.com/adversary-org/wtfnmf/raw/refs/tags/1.0/COPYING.WTFNMFPL": "WTFNMFPL",
"github.com/ap/Text-Tabs/blob/master/lib.modern/Text/Tabs.pm#L148": "TTWL",
"github.com/apache/apr/blob/trunk/LICENSE#L298C6-L298C29": "Caldera-no-preamble",
"github.com/audacious-media-player/audacious/blob/master/COPYING": "BSD-2-Clause-pkgconf-disclaimer",
"github.com/audacious-media-player/audacious/blob/master/src/audacious/main.cc": "BSD-2-Clause-pkgconf-disclaimer",
"git.enlightenment.org/legacy/imlib2.git/tree/COPYING": "Imlib2",
"git.kernel.org/pub/scm/docs/man-pages/man-pages.git/tree/man2/getcpu.2#n4": "Linux-man-pages-1-para",
"git.kernel.org/pub/scm/docs/man-pages/man-pages.git/tree/man2/migrate_pages.2#n8": "Linux-man-pages-copyleft-2-para",
"git.kernel.org/pub/scm/docs/man-pages/man-pages.git/tree/man2/move_pages.2#n5": "Linux-man-pages-copyleft-2-para",
"git.kernel.org/pub/scm/docs/man-pages/man-pages.git/tree/man2/set_mempolicy.2#n5": "Linux-man-pages-copyleft-var",
"git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/tree/drivers/infiniband/core/sa.h": "Linux-OpenIB",
"git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/tree/net/sunrpc/auth_gss/gss_generic_token.c?h=v4.19": "HPND-sell-variant",
"git.openldap.org/openldap/openldap/-/blob/master/COPYRIGHT#L55-63": "BSD-4.3RENO",
"git.openldap.org/openldap/openldap/-/blob/master/COPYRIGHT?ref_type=heads#L39-51": "Furuseth",
"git.openldap.org/openldap/openldap/-/blob/master/libraries/libldap/os-local.c?ref_type=heads#L19-23": "PADL",
"git.savannah.gnu.org/cgit/freetype/freetype2.git/tree/docs/FTL.TXT": "FTL",
"git.savannah.gnu.org/cgit/gcl.git/tree/gcl/lsp/gcl_loop.lsp?h=Version_2_6_13pre": "LOOP",
"git.savannah.gnu.org/cgit/gnulib.git/tree/modules/COPYING?id=7b08932179d0d6b017f7df01a2ddf6e096b038e3": "FSFULLRSD",
"git.savannah.gnu.org/cgit/indent.git/tree/doc/indent.texi?id=a74c6b4ee49397cf330b333da1042bffa60ed14f#n1788": "BSD-4.3TAHOE",
"git.savannah.gnu.org/cgit/indent.git/tree/doc/indent.texi?id=a74c6b4ee49397cf330b333da1042bffa60ed14f#n74": "Latex2e-translated-notice",
"git.savannah.gnu.org/cgit/kazlib.git/tree/except.c?id=0062df360c2d17d57f6af19b0e444c51feb99036": "Kazlib",
"git.savannah.gnu.org/cgit/wget.git/tree/util/trunc.c?h=v1.21.3&id=40747a11e44ced5a8ac628a41f879ced3e2ebce9#n6": "FSFAP-no-warranty-disclaimer",
"git.sr.ht/~nabijaczleweli/ossp-var": "OSSP",
"github.com/389ds/389-ds-base/blob/main/ldap/include/sysexits-compat.h#L15": "BSD-4.3TAHOE",
"github.com/AcademySoftwareFoundation/foundation/blob/main/digital_assets/aswf_digital_assets_license_v1.0.txt": "ASWF-Digital-Assets-1.0",
"github.com/AcademySoftwareFoundation/foundation/blob/main/digital_assets/aswf_digital_assets_license_v1.1.txt": "ASWF-Digital-Assets-1.1",
"github.com/AdaCore/gnatcoll-core/blob/master/docs/index.rst": "AdaCore-doc",
"github.com/AdaCore/gnatcoll-db/blob/master/docs/index.rst": "AdaCore-doc",
"github.com/AdaCore/xmlada/blob/master/docs/index.rst": "AdaCore-doc",
"github.com/BuildCraft/BuildCraft/blob/623d323b1868712f29f4a8b0979a02e8d1835131/LICENSE": "MMPL-1.0.1",
"github.com/CommunitySpecification/1.0/blob/master/1._Community_Specification_License-v1.md": "Community-Spec-1.0",
"github.com/CorsixTH/deps/blob/fd339a9f526d1d9c9f01ccf39e438a015da50035/licences/libgsm.txt": "TU-Berlin-2.0",
"github.com/Cyan4973/xxHash/blob/dev/doc/xxhash_spec.md": "LPD-document",
"github.com/DISIC/politique-de-contribution-open-source/blob/master/LICENSE.pdf": "etalab-2.0",
"github.com/Distrotech/hdparm/blob/4517550db29a91420fb2b020349523b1b4512df2/LICENSE.TXT": "hdparm",
"github.com/Distrotech/transfig/blob/master/transfig/transfig.c": "Xfig",
"github.com/DrItanium/maya/blob/master/LICENSE.CLIPS": "Clips",
"github.com/Dual-Life/mime-base64/blob/master/Base64.xs#L12": "metamail",
"github.com/EthicalSource/hippocratic-license/blob/58c0e646d64ff6fbee275bfe2b9492f914e3ab2a/LICENSE.txt": "Hippocratic-2.1",
"github.com/Fraunhofer-IIS/mpeghdec/blob/main/LICENSE.txt": "FDK-MPEG-H",
"github.com/FreeRADIUS/freeradius-client/blob/master/COPYRIGHT#L32": "HPND-Fenneberg-Livingston",
"github.com/GNOME/gcr/blob/master/docs/COPYING": "GCR-docs",
"github.com/GNOME/libxml2/blob/master/dict.c": "ISC-Veillard",
"github.com/HDFGroup/hdf5/?tab=License-1-ov-file#readme": "HDF5",
"github.com/KhronosGroup/SPIRV-Cross/blob/main/LICENSES/LicenseRef-KhronosFreeUse.txt": "MIT-Khronos-old",
"github.com/KhronosGroup/glslang/blob/main/LICENSE.txt#L949": "AML-glslang",
"github.com/MariaDB/server/blob/11.6/libmysqld/lib_sql.cc": "Boehm-GC-without-fee",
"github.com/Neo23x0/sigma/blob/master/LICENSE.Detection.Rules.md": "DRL-1.0",
"github.com/OGRECave/ogre/blob/master/OgreMain/include/OgreSingleton.h#L28C3-L35C46": "Game-Programming-Gems",
"github.com/Perl/perl5/blob/blead/util.c#L6136": "Martin-Birgmeier",
"github.com/PicoEFI/PicoEFI/blob/2263ac1a2bc98e5534d64a4896d3cfbbae00745a/LICENSES/LicenseRef-scancode-bsd-no-disclaimer-unmodified.txt": "BSD-Source-Code-no-disclaimer",
"github.com/PixarAnimationStudios/OpenSubdiv/blob/v3_5_0/opensubdiv/version.cpp#L2-L22": "Pixar",
"github.com/PixarAnimationStudios/OpenSubdiv/raw/v3_5_0/LICENSE.txt": "Pixar",
"github.com/RTimothyEdwards/magic/blob/master/LICENSE": "HPND-UC-export-US",
"github.com/SELinuxProject/selinux/blob/master/libselinux/LICENSE": "libselinux-1.0",
"github.com/SWI-Prolog/packages-clib/blob/master/sha1/brg_endian.h": "Brian-Gladman-3-Clause",
"github.com/SWI-Prolog/packages-xpce/blob/161a40cd82004f731ba48024f9d30af388a7edf5/src/img/gifwrite.c#L21-L26": "FBM",
"github.com/SWI-Prolog/swipl-devel/blob/master/src/os/dtoa.c": "dtoa",
"github.com/SigmaHQ/Detection-Rule-License/blob/6ec7fbde6101d101b5b5d1fcb8f9b69fbc76c04a/LICENSE.Detection.Rules.md": "DRL-1.1",
"github.com/Slicer/Slicer/blob/main/License.txt": "3D-Slicer-1.0",
"github.com/Unidata/UDUNITS-2/blob/master/COPYRIGHT": "UCAR",
"github.com/acpica/acpica/blob/master/source/common/acfileio.c#L119": "BSD-3-Clause-acpica",
"github.com/adaptivecomputing/torque/blob/master/PBS_License.txt": "OpenPBS-2.3",
"github.com/adversary-org/wtfnmf/raw/3f2cd8235a64350a57a51b9739715edaea63ec1a/COPYING.WTFNMFPL-utf8": "WTFNMFPL",
"github.com/adversary-org/wtfnmf/raw/refs/tags/1.0/COPYING.WTFNMFPL": "WTFNMFPL",
"github.com/ap/Text-Tabs/blob/master/lib.modern/Text/Tabs.pm#L148": "TTWL",
"github.com/apache/apr/blob/trunk/LICENSE#L298C6-L298C29": "Caldera-no-preamble",
"github.com/audacious-media-player/audacious/blob/master/COPYING": "BSD-2-Clause-pkgconf-disclaimer",
"github.com/audacious-media-player/audacious/blob/master/src/audacious/main.cc": "BSD-2-Clause-pkgconf-disclaimer",
"github.com/aws/mit-0": "MIT-0",
"github.com/awsdocs/aws-cloud9-user-guide/blob/master/LICENSE-SAMPLECODE": "MIT-0",
"github.com/bagder/curl/blob/master/COPYING": "curl",
@@ -1772,6 +1804,7 @@ var urlToLicense = map[string]string{
"github.com/file/file/blob/master/COPYING": "BSD-2-Clause-Darwin",
"github.com/freebsd/freebsd-src/blob/main/lib/msun/src/e_acosh.c": "SunPro",
"github.com/freebsd/freebsd-src/blob/main/lib/msun/src/e_lgammal.c": "SunPro",
"github.com/freebsd/pkg/blob/main/src/add.c": "BSD-2-Clause-pos-unchanged",
"github.com/gabegundy/atlserver/blob/master/License.txt": "MS-LPL",
"github.com/greymass/swift-eosio/blob/master/LICENSE": "BSD-3-Clause-No-Military-License",
"github.com/hamano/man2html/blob/master/man2html.c": "man2html",
@@ -1784,6 +1817,7 @@ var urlToLicense = map[string]string{
"github.com/jsommers/pytricia/blob/master/patricia.c#L33-L67": "BSD-4-Clause",
"github.com/kfish/xsel/blob/master/COPYING": "HPND-sell-variant",
"github.com/kohler/t1utils/blob/master/LICENSE": "MIT-Click",
"github.com/krb5/krb5/blob/kfw-4.0.1-beta1/src/lib/crypto/builtin/aes/aesopt.h": "Brian-Gladman-3-Clause-no-conversion",
"github.com/krb5/krb5/blob/krb5-1.21.2-final/NOTICE#L111-L133": "HPND-export2-US",
"github.com/krb5/krb5/blob/krb5-1.21.2-final/NOTICE#L1157-L1182": "HPND-export-US-modify",
"github.com/krb5/krb5/blob/krb5-1.21.2-final/NOTICE#L140-L156": "Brian-Gladman-2-Clause",
@@ -1793,6 +1827,7 @@ var urlToLicense = map[string]string{
"github.com/krb5/krb5/blob/krb5-1.21.2-final/NOTICE#L831-L852": "HPND-export-US-acknowledgement",
"github.com/krzyzanowskim/CryptoSwift/blob/main/LICENSE": "CryptoSwift",
"github.com/lattera/freebsd/blob/master/sys/cam/cam.c#L4": "BSD-Source-beginning-file",
"github.com/libarchive/libarchive/blob/master/libarchive_fe/line_reader.c": "BSD-2-Clause-pos-unchanged",
"github.com/libcheck/check/blob/master/checkmk/checkmk.in": "checkmk",
"github.com/libjpeg-turbo/libjpeg-turbo/blob/main/README.ijg#L117-L161": "IJG",
"github.com/lsof-org/lsof/blob/master/COPYING": "lsof",
@@ -1805,6 +1840,7 @@ var urlToLicense = map[string]string{
"github.com/mm2/Little-CMS/blob/master/src/cmssm.c#L207": "softSurfer",
"github.com/mruby/mruby/blob/83d12f8d52522cdb7c8cc46fad34821359f453e6/mrbgems/mruby-dir/src/Win/dirent.c#L127-L140": "HPND-Kevlin-Henney",
"github.com/mtoyoda/sl/blob/master/LICENSE": "SL",
"github.com/mvt-project/mvt/blob/main/LICENSE": "MVT-1.1",
"github.com/n8n-io/n8n/blob/master/LICENSE.md": "SUL-1.0",
"github.com/ncbi/datasets/blob/0ea4cd16b61e5b799d9cc55aecfa016d6c9bd2bf/LICENSE.md": "NCBI-PD",
"github.com/ncbi/datasets/blob/master/LICENSE.md": "NCBI-PD",
@@ -1815,6 +1851,7 @@ var urlToLicense = map[string]string{
"github.com/novnc/noVNC/blob/master/core/crypto/des.js#L24": "Widget-Workshop",
"github.com/oetiker/rrdtool-1.x/blob/master/src/plbasename.c#L8-L11": "gtkbook",
"github.com/open-quantum-safe/liboqs/blob/40b01fdbb270f8614fde30e65d30e9da18c02393/src/common/rand/rand_nist.c#L1-L15": "NIST-Software",
"github.com/open-webui/open-webui/blob/main/LICENSE": "BSD-3-Clause-OpenWebUI",
"github.com/openjdk/jdk/blob/6162e2c5213c5dd7c1127fd9616b543efa898962/LICENSE": "GPL-2.0-or-later",
"github.com/openssh/openssh-portable/blob/1b11ea7c58cd5c59838b5fa574cd456d6047b2d4/LICENCE#L10": "SSH-OpenSSH",
"github.com/openssh/openssh-portable/blob/1b11ea7c58cd5c59838b5fa574cd456d6047b2d4/pathnames.h": "SSH-short",
@@ -1846,6 +1883,7 @@ var urlToLicense = map[string]string{
"github.com/sigmavirus24/x11-ssh-askpass/blob/master/dynlist.c": "HPND-sell-variant-MIT-disclaimer-rev",
"github.com/signal11/hidapi/blob/master/LICENSE-orig.txt": "HIDAPI",
"github.com/slogan621/gtkbook": "gtkbook",
"github.com/sudo-project/sudo/blob/c1307ea9ff340ce0538779f8e456501461fc44b7/plugins/sudoers/redblack.c#L24-L43": "BSD-ask-to-endorse",
"github.com/swh/ladspa/blob/7bf6f3799fdba70fda297c2d8fd9f526803d9680/gsm/COPYRIGHT": "TU-Berlin-1.0",
"github.com/tats/w3m/blob/master/COPYING": "w3m",
"github.com/tcheneau/Routing/blob/f09f46fcfe636107f22f2c98348188a65a135d98/README.md": "NIST-PD",
@@ -1856,6 +1894,7 @@ var urlToLicense = map[string]string{
"github.com/uim/libgcroots/blob/master/COPYING": "Boehm-GC",
"github.com/usnistgov/fipy/blob/86aaa5c2ba2c6f1be19593c5986071cf6568cc34/LICENSE.rst": "NIST-PD-fallback",
"github.com/usnistgov/jsip/blob/59700e6926cbe96c5cdae897d9a7d2656b42abe3/LICENSE": "NIST-PD-fallback",
"github.com/vattam/BSDGames/blob/master/atc/extern.c#L36": "atc-game",
"github.com/vixie/cron/tree/545b3f5246824a9cda5905eeb7cf019c95e66995": "Vixie-Cron",
"github.com/vstroebel/jpeg-encoder/blob/main/src/fdct.rs#L1-L72": "IJG",
"github.com/wernerd/ZRTPCPP/blob/6b3cd8e6783642292bad0c21e3e5e5ce45ff3e03/cryptcommon/twofish.c#L113C3-L127": "Ferguson-Twofish",
@@ -1869,6 +1908,7 @@ var urlToLicense = map[string]string{
"gitlab.com/embeddable-common-lisp/ecl/-/blob/develop/src/lsp/loop.lsp": "LOOP",
"gitlab.com/libeigen/eigen/-/blob/master/COPYING.MINPACK": "Minpack",
"gitlab.common-lisp.net/cmucl/cmucl/-/blob/master/src/code/loop.lisp": "LOOP",
"gitlab.freedesktop.org/dbus/dbus/-/blob/789d97ad53044f94351a958e785c2923643b346a/COPYING#L11-207": "AFL-2.1",
"gitlab.freedesktop.org/freetype/freetype/-/raw/master/docs/FTL.TXT": "FTL",
"gitlab.freedesktop.org/libopenraw/libopenraw/blob/master/lib/ljpegdecompressor.cpp#L32": "Cornell-Lossless-JPEG",
"gitlab.freedesktop.org/mesa/glw/-/blob/master/README?ref_type=heads": "SGI-OpenGL",
@@ -1892,6 +1932,7 @@ var urlToLicense = map[string]string{
"gitlab.freedesktop.org/xorg/lib/libxinerama/-/blob/cc22c2f60c3862482562955116d5455263b443dc/COPYING#L44-66": "X11-no-permit-persons",
"gitlab.freedesktop.org/xorg/lib/libxtst/-/blob/master/COPYING?ref_type=heads#L108-117": "HPND-doc-sell",
"gitlab.freedesktop.org/xorg/lib/libxtst/-/blob/master/COPYING?ref_type=heads#L70-77": "HPND-doc",
"gitlab.freedesktop.org/xorg/xserver/-/blob/dd5c2595a42d3ff0c4f18d9b53d1f6c3fd934fd4/COPYING#L365-389": "MIT",
"gitlab.freedesktop.org/xorg/xserver/-/blob/master/COPYING?ref_type=heads#L1781": "HPND-sell-MIT-disclaimer-xserver",
"gitlab.freedesktop.org/xorg/xserver/-/blob/master/COPYING?ref_type=heads#L239": "DEC-3-Clause",
"gitlab.freedesktop.org/xorg/xserver/-/blob/master/COPYING?ref_type=heads#L752": "Adobe-Display-PostScript",
@@ -1906,6 +1947,7 @@ var urlToLicense = map[string]string{
"heasarc.gsfc.nasa.gov/docs/software/ftools/fv/doc/license.html": "CFITSIO",
"helixcommunity.org/content/rpsl": "RPSL-1.0",
"hstechdocs.helpsystems.com/manuals/globalscape/archive/cuteftp6/open_ssl_license_agreement.htm": "OpenSSL-standalone",
"interoperable-europe.ec.europa.eu/sites/default/files/inline-files/EUPL%20v1_2%20EN(1).txt": "EUPL-1.2",
"java.net/projects/javaeetutorial/pages/BerkeleyLicense": "BSD-3-Clause-No-Nuclear-License-2014",
"javaee.github.io/glassfish/LICENSE": "CDDL-1.1",
"jogamp.org/git/?p=gluegen.git;a=blob_plain;f=LICENSE.txt": "BSD-3-Clause-No-Nuclear-Warranty",
@@ -1924,6 +1966,7 @@ var urlToLicense = map[string]string{
"library.netapp.com/ecm/ecm_download_file/ECMP1196395": "OpenSSL-standalone",
"license.coscl.org.cn/MulanPSL/": "MulanPSL-1.0",
"license.coscl.org.cn/MulanPSL2": "MulanPSL-2.0",
"license.mvt.re/1.1/": "MVT-1.1",
"lists.gnu.org/archive/html/autoconf/2012-04/msg00061.html": "FSFULLRWD",
"lists.opensource.org/pipermail/license-discuss_lists.opensource.org/2002-November/006304.html": "HPND",
"mariadb.com/bsl11/": "BUSL-1.1",
@@ -1965,99 +2008,105 @@ var urlToLicense = map[string]string{
"opensource.linux-mirror.org/licenses/afl-1.1.txt": "AFL-1.1",
"opensource.linux-mirror.org/licenses/afl-1.2.txt": "AFL-1.2",
"opensource.linux-mirror.org/licenses/afl-2.1.txt": "AFL-2.1",
"opensource.org/license/apache-2-0": "Apache-2.0",
"opensource.org/license/mit/": "MIT",
"opensource.org/license/olfl-1-3/": "OLFL-1.3",
"opensource.org/license/osc-license-1-0": "OSC-1.0",
"opensource.org/licenses/0BSD": "0BSD",
"opensource.org/licenses/AGPL-3.0": "AGPL-3.0-or-later",
"opensource.org/licenses/APL-1.0": "APL-1.0",
"opensource.org/licenses/Apache-1.1": "Apache-1.1",
"opensource.org/licenses/Apache-2.0": "Apache-2.0",
"opensource.org/licenses/Artistic-1.0": "Artistic-1.0-cl8",
"opensource.org/licenses/BSD-2-Clause": "BSD-2-Clause",
"opensource.org/licenses/BSD-3-Clause": "BSD-3-Clause",
"opensource.org/licenses/BSDplusPatent": "BSD-2-Clause-Patent",
"opensource.org/licenses/BSL-1.0": "BSL-1.0",
"opensource.org/licenses/CAL-1.0": "CAL-1.0-Combined-Work-Exception",
"opensource.org/licenses/CATOSL-1.1": "CATOSL-1.1",
"opensource.org/licenses/CNRI-Python": "CNRI-Python",
"opensource.org/licenses/CPAL-1.0": "CPAL-1.0",
"opensource.org/licenses/CPL-1.0": "CPL-1.0",
"opensource.org/licenses/CUA-OPL-1.0": "CUA-OPL-1.0",
"opensource.org/licenses/ECL-1.0": "ECL-1.0",
"opensource.org/licenses/ECL-2.0": "ECL-2.0",
"opensource.org/licenses/EFL-1.0": "EFL-1.0",
"opensource.org/licenses/EFL-2.0": "EFL-2.0",
"opensource.org/licenses/EPL-1.0": "EPL-1.0",
"opensource.org/licenses/EUDatagrid": "EUDatagrid",
"opensource.org/licenses/EUPL-1.1": "EUPL-1.1",
"opensource.org/licenses/EUPL-1.2": "EUPL-1.2",
"opensource.org/licenses/Entessa": "Entessa",
"opensource.org/licenses/Fair": "Fair",
"opensource.org/licenses/Frameworx-1.0": "Frameworx-1.0",
"opensource.org/licenses/GPL-2.0": "GPL-2.0-or-later",
"opensource.org/licenses/GPL-3.0": "GPL-3.0-or-later",
"opensource.org/licenses/HPND": "HPND",
"opensource.org/licenses/IPA": "IPA",
"opensource.org/licenses/IPL-1.0": "IPL-1.0",
"opensource.org/licenses/ISC": "ISC",
"opensource.org/licenses/Intel": "Intel",
"opensource.org/licenses/LGPL-2.1": "LGPL-2.1-or-later",
"opensource.org/licenses/LGPL-3.0": "LGPL-3.0-or-later",
"opensource.org/licenses/LPL-1.0": "LPL-1.0",
"opensource.org/licenses/LPL-1.02": "LPL-1.02",
"opensource.org/licenses/LPPL-1.3c": "LPPL-1.3c",
"opensource.org/licenses/LiLiQ-P-1.1": "LiLiQ-P-1.1",
"opensource.org/licenses/LiLiQ-R-1.1": "LiLiQ-R-1.1",
"opensource.org/licenses/LiLiQ-Rplus-1.1": "LiLiQ-Rplus-1.1",
"opensource.org/licenses/MIT": "MIT",
"opensource.org/licenses/MPL-1.0": "MPL-1.0",
"opensource.org/licenses/MPL-1.1": "MPL-1.1",
"opensource.org/licenses/MPL-2.0": "MPL-2.0-no-copyleft-exception",
"opensource.org/licenses/MS-PL": "MS-PL",
"opensource.org/licenses/MS-RL": "MS-RL",
"opensource.org/licenses/MirOS": "MirOS",
"opensource.org/licenses/Motosoto": "Motosoto",
"opensource.org/licenses/Multics": "Multics",
"opensource.org/licenses/NASA-1.3": "NASA-1.3",
"opensource.org/licenses/NCSA": "NCSA",
"opensource.org/licenses/NGPL": "NGPL",
"opensource.org/licenses/NOSL3.0": "NPOSL-3.0",
"opensource.org/licenses/NTP": "NTP",
"opensource.org/licenses/Naumen": "Naumen",
"opensource.org/licenses/OCLC-2.0": "OCLC-2.0",
"opensource.org/licenses/OFL-1.1": "OFL-1.1-no-RFN",
"opensource.org/licenses/OGTSL": "OGTSL",
"opensource.org/licenses/OPL-2.1": "OSET-PL-2.1",
"opensource.org/licenses/OSL-1.0": "OSL-1.0",
"opensource.org/licenses/OSL-2.1": "OSL-2.1",
"opensource.org/licenses/OSL-3.0": "OSL-3.0",
"opensource.org/licenses/PHP-3.0": "PHP-3.0",
"opensource.org/licenses/PostgreSQL": "PostgreSQL",
"opensource.org/licenses/Python-2.0": "Python-2.0",
"opensource.org/licenses/QPL-1.0": "QPL-1.0",
"opensource.org/licenses/RPL-1.1": "RPL-1.1",
"opensource.org/licenses/RPL-1.5": "RPL-1.5",
"opensource.org/licenses/RPSL-1.0": "RPSL-1.0",
"opensource.org/licenses/RSCPL": "RSCPL",
"opensource.org/licenses/SISSL": "SISSL",
"opensource.org/licenses/SPL-1.0": "SPL-1.0",
"opensource.org/licenses/SimPL-2.0": "SimPL-2.0",
"opensource.org/licenses/Sleepycat": "Sleepycat",
"opensource.org/licenses/UCL-1.0": "UCL-1.0",
"opensource.org/licenses/UPL": "UPL-1.0",
"opensource.org/licenses/VSL-1.0": "VSL-1.0",
"opensource.org/licenses/W3C": "W3C",
"opensource.org/licenses/Watcom-1.0": "Watcom-1.0",
"opensource.org/licenses/Xnet": "Xnet",
"opensource.org/licenses/ZPL-2.0": "ZPL-2.0",
"opensource.org/licenses/Zlib": "Zlib",
"opensource.org/licenses/afl-3.0": "AFL-3.0",
"opensource.org/licenses/artistic-license-2.0": "Artistic-2.0",
"opensource.org/licenses/attribution": "AAL",
"opensource.org/licenses/cddl1": "CDDL-1.0",
"opensource.org/licenses/nokia": "Nokia",
"opensource.org/license/0BSD": "0BSD",
"opensource.org/license/AAL": "AAL",
"opensource.org/license/AFL-3.0": "AFL-3.0",
"opensource.org/license/AGPL-3.0": "AGPL-3.0-or-later",
"opensource.org/license/AGPL-3.0-only": "AGPL-3.0-only",
"opensource.org/license/APL-1.0": "APL-1.0",
"opensource.org/license/Artistic-1.0": "Artistic-1.0-cl8",
"opensource.org/license/Artistic-2.0": "Artistic-2.0",
"opensource.org/license/BSD-2-Clause": "BSD-2-Clause",
"opensource.org/license/BSD-2-Clause-Patent": "BSD-2-Clause-Patent",
"opensource.org/license/BSD-3-Clause": "BSD-3-Clause",
"opensource.org/license/BSL-1.0": "BSL-1.0",
"opensource.org/license/CAL-1.0": "CAL-1.0-Combined-Work-Exception",
"opensource.org/license/CATOSL-1.1": "CATOSL-1.1",
"opensource.org/license/CDDL-1.0": "CDDL-1.0",
"opensource.org/license/CNRI-Python": "CNRI-Python",
"opensource.org/license/CNRI-Python-GPL-Compatible": "CNRI-Python-GPL-Compatible",
"opensource.org/license/CPAL-1.0": "CPAL-1.0",
"opensource.org/license/CPL-1.0": "CPL-1.0",
"opensource.org/license/CUA-OPL-1.0": "CUA-OPL-1.0",
"opensource.org/license/ECL-1.0": "ECL-1.0",
"opensource.org/license/ECL-2.0": "ECL-2.0",
"opensource.org/license/EFL-1.0": "EFL-1.0",
"opensource.org/license/EFL-2.0": "EFL-2.0",
"opensource.org/license/EPL-1.0": "EPL-1.0",
"opensource.org/license/EUDatagrid": "EUDatagrid",
"opensource.org/license/EUPL-1.1": "EUPL-1.1",
"opensource.org/license/EUPL-1.2": "EUPL-1.2",
"opensource.org/license/Entessa": "Entessa",
"opensource.org/license/Fair": "Fair",
"opensource.org/license/Frameworx-1.0": "Frameworx-1.0",
"opensource.org/license/GPL-2.0": "GPL-2.0-or-later",
"opensource.org/license/GPL-2.0-only": "GPL-2.0-only",
"opensource.org/license/GPL-3.0": "GPL-3.0-or-later",
"opensource.org/license/GPL-3.0-only": "GPL-3.0-only",
"opensource.org/license/HPND": "HPND",
"opensource.org/license/IPA": "IPA",
"opensource.org/license/IPL-1.0": "IPL-1.0",
"opensource.org/license/ISC": "ISC",
"opensource.org/license/Intel": "Intel",
"opensource.org/license/LGPL-2.1": "LGPL-2.1-or-later",
"opensource.org/license/LGPL-2.1-only": "LGPL-2.1-only",
"opensource.org/license/LGPL-3.0": "LGPL-3.0-or-later",
"opensource.org/license/LPL-1.0": "LPL-1.0",
"opensource.org/license/LPL-1.02": "LPL-1.02",
"opensource.org/license/LPPL-1.3c": "LPPL-1.3c",
"opensource.org/license/LiLiQ-P-1.1": "LiLiQ-P-1.1",
"opensource.org/license/LiLiQ-R-1.1": "LiLiQ-R-1.1",
"opensource.org/license/LiLiQ-Rplus-1.1": "LiLiQ-Rplus-1.1",
"opensource.org/license/MIT": "MIT",
"opensource.org/license/MPL-1.0": "MPL-1.0",
"opensource.org/license/MPL-1.1": "MPL-1.1",
"opensource.org/license/MPL-2.0": "MPL-2.0-no-copyleft-exception",
"opensource.org/license/MS-PL": "MS-PL",
"opensource.org/license/MS-RL": "MS-RL",
"opensource.org/license/MirOS": "MirOS",
"opensource.org/license/Motosoto": "Motosoto",
"opensource.org/license/Multics": "Multics",
"opensource.org/license/NASA-1.3": "NASA-1.3",
"opensource.org/license/NCSA": "NCSA",
"opensource.org/license/NGPL": "NGPL",
"opensource.org/license/NPOSL-3.0": "NPOSL-3.0",
"opensource.org/license/NTP": "NTP",
"opensource.org/license/Naumen": "Naumen",
"opensource.org/license/Nokia": "Nokia",
"opensource.org/license/OCLC-2.0": "OCLC-2.0",
"opensource.org/license/OFL-1.1": "OFL-1.1-no-RFN",
"opensource.org/license/OGTSL": "OGTSL",
"opensource.org/license/OLFL-1.3": "OLFL-1.3",
"opensource.org/license/OSC-1.0": "OSC-1.0",
"opensource.org/license/OSET-PL-2.1": "OSET-PL-2.1",
"opensource.org/license/OSL-1.0": "OSL-1.0",
"opensource.org/license/OSL-2.1": "OSL-2.1",
"opensource.org/license/OSL-3.0": "OSL-3.0",
"opensource.org/license/PHP-3.0": "PHP-3.0",
"opensource.org/license/PSF-2-0": "PSF-2.0",
"opensource.org/license/PostgreSQL": "PostgreSQL",
"opensource.org/license/Python-2.0": "Python-2.0",
"opensource.org/license/Python-2.0.1": "Python-2.0.1",
"opensource.org/license/QPL-1.0": "QPL-1.0",
"opensource.org/license/RPL-1.1": "RPL-1.1",
"opensource.org/license/RPL-1.5": "RPL-1.5",
"opensource.org/license/RPSL-1.0": "RPSL-1.0",
"opensource.org/license/RSCPL": "RSCPL",
"opensource.org/license/SISSL": "SISSL",
"opensource.org/license/SPL-1.0": "SPL-1.0",
"opensource.org/license/SiMPL-2.0": "SimPL-2.0",
"opensource.org/license/Sleepycat": "Sleepycat",
"opensource.org/license/UCL-1.0": "UCL-1.0",
"opensource.org/license/UPL-1.0": "UPL-1.0",
"opensource.org/license/VSL-1.0": "VSL-1.0",
"opensource.org/license/W3C": "W3C",
"opensource.org/license/Watcom-1.0": "Watcom-1.0",
"opensource.org/license/Xnet": "Xnet",
"opensource.org/license/ZPL-2.0": "ZPL-2.0",
"opensource.org/license/Zlib": "Zlib",
"opensource.org/license/apache-1.1": "Apache-1.1",
"opensource.org/license/apache-2.0": "Apache-2.0",
"opensource.org/license/curl": "curl",
"oss.sgi.com/projects/FreeB/": "SGI-B-1.1",
"oss.sgi.com/projects/FreeB/SGIFreeSWLicB.1.0.html": "SGI-B-1.0",
"oss.sgi.com/projects/FreeB/SGIFreeSWLicB.2.0.pdf": "SGI-B-2.0",
@@ -2082,37 +2131,40 @@ var urlToLicense = map[string]string{
"raw.githubusercontent.com/NTIA/scos-sensor/refs/heads/master/LICENSE.md": "NTIA-PD",
"raw.githubusercontent.com/OpenMDW/OpenMDW/refs/heads/main/1.0/LICENSE.openmdw": "OpenMDW-1.0",
"raw.githubusercontent.com/blackhole89/notekit/master/data/fonts/Charter%20license.txt": "Bitstream-Charter",
"rc.quest.com/topics/openssh/license.php#tcpwrappers": "TCP-wrappers",
"research.cs.wisc.edu/condor/license.html#condor": "Condor-1.1",
"research.scea.com/scea_shared_source_license.html": "SCEA",
"romanrm.net/mit-zero": "MIT-0",
"sam.zoy.org/wtfpl/COPYING": "WTFPL",
"scripts.sil.org/cms/scripts/page.php?item_id=OFL10_web": "OFL-1.0-no-RFN",
"scripts.sil.org/cms/scripts/page.php?item_id=OFL_web": "OFL-1.1-no-RFN",
"slicer.org/LICENSE": "3D-Slicer-1.0",
"solderpad.org/licenses/SHL-0.5/": "SHL-0.5",
"solderpad.org/licenses/SHL-0.51/": "SHL-0.51",
"source.icu-project.org/repos/icu/icu/trunk/license.html": "ICU",
"sourceforge.net/p/buddy/gitcode/ci/master/tree/README": "Buddy",
"sourceforge.net/p/ctrio/git/ci/master/tree/README": "ISC-Veillard",
"sourceforge.net/p/mcj/xfig/ci/master/tree/src/Makefile.am": "Xfig",
"sourceforge.net/p/netpbm/code/HEAD/tree/super_stable/converter/other/pnmtorle.c": "URT-RLE",
"sourceforge.net/p/netpbm/code/HEAD/tree/super_stable/converter/other/rletopnm.c": "URT-RLE",
"sourceforge.net/p/netpbm/code/HEAD/tree/super_stable/converter/ppm/ppmtompeg/jrevdct.c#l1189": "MPEG-SSG",
"sourceforge.net/p/netpbm/code/HEAD/tree/super_stable/editor/pnmstitch.c#l2": "pnmstitch",
"sourceforge.net/p/netpbm/code/HEAD/tree/super_stable/netpbm.c#l8": "HPND-Pbmplus",
"sourceforge.net/p/sbcl/sbcl/ci/master/tree/src/code/loop.lisp": "LOOP",
"sourceforge.net/p/trustedqsl/tqsl/ci/master/tree/LICENSE.txt": "TrustedQSL",
"sourceforge.net/p/xmedcon/code/ci/master/tree/libs/dicom/README": "OFFIS",
"sourceforge.net/p/xmedcon/code/ci/master/tree/libs/ljpg/": "IJG-short",
"sourceforge.net/projects/sevenzip/files/LZMA%20SDK/": "LZMA-SDK-9.22",
"sources.debian.org/copyright/license/debianutils/4.11.2/": "SMAIL-GPL",
"sources.debian.org/src/openmpi/4.1.0-10/ompi/debuggers/msgq_interface.h/?hl=19#L19": "mpi-permissive",
"sources.gentoo.org/cgi-bin/viewvc.cgi/gentoo-x86/licenses/BitTorrent?r1=1.1&r2=1.1.1.1&diff_format=s": "BitTorrent-1.0",
"sourceware.org/bzip2/1.0.5/bzip2-manual-1.0.5.html": "bzip2-1.0.6",
"sourceware.org/cgit/binutils-gdb/tree/include/coff/sym.h#n11": "MIPS",
"sourceware.org/cgit/binutils-gdb/tree/include/coff/symconst.h#n11": "ThirdEye",
"sourceware.org/cgit/valgrind/tree/mpi/libmpiwrap.c": "bzip2-1.0.6",
"raw.githubusercontent.com/landtuna/open-mtools/6c04c2882eca7c4313ec267766ea53ac9bfe129b/mdump.c": "Informatica",
"raw.githubusercontent.com/landtuna/open-mtools/6c04c2882eca7c4313ec267766ea53ac9bfe129b/msend.c": "Informatica",
"rc.quest.com/topics/openssh/license.php#tcpwrappers": "TCP-wrappers",
"research.cs.wisc.edu/condor/license.html#condor": "Condor-1.1",
"research.scea.com/scea_shared_source_license.html": "SCEA",
"romanrm.net/mit-zero": "MIT-0",
"sam.zoy.org/wtfpl/COPYING": "WTFPL",
"scancode-licensedb.aboutcode.org/bsd-unchanged": "BSD-2-Clause-pos-unchanged",
"scripts.sil.org/cms/scripts/page.php?item_id=OFL10_web": "OFL-1.0-no-RFN",
"scripts.sil.org/cms/scripts/page.php?item_id=OFL_web": "OFL-1.1-no-RFN",
"slicer.org/LICENSE": "3D-Slicer-1.0",
"solderpad.org/licenses/SHL-0.5/": "SHL-0.5",
"solderpad.org/licenses/SHL-0.51/": "SHL-0.51",
"source.icu-project.org/repos/icu/icu/trunk/license.html": "ICU",
"sourceforge.net/p/buddy/gitcode/ci/master/tree/README": "Buddy",
"sourceforge.net/p/ctrio/git/ci/master/tree/README": "ISC-Veillard",
"sourceforge.net/p/mcj/xfig/ci/master/tree/src/Makefile.am": "Xfig",
"sourceforge.net/p/netpbm/code/HEAD/tree/super_stable/converter/other/pnmtorle.c": "URT-RLE",
"sourceforge.net/p/netpbm/code/HEAD/tree/super_stable/converter/other/rletopnm.c": "URT-RLE",
"sourceforge.net/p/netpbm/code/HEAD/tree/super_stable/converter/ppm/ppmtompeg/jrevdct.c#l1189": "MPEG-SSG",
"sourceforge.net/p/netpbm/code/HEAD/tree/super_stable/editor/pnmstitch.c#l2": "pnmstitch",
"sourceforge.net/p/netpbm/code/HEAD/tree/super_stable/netpbm.c#l8": "HPND-Pbmplus",
"sourceforge.net/p/sbcl/sbcl/ci/master/tree/src/code/loop.lisp": "LOOP",
"sourceforge.net/p/trustedqsl/tqsl/ci/master/tree/LICENSE.txt": "TrustedQSL",
"sourceforge.net/p/xmedcon/code/ci/master/tree/libs/dicom/README": "OFFIS",
"sourceforge.net/p/xmedcon/code/ci/master/tree/libs/ljpg/": "IJG-short",
"sourceforge.net/projects/sevenzip/files/LZMA%20SDK/": "LZMA-SDK-9.22",
"sources.debian.org/copyright/license/debianutils/4.11.2/": "SMAIL-GPL",
"sources.debian.org/src/openmpi/4.1.0-10/ompi/debuggers/msgq_interface.h/?hl=19#L19": "mpi-permissive",
"sources.gentoo.org/cgi-bin/viewvc.cgi/gentoo-x86/licenses/BitTorrent?r1=1.1&r2=1.1.1.1&diff_format=s": "BitTorrent-1.0",
"sourceware.org/bzip2/1.0.5/bzip2-manual-1.0.5.html": "bzip2-1.0.6",
"sourceware.org/cgit/binutils-gdb/tree/include/coff/sym.h#n11": "MIPS",
"sourceware.org/cgit/binutils-gdb/tree/include/coff/symconst.h#n11": "ThirdEye",
"sourceware.org/cgit/valgrind/tree/mpi/libmpiwrap.c": "bzip2-1.0.6",
"sourceware.org/git/?p=binutils-gdb.git;a=blob;f=libiberty/strcasecmp.c;h=131d81c2ce7881fa48c363dc5bf5fb302c61ce0b;hb=HEAD": "BSD-4.3RENO",
"sourceware.org/git/?p=binutils-gdb.git;a=blob;f=readline/readline/support/wcwidth.c;h=0f5ec995796f4813abbcf4972aec0378ab74722a;hb=HEAD#l55": "HPND-Markus-Kuhn",
"sourceware.org/git/?p=bzip2.git;a=blob;f=LICENSE;hb=bzip2-1.0.6": "bzip2-1.0.6",
@@ -2129,6 +2181,7 @@ var urlToLicense = map[string]string{
"tahoe-lafs.org/trac/tahoe-lafs/browser/trunk/COPYING.TGPPL.rst": "TGPPL-1.0",
"ti.arc.nasa.gov/opensource/nosa/": "NASA-1.3",
"trac.enlightenment.org/e/browser/trunk/imlib2/COPYING": "Imlib2",
"tunes.org/legalese/bugroff.html": "Bugroff",
"ubuntu.com/legal/font-licence": "Ubuntu-font-1.0",
"unlicense.org/": "Unlicense",
"vimdoc.sourceforge.net/htmldoc/uganda.html": "Vim",
@@ -2139,6 +2192,7 @@ var urlToLicense = map[string]string{
"web.archive.org/web/20041020171434/http://www.rosenlaw.com/osl2.0.html": "OSL-2.0",
"web.archive.org/web/20050212003940/http://www.rosenlaw.com/osl21.htm": "OSL-2.1",
"web.archive.org/web/20060319014854/http://info.borland.com/devsupport/interbase/opensource/IPL.html": "Interbase-1.0",
"web.archive.org/web/20061013013632/http://opensource.linux-mirror.org/licenses/afl-2.1.txt": "AFL-2.1",
"web.archive.org/web/20080207013128/http://www.gnome.org/fonts/": "Bitstream-Vera",
"web.archive.org/web/20100302225219/http://www.zimbra.com/license/zimbra-public-license-1-3.html": "Zimbra-1.3",
"web.archive.org/web/20111123062036/http://research.cs.wisc.edu/condor/license.html#condor": "Condor-1.1",
@@ -2243,11 +2297,11 @@ var urlToLicense = map[string]string{
"www.jython.org/license.html": "CNRI-Jython",
"www.kermitproject.org/ck90.html#source": "HPND-export-US",
"www.kernel.org/doc/man-pages/licenses.html": "Linux-man-pages-copyleft",
"www.latex-project.org/lppl/LPPL-1.3c.txt": "LPPL-1.3c",
"www.latex-project.org/lppl/lppl-1-0.txt": "LPPL-1.0",
"www.latex-project.org/lppl/lppl-1-1.txt": "LPPL-1.1",
"www.latex-project.org/lppl/lppl-1-2.txt": "LPPL-1.2",
"www.latex-project.org/lppl/lppl-1-3a.txt": "LPPL-1.3a",
"www.latex-project.org/lppl/lppl-1-3c.txt": "LPPL-1.3c",
"www.lemis.com/grog/UNIX/ancient-source-all.pdf": "Caldera",
"www.libpng.org/pub/png/src/libpng-LICENSE.txt": "libpng-2.0",
"www.mail-archive.com/debian-bugs-rc@lists.debian.org/msg11494.html": "Symlinks",
@@ -2269,16 +2323,17 @@ var urlToLicense = map[string]string{
"www.netlib.org/lapack/LICENSE.txt": "BSD-3-Clause-Open-MPI",
"www.netlib.org/minpack/disclaimer": "Minpack",
"www.networkcultures.org/_uploads/%233notebook_telekommunist.pdf": "PPL",
"www.oclc.org/research/activities/software/license/v2final.htm": "OCLC-2.0",
"www.ogc.org/ogc/software/1.0": "OGC-1.0",
"www.ohwr.org/project/cernohl/wikis/Documents/CERN-OHL-version-2": "CERN-OHL-W-2.0",
"www.ohwr.org/project/licenses/wikis/cern-ohl-v1.1": "CERN-OHL-1.1",
"www.ohwr.org/project/licenses/wikis/cern-ohl-v1.2": "CERN-OHL-1.2",
"www.open-mpi.org/community/license.php": "BSD-3-Clause-Open-MPI",
"www.opencascade.com/content/occt-public-license": "OCCT-PL",
"www.opendatacommons.org/licenses/odbl/1.0/": "ODbL-1.0",
"www.opengroup.org/testing/downloads/The_Open_Group_TSL.txt": "OGTSL",
"www.openhub.net/licenses/mslpl": "MS-LPL",
"www.nist.gov/open/copyright-fair-use-and-licensing-statements-srd-data-software-and-technical-series-publications": "NIST-Software",
"www.oclc.org/research/activities/software/license/v2final.htm": "OCLC-2.0",
"www.ogc.org/ogc/software/1.0": "OGC-1.0",
"www.ohwr.org/project/cernohl/wikis/Documents/CERN-OHL-version-2": "CERN-OHL-W-2.0",
"www.ohwr.org/project/licenses/wikis/cern-ohl-v1.1": "CERN-OHL-1.1",
"www.ohwr.org/project/licenses/wikis/cern-ohl-v1.2": "CERN-OHL-1.2",
"www.open-mpi.org/community/license.php": "BSD-3-Clause-Open-MPI",
"www.opencascade.com/content/occt-public-license": "OCCT-PL",
"www.opendatacommons.org/licenses/odbl/1.0/": "ODbL-1.0",
"www.opengroup.org/testing/downloads/The_Open_Group_TSL.txt": "OGTSL",
"www.openhub.net/licenses/mslpl": "MS-LPL",
"www.openldap.org/devel/gitweb.cgi?p=openldap.git;a=blob;f=LICENSE;hb=1cae062821881f41b73012ba816434897abf4205": "OLDAP-2.6",
"www.openldap.org/devel/gitweb.cgi?p=openldap.git;a=blob;f=LICENSE;hb=37b4b3f6cc4bf34e1d3dec61e69914b9819d8894": "NBPL-1.0",
"www.openldap.org/devel/gitweb.cgi?p=openldap.git;a=blob;f=LICENSE;hb=42b0383c50c299977b5893ee695cf4e486fb0dc7": "OLDAP-1.2",
@@ -2301,7 +2356,7 @@ var urlToLicense = map[string]string{
"www.opensource.apple.com/license/apsl/": "APSL-2.0",
"www.opensource.apple.com/source/IOSerialFamily/IOSerialFamily-7/APPLE_LICENSE": "APSL-1.1",
"www.opensource.apple.com/source/tcl/tcl-5/tcl/generic/regfronts.c": "Spencer-99",
"www.opensource.org/licenses/EPL-2.0": "EPL-2.0",
"www.opensource.org/license/EPL-2.0": "EPL-2.0",
"www.openssl.org/source/license.html": "OpenSSL",
"www.osetfoundation.org/public-license": "OSET-PL-2.1",
"www.parliament.uk/site-information/copyright-parliament/open-parliament-licence/": "OPL-UK-3.0",
internal/spdxlicense/license_url_test.go
+90
View File
@@ -123,6 +123,96 @@ func TestLicenseByURL_AlternateScheme(t *testing.T) {
}
}
func TestLicenseByURL_OpenSourceOrgVariants(t *testing.T) {
// SPDX has changed the form of opensource.org URLs across list versions (e.g. 3.29.0 moved
// opensource.org/licenses/MIT to opensource.org/license/MIT); every form should keep resolving.
tests := []struct {
name string
url string
wantID string
}{
{name: "old plural path", url: "http://opensource.org/licenses/MIT", wantID: "MIT"},
{name: "new singular path", url: "https://opensource.org/license/MIT", wantID: "MIT"},
{name: "lowercase with trailing slash", url: "https://opensource.org/license/mit/", wantID: "MIT"},
{name: "plural path with trailing slash", url: "https://opensource.org/licenses/MIT/", wantID: "MIT"},
{name: "www prefix", url: "https://www.opensource.org/licenses/MIT", wantID: "MIT"},
{name: "uppercase host", url: "https://WWW.OpenSource.org/licenses/MIT", wantID: "MIT"},
{name: "old plural path BSD-3-Clause", url: "https://opensource.org/licenses/BSD-3-Clause", wantID: "BSD-3-Clause"},
{name: "new singular path BSD-3-Clause", url: "https://opensource.org/license/BSD-3-Clause", wantID: "BSD-3-Clause"},
{name: "lowercase BSD-3-Clause", url: "https://opensource.org/licenses/bsd-3-clause", wantID: "BSD-3-Clause"},
{name: "old plural path Apache-2.0", url: "http://opensource.org/licenses/Apache-2.0", wantID: "Apache-2.0"},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
info, found := LicenseByURL(tt.url)
if !found {
t.Fatalf("LicenseByURL(%q) found = false, want true", tt.url)
}
if info.ID != tt.wantID {
t.Errorf("LicenseByURL(%q) ID = %v, want %v", tt.url, info.ID, tt.wantID)
}
})
}
}
func TestLicenseByURL_NonOpenSourceOrgStaysExact(t *testing.T) {
// the relaxed matching is scoped to opensource.org; other hosts may have case-sensitive paths
if _, found := LicenseByURL("https://www.apache.org/licenses/license-2.0"); found {
t.Errorf("LicenseByURL() found = true for case-altered non-opensource.org URL, want false")
}
if _, found := LicenseByURL("https://opensource.org/licenses/not-a-real-license"); found {
t.Errorf("LicenseByURL() found = true for unknown opensource.org URL, want false")
}
}
func TestNormalizeOpenSourceOrgURL(t *testing.T) {
tests := []struct {
url string
want string
wantOK bool
}{
{url: "opensource.org/licenses/MIT", want: "opensource.org/license/mit", wantOK: true},
{url: "opensource.org/license/mit/", want: "opensource.org/license/mit", wantOK: true},
{url: "www.opensource.org/licenses/EPL-2.0", want: "opensource.org/license/epl-2.0", wantOK: true},
{url: "lists.opensource.org/pipermail/license-discuss", wantOK: false},
{url: "www.apache.org/licenses/LICENSE-2.0", wantOK: false},
}
for _, tt := range tests {
t.Run(tt.url, func(t *testing.T) {
got, ok := normalizeOpenSourceOrgURL(tt.url)
if ok != tt.wantOK {
t.Fatalf("normalizeOpenSourceOrgURL(%q) ok = %v, want %v", tt.url, ok, tt.wantOK)
}
if got != tt.want {
t.Errorf("normalizeOpenSourceOrgURL(%q) = %q, want %q", tt.url, got, tt.want)
}
})
}
}
func TestBuildOpenSourceOrgURLToLicense(t *testing.T) {
index := buildOpenSourceOrgURLToLicense(map[string]string{
"opensource.org/licenses/MIT": "MIT",
"opensource.org/license/mit/": "MIT",
"opensource.org/licenses/Foo": "Foo-1.0",
"opensource.org/license/foo": "Foo-2.0",
"opensource.org/license/FOO/": "Foo-1.0",
"www.apache.org/licenses/Bar": "Bar",
})
if got := index["opensource.org/license/mit"]; got != "MIT" {
t.Errorf("index[mit] = %q, want MIT", got)
}
if got, exists := index["opensource.org/license/foo"]; exists {
t.Errorf("index[foo] = %q, want no entry for URL forms mapping to different licenses", got)
}
if len(index) != 1 {
t.Errorf("len(index) = %d, want 1 (non-opensource.org URLs excluded): %v", len(index), index)
}
}
func TestStripScheme(t *testing.T) {
tests := []struct {
name string
internal/task/archive_tasks.go
+13 -1
View File
@@ -302,7 +302,7 @@ func (c *archiveCataloger) runSubPipeline(ctx context.Context, resolver file.Res
scratch := &sbom.SBOM{Artifacts: sbom.Artifacts{Packages: pkg.NewCollection()}}
scratchBuilder := sbomsync.NewBuilder(scratch)
for _, t := range c.subPipeline {
err := runTaskSafely(ctx, t, resolver, scratchBuilder)
err := runSubTaskSafely(ctx, t, resolver, scratchBuilder)
if err == nil {
continue
}
@@ -317,6 +317,18 @@ func (c *archiveCataloger) runSubPipeline(ctx context.Context, resolver file.Res
return scratch
}
// runSubTaskSafely is runTaskSafely without the task name in a panic's error, since appendUnknowns
// already prefixes it.
func runSubTaskSafely(ctx context.Context, t Task, resolver file.Resolver, builder sbomsync.Builder) (err error) {
defer func() {
if r := recover(); r != nil {
log.WithFields("task", t.Name()).Debugf("archive sub-cataloger task panic stack:\n%s", debug.Stack())
err = fmt.Errorf("panic: %v", r)
}
}()
return t.Execute(ctx, resolver, builder)
}
// archiveCandidate is one file the walk will try to open.
type archiveCandidate struct {
location file.Location
internal/task/archive_tasks_test.go
+1 -1
View File
@@ -577,7 +577,7 @@ func Test_archiveCataloger_subPipelineFailureIsRecordedAsAnUnknown(t *testing.T)
reasons, ok := s.Artifacts.Unknowns[file.Coordinates{RealPath: "/outer.zip"}]
require.True(t, ok)
require.Len(t, reasons, 1)
assert.Contains(t, reasons[0], "panicking-cataloger: nope")
assert.Equal(t, "panicking-cataloger: panic: nope", reasons[0])
})
}
internal/task/executor.go
+2 -1
View File
@@ -51,7 +51,8 @@ func runTaskSafely(ctx context.Context, t Task, resolver file.Resolver, s sbomsy
// handle individual cataloger panics
defer func() {
if e := recover(); e != nil {
err = fmt.Errorf("%v at:\n%s", e, string(debug.Stack()))
log.WithFields("task", t.Name()).Debugf("task panic stack:\n%s", debug.Stack())
err = fmt.Errorf("panic in task %q: %v", t.Name(), e)
}
}()
internal/task/executor_test.go
+20 -2
View File
@@ -8,8 +8,12 @@ import (
"github.com/wagoodman/go-progress"
"github.com/anchore/syft/internal/sbomsync"
"github.com/anchore/syft/syft/artifact"
"github.com/anchore/syft/syft/event/monitor"
"github.com/anchore/syft/syft/file"
"github.com/anchore/syft/syft/pkg"
"github.com/anchore/syft/syft/pkg/cataloger/generic"
"github.com/anchore/syft/syft/sbom"
)
func Test_TaskExecutor_PanicHandling(t *testing.T) {
@@ -21,6 +25,20 @@ func Test_TaskExecutor_PanicHandling(t *testing.T) {
Manual: progress.NewManual(-1),
})
require.Error(t, err)
require.Contains(t, err.Error(), "executor_test.go")
require.EqualError(t, err, `panic in task "panicking-cataloger": something bad happened`)
}
func Test_RunTask_GenericParserPanicBecomesUnknown(t *testing.T) {
c := generic.NewCataloger("panicking-cataloger").
WithParserByPath(func(context.Context, file.Resolver, *generic.Environment, file.LocationReadCloser) ([]pkg.Package, []artifact.Relationship, error) {
panic("boom")
}, "executor_test.go")
s := &sbom.SBOM{}
err := RunTask(context.Background(), NewPackageTask(DefaultCatalogingFactoryConfig(), c), file.NewMockResolverForPaths("executor_test.go"), sbomsync.NewBuilder(s), &monitor.TaskProgress{
Manual: progress.NewManual(-1),
})
require.NoError(t, err)
require.Len(t, s.Artifacts.Unknowns, 1)
}
internal/unknown/coordinate_error.go
+4
View File
@@ -183,6 +183,10 @@ func visitErrors(err error, fn func(error) error) error {
// unwrap singly wrapped errors
if e, ok := err.(interface{ Unwrap() error }); ok {
wrapped := e.Unwrap()
if wrapped == nil {
// nothing is wrapped (e.g. a sync.PanicError with a non-error value), so this is a leaf
return fn(err)
}
got := visitErrors(wrapped, fn)
if got == nil {
return nil
internal/unknown/coordinate_error_test.go
+9
View File
@@ -8,6 +8,7 @@ import (
"github.com/stretchr/testify/require"
"github.com/anchore/go-sync"
"github.com/anchore/syft/syft/file"
)
@@ -234,3 +235,11 @@ func Test_Append(t *testing.T) {
e3 = Appendf(e2, file.NewLocation("l3"), "%s", "e3")
require.Equal(t, "l1: e1\nl3: e3", e3.Error())
}
func Test_ExtractCoordinateErrors_keepsNilUnwrap(t *testing.T) {
// sync.PanicError unwraps to nil when the panic value is not an error
err := sync.PanicError{Value: "x"}
unknowns, remaining := ExtractCoordinateErrors(err)
require.Empty(t, unknowns)
require.Equal(t, err, remaining)
}
syft/file/cataloger/executable/cataloger.go
+11 -1
View File
@@ -7,6 +7,7 @@ import (
"debug/macho"
"encoding/binary"
"fmt"
"runtime/debug"
"sort"
"github.com/bmatcuk/doublestar/v4"
@@ -89,7 +90,7 @@ func (i *Cataloger) CatalogCtx(ctx context.Context, resolver file.Resolver) (map
return results, errs
}
func processExecutableLocation(loc file.Location, resolver file.Resolver) (*file.Executable, error) {
func processExecutableLocation(loc file.Location, resolver file.Resolver) (exec *file.Executable, err error) {
reader, err := resolver.FileContentsByLocation(loc)
if err != nil {
log.WithFields("error", err, "path", loc.RealPath).Debug("unable to get file contents")
@@ -97,6 +98,15 @@ func processExecutableLocation(loc file.Location, resolver file.Resolver) (*file
}
defer internal.CloseAndLogError(reader, loc.RealPath)
defer func() {
if r := recover(); r != nil {
// a malformed binary must not take down the whole SBOM; the caller records this as an unknown
log.WithFields("path", loc.RealPath, "panic", r).Debug("recovered from panic while reading executable")
log.Tracef("executable panic stack:\n%s", debug.Stack())
exec, err = nil, fmt.Errorf("recovered from panic while reading executable: %v", r)
}
}()
uReader, err := unionreader.GetUnionReader(reader)
if err != nil {
log.WithFields("error", err, "path", loc.RealPath).Debug("unable to get union reader")
syft/file/cataloger/executable/cataloger_test.go
+40
View File
@@ -0,0 +1,40 @@
package executable
import (
"context"
"io"
"testing"
"github.com/stretchr/testify/require"
"github.com/anchore/syft/internal/unknown"
"github.com/anchore/syft/syft/file"
)
type panicReader struct{}
func (panicReader) Read([]byte) (int, error) { panic("boom") }
type panickingContentsResolver struct {
*file.MockResolver
}
func (r panickingContentsResolver) FilesByMIMEType(...string) ([]file.Location, error) {
return r.FilesByPath("cataloger_test.go")
}
func (r panickingContentsResolver) FileContentsByLocation(file.Location) (io.ReadCloser, error) {
return io.NopCloser(panicReader{}), nil
}
func TestCatalogCtx_panicBecomesUnknown(t *testing.T) {
resolver := panickingContentsResolver{file.NewMockResolverForPaths("cataloger_test.go")}
results, err := NewCataloger(DefaultConfig()).CatalogCtx(context.Background(), resolver)
require.Empty(t, results)
unknowns, remaining := unknown.ExtractCoordinateErrors(err)
require.NoError(t, remaining)
require.Len(t, unknowns, 1)
require.ErrorContains(t, unknowns[0].Reason, "recovered from panic while reading executable: boom")
}
syft/file/cataloger/executable/elf.go
+3 -23
View File
@@ -156,32 +156,12 @@ func hasElfDynFlag1(f *elf.File, flag elf.DynFlag1) bool {
}
func hasElfDynTag(f *elf.File, tag elf.DynTag) bool {
// source https://github.com/golang/go/blob/9b4b3e5acca2dabe107fa2c3ed963097d78a4562/src/cmd/cgo/internal/testshared/shared_test.go#L280
ds := f.SectionByType(elf.SHT_DYNAMIC)
if ds == nil {
return false
}
d, err := ds.Data()
vals, err := f.DynValue(tag)
if err != nil {
log.WithFields("error", err).Trace("unable to read dynamic section from elf file")
return false
}
for len(d) > 0 {
var t elf.DynTag
switch f.Class {
case elf.ELFCLASS32:
t = elf.DynTag(f.ByteOrder.Uint32(d[0:4]))
d = d[8:]
case elf.ELFCLASS64:
t = elf.DynTag(f.ByteOrder.Uint64(d[0:8]))
d = d[16:]
}
if t == tag {
return true
}
}
return false
return len(vals) > 0
}
func isELFPIE(f *elf.File) bool {
syft/file/cataloger/executable/elf_test.go
+88
View File
@@ -1,7 +1,9 @@
package executable
import (
"bytes"
"debug/elf"
"encoding/binary"
"os"
"path/filepath"
"testing"
@@ -226,3 +228,89 @@ func Test_elfHasExports(t *testing.T) {
})
}
}
func Test_hasElfDynTag(t *testing.T) {
dynEntry := func(class elf.Class, tag elf.DynTag) []byte {
if class == elf.ELFCLASS32 {
return binary.LittleEndian.AppendUint32(binary.LittleEndian.AppendUint32(nil, uint32(tag)), 0)
}
return binary.LittleEndian.AppendUint64(binary.LittleEndian.AppendUint64(nil, uint64(tag)), 0)
}
tests := []struct {
name string
class elf.Class
dynamic []byte
want bool
}{
{
name: "32-bit tag present",
class: elf.ELFCLASS32,
dynamic: dynEntry(elf.ELFCLASS32, elf.DT_BIND_NOW),
want: true,
},
{
name: "64-bit tag present",
class: elf.ELFCLASS64,
dynamic: dynEntry(elf.ELFCLASS64, elf.DT_BIND_NOW),
want: true,
},
{
name: "64-bit tag absent",
class: elf.ELFCLASS64,
dynamic: dynEntry(elf.ELFCLASS64, elf.DT_NULL),
want: false,
},
{
name: "32-bit truncated dynamic section",
class: elf.ELFCLASS32,
dynamic: dynEntry(elf.ELFCLASS32, elf.DT_BIND_NOW)[:4],
want: false,
},
{
name: "64-bit truncated dynamic section",
class: elf.ELFCLASS64,
dynamic: dynEntry(elf.ELFCLASS64, elf.DT_BIND_NOW)[:12],
want: false,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
f, err := elf.NewFile(bytes.NewReader(elfWithDynamicSection(t, tt.class, tt.dynamic)))
require.NoError(t, err)
assert.Equal(t, tt.want, hasElfDynTag(f, elf.DT_BIND_NOW))
})
}
}
// elfWithDynamicSection builds a little-endian ELF holding only a null section and a SHT_DYNAMIC section.
func elfWithDynamicSection(t *testing.T, class elf.Class, dynamic []byte) []byte {
t.Helper()
ident := [elf.EI_NIDENT]byte{0x7f, 'E', 'L', 'F', byte(class), byte(elf.ELFDATA2LSB), byte(elf.EV_CURRENT)}
var buf bytes.Buffer
write := func(v any) {
require.NoError(t, binary.Write(&buf, binary.LittleEndian, v))
}
if class == elf.ELFCLASS32 {
ehsize, shentsize := binary.Size(elf.Header32{}), binary.Size(elf.Section32{})
dataOff := ehsize + 2*shentsize
write(elf.Header32{
Ident: ident, Type: uint16(elf.ET_DYN), Machine: uint16(elf.EM_386), Version: uint32(elf.EV_CURRENT),
Shoff: uint32(ehsize), Ehsize: uint16(ehsize), Shentsize: uint16(shentsize), Shnum: 2,
})
write(elf.Section32{})
write(elf.Section32{Type: uint32(elf.SHT_DYNAMIC), Off: uint32(dataOff), Size: uint32(len(dynamic)), Addralign: 1})
} else {
ehsize, shentsize := binary.Size(elf.Header64{}), binary.Size(elf.Section64{})
dataOff := ehsize + 2*shentsize
write(elf.Header64{
Ident: ident, Type: uint16(elf.ET_DYN), Machine: uint16(elf.EM_X86_64), Version: uint32(elf.EV_CURRENT),
Shoff: uint64(ehsize), Ehsize: uint16(ehsize), Shentsize: uint16(shentsize), Shnum: 2,
})
write(elf.Section64{})
write(elf.Section64{Type: uint32(elf.SHT_DYNAMIC), Off: uint64(dataOff), Size: uint64(len(dynamic)), Addralign: 1})
}
buf.Write(dynamic)
return buf.Bytes()
}
syft/file/cataloger/filedigest/cataloger.go
+12 -2
View File
@@ -5,6 +5,7 @@ import (
"crypto"
"errors"
"fmt"
"runtime/debug"
"github.com/dustin/go-humanize"
@@ -87,7 +88,7 @@ func (i *Cataloger) Catalog(ctx context.Context, resolver file.Resolver, coordin
return results, err
}
func (i *Cataloger) catalogLocation(ctx context.Context, resolver file.Resolver, location file.Location) ([]file.Digest, error) {
func (i *Cataloger) catalogLocation(ctx context.Context, resolver file.Resolver, location file.Location) (digests []file.Digest, err error) {
meta, err := resolver.FileMetadataByLocation(location)
if err != nil {
return nil, err
@@ -104,7 +105,16 @@ func (i *Cataloger) catalogLocation(ctx context.Context, resolver file.Resolver,
}
defer internal.CloseAndLogError(contentReader, location.AccessPath)
digests, err := intFile.NewDigestsFromFile(ctx, contentReader, i.hashes)
defer func() {
if r := recover(); r != nil {
// the caller records this as an unknown for the location instead of failing the whole SBOM
log.WithFields("path", location.RealPath, "panic", r).Debug("recovered from panic while digesting file")
log.Tracef("file digest panic stack:\n%s", debug.Stack())
digests, err = nil, fmt.Errorf("recovered from panic while digesting file: %v", r)
}
}()
digests, err = intFile.NewDigestsFromFile(ctx, contentReader, i.hashes)
if err != nil {
return nil, internal.ErrPath{Context: "digests-cataloger", Path: location.RealPath, Err: err}
}
syft/file/cataloger/filedigest/cataloger_test.go
+25
View File
@@ -15,6 +15,7 @@ import (
stereoscopeFile "github.com/anchore/stereoscope/pkg/file"
"github.com/anchore/stereoscope/pkg/imagetest"
intFile "github.com/anchore/syft/internal/file"
"github.com/anchore/syft/internal/unknown"
"github.com/anchore/syft/syft/file"
"github.com/anchore/syft/syft/source"
"github.com/anchore/syft/syft/source/directorysource"
@@ -202,3 +203,27 @@ func TestFileDigestCataloger_GivenCoordinates(t *testing.T) {
}
}
type panicReader struct{}
func (panicReader) Read([]byte) (int, error) { panic("boom") }
type panickingContentsResolver struct {
*file.MockResolver
}
func (r panickingContentsResolver) FileContentsByLocation(file.Location) (io.ReadCloser, error) {
return io.NopCloser(panicReader{}), nil
}
func TestDigestsCataloger_panicBecomesUnknown(t *testing.T) {
resolver := panickingContentsResolver{file.NewMockResolverForPaths("cataloger_test.go")}
results, err := NewCataloger([]crypto.Hash{crypto.SHA256}).Catalog(context.Background(), resolver, file.NewLocation("cataloger_test.go").Coordinates)
require.Empty(t, results)
unknowns, remaining := unknown.ExtractCoordinateErrors(err)
require.NoError(t, remaining)
require.Len(t, unknowns, 1)
require.ErrorContains(t, unknowns[0].Reason, "recovered from panic while digesting file: boom")
}
syft/format/common/spdxhelpers/to_format_model.go
+30 -10
View File
@@ -2,6 +2,7 @@
package spdxhelpers
import (
"cmp"
"crypto/sha1"
"fmt"
"path"
@@ -180,7 +181,8 @@ func toRootRelationships(rootPackage *spdx.Package, packages []*spdx.Package) (o
func toRootPackage(s source.Description) *spdx.Package {
var prefix string
name := s.Name
// the SPDX package name is mandatory, and a source decoded from another SBOM may not carry one
name := cmp.Or(s.Name, imageRepoName(s.Metadata), s.ID, helpers.DocumentName(s))
version := s.Version
var purl *packageurl.PackageURL
@@ -211,7 +213,7 @@ func toRootPackage(s source.Description) *spdx.Package {
checksums = append(checksums, *c)
purl = &packageurl.PackageURL{
Type: "oci",
Name: s.Name,
Name: name,
Version: m.ManifestDigest,
Qualifiers: qualifiers,
}
@@ -241,7 +243,7 @@ func toRootPackage(s source.Description) *spdx.Package {
checksums = append(checksums, *c)
purl = &packageurl.PackageURL{
Type: "oci",
Name: s.Name,
Name: name,
Version: m.ManifestDigest,
Qualifiers: qualifiers,
}
@@ -276,10 +278,6 @@ func toRootPackage(s source.Description) *spdx.Package {
default:
prefix = prefixUnknown
purpose = spdxPrimaryPurposeOther
if name == "" {
name = s.ID
}
}
p := &spdx.Package{
@@ -309,6 +307,25 @@ func toRootPackage(s source.Description) *spdx.Package {
return p
}
// imageRepoName returns the repository name from an image or model user input, the same way the image source names
// itself, so a source decoded from another SBOM (which may only carry the user input) gets a name without the tag.
func imageRepoName(metadata any) string {
var userInput string
switch m := metadata.(type) {
case source.ImageMetadata:
userInput = m.UserInput
case source.OCIModelMetadata:
userInput = m.UserInput
default:
return ""
}
ref, _ := reference.Parse(userInput)
if named, ok := ref.(reference.Named); ok {
return named.Name()
}
return ""
}
func toSPDXID(identifiable artifact.Identifiable) spdx.ElementID {
id := string(identifiable.ID())
if after, ok := strings.CutPrefix(id, "SPDXRef-"); ok {
@@ -582,10 +599,13 @@ func toPackageChecksums(p pkg.Package) ([]spdx.Checksum, bool) {
})
case pkg.OpamPackage:
for _, checksum := range meta.Checksums {
parts := strings.Split(checksum, "=")
algorithm, value, ok := strings.Cut(checksum, "=")
if !ok {
continue
}
checksums = append(checksums, spdx.Checksum{
Algorithm: spdx.ChecksumAlgorithm(strings.ToUpper(parts[0])),
Value: parts[1],
Algorithm: spdx.ChecksumAlgorithm(strings.ToUpper(algorithm)),
Value: value,
})
}
}
syft/format/common/spdxhelpers/to_format_model_test.go
+60
View File
@@ -516,6 +516,20 @@ func Test_toPackageChecksums(t *testing.T) {
},
filesAnalyzed: false,
},
{
name: "Opam Package with a checksum missing its algorithm separator",
pkg: pkg.Package{
Name: "test",
Version: "1.0.0",
Metadata: pkg.OpamPackage{
Checksums: []string{"f5f1c0b4", "md5=abc"},
},
},
expected: []spdx.Checksum{
{Algorithm: "MD5", Value: "abc"},
},
filesAnalyzed: false,
},
{
name: "Package with no metadata type",
pkg: pkg.Package{
@@ -1174,3 +1188,49 @@ func Test_toFiles_nestedFilesAreNamedByTheirChainAndRoundTrip(t *testing.T) {
require.NotNil(t, b, "names: %v", byName)
assert.Equal(t, inB, toSyftCoordinates(b), "the entry path's escaping is undone on decode")
}
func Test_toRootPackage_name(t *testing.T) {
tests := []struct {
name string
src source.Description
wantName string
wantPurl string
}{
{
name: "source name wins",
src: source.Description{Name: "named", ID: "id", Metadata: source.ImageMetadata{UserInput: "alpine:3.18"}},
wantName: "named",
},
{
// a CycloneDX container component decodes to an image source with only the user input set
name: "image user input without a source name",
src: source.Description{Metadata: source.ImageMetadata{
UserInput: "alpine:3.18",
ManifestDigest: "sha256:0e4b2e0d6b2f3c2b1d3bb3a1c8f1c4d9e6a9b1e8c6f4d2b3a1f0e9d8c7b6a5f4",
}},
wantName: "alpine",
wantPurl: "pkg:oci/alpine@sha256%3A0e4b2e0d6b2f3c2b1d3bb3a1c8f1c4d9e6a9b1e8c6f4d2b3a1f0e9d8c7b6a5f4?arch=&tag=3.18",
},
{
name: "id without a source name",
src: source.Description{ID: "some-id"},
wantName: "some-id",
},
{
name: "nothing to name it by",
src: source.Description{Metadata: source.DirectoryMetadata{}},
wantName: "unknown",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
p := toRootPackage(tt.src)
assert.Equal(t, tt.wantName, p.PackageName)
var purl string
if len(p.PackageExternalReferences) > 0 {
purl = p.PackageExternalReferences[0].Locator
}
assert.Equal(t, tt.wantPurl, purl)
})
}
}
syft/format/common/spdxhelpers/to_syft_model.go
+62 -2
View File
@@ -35,6 +35,8 @@ func ToSyftModel(doc *spdx.Document) (*sbom.SBOM, error) {
return nil, errors.New("cannot convert SPDX document to Syft model because document is nil")
}
removeNilElements(doc)
spdxIDMap := make(map[string]any)
s := &sbom.SBOM{
@@ -56,6 +58,28 @@ func ToSyftModel(doc *spdx.Document) (*sbom.SBOM, error) {
return s, nil
}
// removeNilElements drops null entries from the document's lists. A JSON null (or an element the SPDX version
// converter cannot map) decodes to a nil pointer, and the rest of the conversion dereferences these elements.
func removeNilElements(doc *spdx.Document) {
doc.Packages = withoutNil(doc.Packages)
for _, p := range doc.Packages {
p.Files = withoutNil(p.Files)
p.PackageExternalReferences = withoutNil(p.PackageExternalReferences)
}
doc.Files = withoutNil(doc.Files)
doc.Relationships = withoutNil(doc.Relationships)
}
func withoutNil[T any](items []*T) []*T {
var out []*T
for _, item := range items {
if item != nil {
out = append(out, item)
}
}
return out
}
func isDirectory(name string) bool {
if name == "." || name == ".." || strings.HasSuffix(name, "/") || !strings.Contains(path.Base(name), ".") {
return true
@@ -124,12 +148,24 @@ func extractSource(spdxIDMap map[string]any, doc *spdx.Document) source.Descript
}
p := rootPackages[0]
rootType := syftRootType(p.PackageSPDXIdentifier)
switch p.PrimaryPackagePurpose {
purpose := p.PrimaryPackagePurpose
if purpose == "" {
// SPDX 2.1 and 2.2 have no primary package purpose, so fall back to the source type syft puts in the root's ID
purpose = syftRootPurpose(rootType)
}
switch purpose {
case spdxPrimaryPurposeContainer:
src = containerSource(p)
case spdxPrimaryPurposeFile:
src = fileSource(p)
case spdxPrimaryPurposeOther:
// syft writes a root package even for a source it knows nothing about, which is not a package in the SBOM
if rootType != prefixUnknown {
return src
}
default:
return src
}
@@ -142,6 +178,30 @@ func extractSource(spdxIDMap map[string]any, doc *spdx.Document) source.Descript
return src
}
// syftRootType returns the source type syft encodes in a root package ID ("DocumentRoot-<type>-<name>"), or "" for
// any other ID.
func syftRootType(id spdx.ElementID) string {
rest, ok := strings.CutPrefix(string(id), "DocumentRoot-")
if !ok {
return ""
}
rootType, _, _ := strings.Cut(rest, "-")
return rootType
}
// syftRootPurpose returns the primary package purpose syft writes for a root package of the given source type.
func syftRootPurpose(rootType string) string {
switch rootType {
case prefixImage, prefixOCIModel, prefixSnap:
return spdxPrimaryPurposeContainer
case prefixDirectory, prefixFile:
return spdxPrimaryPurposeFile
case prefixUnknown:
return spdxPrimaryPurposeOther
}
return ""
}
func containerSource(p *spdx.Package) source.Description {
id := string(p.PackageSPDXIdentifier)
@@ -202,7 +262,7 @@ func fileSource(p *spdx.Package) source.Description {
}
supplier := ""
if p.PackageSupplier.Supplier != helpers.NOASSERTION {
if p.PackageSupplier != nil && p.PackageSupplier.Supplier != helpers.NOASSERTION {
supplier = p.PackageSupplier.Supplier
}
syft/format/cpes/decoder.go
+5 -1
View File
@@ -7,6 +7,7 @@ import (
"io"
"strings"
syftinternal "github.com/anchore/syft/internal"
"github.com/anchore/syft/internal/log"
"github.com/anchore/syft/syft/cpe"
"github.com/anchore/syft/syft/format/internal"
@@ -61,7 +62,7 @@ func toSyftModel(r io.Reader) (*sbom.SBOM, error) {
var errs []error
pkgs := pkg.NewCollection()
scanner := bufio.NewScanner(r)
scanner := syftinternal.NewLineScanner(r)
for scanner.Scan() {
line := strings.TrimSpace(scanner.Text())
@@ -86,6 +87,9 @@ func toSyftModel(r io.Reader) (*sbom.SBOM, error) {
p.SetID()
pkgs.Add(p)
}
if err := scanner.Err(); err != nil {
errs = append(errs, err)
}
return &sbom.SBOM{
Artifacts: sbom.Artifacts{
syft/format/internal/spdxutil/helpers/document_name.go
+10 -5
View File
@@ -9,16 +9,21 @@ func DocumentName(src source.Description) string {
return src.Name
}
var name string
switch metadata := src.Metadata.(type) {
case source.ImageMetadata:
return metadata.UserInput
name = metadata.UserInput
case source.OCIModelMetadata:
return metadata.UserInput
name = metadata.UserInput
case source.DirectoryMetadata:
return metadata.Path
name = metadata.Path
case source.FileMetadata:
return metadata.Path
default:
name = metadata.Path
}
// the SPDX document name is mandatory, and a source read from another SBOM may not have one
if name == "" {
return "unknown"
}
return name
}
syft/format/internal/spdxutil/helpers/document_name_test.go
+13 -3
View File
@@ -1,8 +1,6 @@
package helpers
import (
"fmt"
"strings"
"testing"
"github.com/stretchr/testify/assert"
@@ -73,11 +71,23 @@ func Test_DocumentName(t *testing.T) {
},
expected: "some/name",
},
{
name: "no name or metadata",
srcMetadata: source.Description{},
expected: "unknown",
},
{
name: "metadata without a name",
srcMetadata: source.Description{
Metadata: source.ImageMetadata{},
},
expected: "unknown",
},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
actual := DocumentName(test.srcMetadata)
assert.True(t, strings.HasPrefix(actual, test.expected), fmt.Sprintf("actual name %q", actual))
assert.Equal(t, test.expected, actual)
// track each scheme tested (passed or not)
tracker.Tested(t, test.srcMetadata.Metadata)
syft/format/purls/decoder.go
+5 -1
View File
@@ -8,6 +8,7 @@ import (
"strings"
"github.com/anchore/packageurl-go"
syftinternal "github.com/anchore/syft/internal"
"github.com/anchore/syft/internal/log"
"github.com/anchore/syft/syft/format/internal"
"github.com/anchore/syft/syft/pkg"
@@ -61,7 +62,7 @@ func toSyftModel(r io.Reader) (*sbom.SBOM, error) {
var errs []error
pkgs := pkg.NewCollection()
scanner := bufio.NewScanner(r)
scanner := syftinternal.NewLineScanner(r)
for scanner.Scan() {
line := strings.TrimSpace(scanner.Text())
@@ -84,6 +85,9 @@ func toSyftModel(r io.Reader) (*sbom.SBOM, error) {
p.SetID()
pkgs.Add(p)
}
if err := scanner.Err(); err != nil {
errs = append(errs, err)
}
return &sbom.SBOM{
Artifacts: sbom.Artifacts{
syft/format/spdxjson/decoder_test.go
+35
View File
@@ -5,6 +5,7 @@ import (
"io"
"os"
"path/filepath"
"strings"
"testing"
"github.com/stretchr/testify/assert"
@@ -183,3 +184,37 @@ func TestDecoder_Identify(t *testing.T) {
})
}
}
func TestDecoder_DecodeNullElements(t *testing.T) {
// a JSON null in these lists (or an SPDX 2.1 package the version converter cannot map) decodes to a nil pointer.
// a null entry directly in "packages" is not covered: for SPDX 2.2 and 2.3 it panics inside spdx/tools-golang
for _, doc := range []string{
`{"spdxVersion":"SPDX-2.1","packages":[{}]}`,
`{"spdxVersion":"SPDX-2.3","files":[null]}`,
`{"spdxVersion":"SPDX-2.3","packages":[{"SPDXID":"SPDXRef-a","name":"a","files":[null]}]}`,
`{"spdxVersion":"SPDX-2.3","packages":[{"SPDXID":"SPDXRef-a","name":"a","externalRefs":[null]}]}`,
} {
t.Run(doc, func(t *testing.T) {
assert.NotPanics(t, func() {
_, _, _, _ = NewFormatDecoder().Decode(strings.NewReader(doc))
})
})
}
}
func TestDecoder_DecodeFileRootWithoutSupplier(t *testing.T) {
// the package supplier is optional in SPDX, including on the package the document describes
f, err := os.Open("testdata/spdx/file-root-without-supplier.spdx.json")
require.NoError(t, err)
defer f.Close()
s, _, _, err := NewFormatDecoder().Decode(f)
require.NoError(t, err)
assert.Equal(t, "app.tar.gz", s.Source.Name)
var names []string
for _, p := range s.Artifacts.Packages.Sorted() {
names = append(names, p.Name)
}
assert.Equal(t, []string{"requests"}, names)
}
syft/format/spdxjson/encoder_test.go
+2 -23
View File
@@ -236,23 +236,6 @@ func TestSupportedVersions(t *testing.T) {
subject := testutil.DirectoryInput(t, t.TempDir())
dec := NewFormatDecoder()
relationshipOffsetPerVersion := map[string]int{
// the package representing the source gets a relationship from the source package to all other packages found
// these relationships cannot be removed until the primaryPackagePurpose info is available in 2.3
"2.1": 2,
"2.2": 2,
// the source-to-package relationships can be removed since the primaryPackagePurpose info is available in 2.3
"2.3": 0,
"3.0": 0,
}
pkgCountOffsetPerVersion := map[string]int{
"2.1": 1, // the source is mapped as a package, but cannot distinguish it since the primaryPackagePurpose info is not available until 2.3
"2.2": 1, // the source is mapped as a package, but cannot distinguish it since the primaryPackagePurpose info is not available until 2.3
"2.3": 0, // the source package can be removed since the primaryPackagePurpose info is available
"3.0": 0,
}
for _, enc := range encs {
t.Run(enc.Version(), func(t *testing.T) {
require.Contains(t, versions, enc.Version())
@@ -274,13 +257,9 @@ func TestSupportedVersions(t *testing.T) {
require.NotEmpty(t, s.Artifacts.Packages.PackageCount())
offset := relationshipOffsetPerVersion[enc.Version()]
assert.Equal(t, len(subject.Relationships), len(s.Relationships), "mismatched relationship count")
assert.Equal(t, len(subject.Relationships)+offset, len(s.Relationships), "mismatched relationship count")
offset = pkgCountOffsetPerVersion[enc.Version()]
if !assert.Equal(t, subject.Artifacts.Packages.PackageCount()+offset, s.Artifacts.Packages.PackageCount(), "mismatched package count") {
if !assert.Equal(t, subject.Artifacts.Packages.PackageCount(), s.Artifacts.Packages.PackageCount(), "mismatched package count") {
t.Logf("expected: %d", subject.Artifacts.Packages.PackageCount())
for _, p := range subject.Artifacts.Packages.Sorted() {
t.Logf(" - %s", p.String())
syft/format/spdxjson/testdata/snapshot/TestSPDX22JSONRequredProperties.golden
Vendored
+5 -5
View File
@@ -47,13 +47,13 @@
"copyrightText": "NOASSERTION"
},
{
"SPDXID": "SPDXRef-DocumentRoot-Unknown-",
"SPDXID": "SPDXRef-DocumentRoot-Unknown-unknown",
"copyrightText": "NOASSERTION",
"downloadLocation": "NOASSERTION",
"filesAnalyzed": false,
"licenseConcluded": "NOASSERTION",
"licenseDeclared": "NOASSERTION",
"name": "",
"name": "unknown",
"supplier": "NOASSERTION"
}
],
@@ -82,18 +82,18 @@
"relationshipType": "CONTAINS"
},
{
"spdxElementId": "SPDXRef-DocumentRoot-Unknown-",
"spdxElementId": "SPDXRef-DocumentRoot-Unknown-unknown",
"relatedSpdxElement": "SPDXRef-Package-files-analyzed-false-7d37ba9d2f7c574b",
"relationshipType": "CONTAINS"
},
{
"spdxElementId": "SPDXRef-DocumentRoot-Unknown-",
"spdxElementId": "SPDXRef-DocumentRoot-Unknown-unknown",
"relatedSpdxElement": "SPDXRef-Package-files-analyzed-true-035066c2086b8bb4",
"relationshipType": "CONTAINS"
},
{
"spdxElementId": "SPDXRef-DOCUMENT",
"relatedSpdxElement": "SPDXRef-DocumentRoot-Unknown-",
"relatedSpdxElement": "SPDXRef-DocumentRoot-Unknown-unknown",
"relationshipType": "DESCRIBES"
}
]
syft/format/spdxjson/testdata/spdx/file-root-without-supplier.spdx.json
Vendored
+29
View File
@@ -0,0 +1,29 @@
{
"spdxVersion": "SPDX-2.3",
"dataLicense": "CC0-1.0",
"SPDXID": "SPDXRef-DOCUMENT",
"name": "app.tar.gz",
"documentNamespace": "https://example.com/spdx/app-1.0",
"creationInfo": { "created": "2026-09-24T00:00:00Z", "creators": ["Tool: example-1.0"] },
"packages": [
{
"SPDXID": "SPDXRef-app",
"name": "app.tar.gz",
"downloadLocation": "NOASSERTION",
"filesAnalyzed": false,
"primaryPackagePurpose": "FILE"
},
{
"SPDXID": "SPDXRef-requests",
"name": "requests",
"versionInfo": "2.31.0",
"downloadLocation": "NOASSERTION",
"filesAnalyzed": false,
"externalRefs": [{ "referenceCategory": "PACKAGE-MANAGER", "referenceType": "purl", "referenceLocator": "pkg:pypi/requests@2.31.0" }]
}
],
"relationships": [
{ "spdxElementId": "SPDXRef-DOCUMENT", "relationshipType": "DESCRIBES", "relatedSpdxElement": "SPDXRef-app" },
{ "spdxElementId": "SPDXRef-app", "relationshipType": "CONTAINS", "relatedSpdxElement": "SPDXRef-requests" }
]
}
syft/format/spdxtagvalue/encoder_test.go
+39 -21
View File
@@ -3,6 +3,7 @@ package spdxtagvalue
import (
"bytes"
"flag"
"fmt"
"testing"
"github.com/stretchr/testify/assert"
@@ -143,21 +144,6 @@ func TestSupportedVersions(t *testing.T) {
subject := testutil.DirectoryInput(t, t.TempDir())
dec := NewFormatDecoder()
relationshipOffsetPerVersion := map[string]int{
// the package representing the source gets a relationship from the source package to all other packages found
// these relationships cannot be removed until the primaryPackagePurpose info is available in 2.3
"2.1": 2,
"2.2": 2,
// the source-to-package relationships can be removed since the primaryPackagePurpose info is available in 2.3
"2.3": 0,
}
pkgCountOffsetPerVersion := map[string]int{
"2.1": 1, // the source is mapped as a package, but cannot distinguish it since the primaryPackagePurpose info is not available until 2.3
"2.2": 1, // the source is mapped as a package, but cannot distinguish it since the primaryPackagePurpose info is not available until 2.3
"2.3": 0, // the source package can be removed since the primaryPackagePurpose info is available
}
for _, enc := range encs {
t.Run(enc.Version(), func(t *testing.T) {
require.Contains(t, versions, enc.Version())
@@ -178,13 +164,9 @@ func TestSupportedVersions(t *testing.T) {
require.NotEmpty(t, s.Artifacts.Packages.PackageCount())
offset := relationshipOffsetPerVersion[enc.Version()]
assert.Equal(t, len(subject.Relationships), len(s.Relationships), "mismatched relationship count")
assert.Equal(t, len(subject.Relationships)+offset, len(s.Relationships), "mismatched relationship count")
offset = pkgCountOffsetPerVersion[enc.Version()]
if !assert.Equal(t, subject.Artifacts.Packages.PackageCount()+offset, s.Artifacts.Packages.PackageCount(), "mismatched package count") {
if !assert.Equal(t, subject.Artifacts.Packages.PackageCount(), s.Artifacts.Packages.PackageCount(), "mismatched package count") {
t.Logf("expected: %d", subject.Artifacts.Packages.PackageCount())
for _, p := range subject.Artifacts.Packages.Sorted() {
t.Logf(" - %s", p.String())
@@ -209,3 +191,39 @@ func defaultFormatEncoders() []sbom.FormatEncoder {
}
return encs
}
func TestEncodeDecodeSourceWithoutName(t *testing.T) {
// a source read from another SBOM (a purl list, a CycloneDX document without metadata.component) may have no name,
// but the SPDX package name is mandatory and the tag-value decoder needs it to start the root package section
p := pkg.Package{Name: "left-pad", Version: "1.3.0", Type: pkg.NpmPkg}
p.SetID()
for _, src := range []source.Description{
{},
{Metadata: source.FileMetadata{}},
{Metadata: source.DirectoryMetadata{}},
{Metadata: source.ImageMetadata{}},
{Metadata: source.OCIModelMetadata{}},
{Metadata: source.SnapMetadata{}},
} {
for _, enc := range defaultFormatEncoders() {
t.Run(fmt.Sprintf("%T/%s", src.Metadata, enc.Version()), func(t *testing.T) {
var buf bytes.Buffer
require.NoError(t, enc.Encode(&buf, sbom.SBOM{
Source: src,
Artifacts: sbom.Artifacts{Packages: pkg.NewCollection(p)},
}))
s, _, _, err := NewFormatDecoder().Decode(bytes.NewReader(buf.Bytes()))
require.NoError(t, err, "encoded document:\n%s", buf.String())
var names []string
for _, decoded := range s.Artifacts.Packages.Sorted() {
names = append(names, decoded.Name)
}
// the root package describes the source, so it must not come back as a package of the SBOM
assert.Equal(t, []string{"left-pad"}, names)
})
}
}
}
syft/format/untyped_metadata_test.go
+94
View File
@@ -0,0 +1,94 @@
package format
import (
"bytes"
"fmt"
"math"
"testing"
"github.com/anchore/syft/syft/format/syftjson"
"github.com/anchore/syft/syft/pkg"
"github.com/anchore/syft/syft/sbom"
"github.com/anchore/syft/syft/source"
)
// TestEncodersHandleUntypedMetadataValues pushes awkward values through the untyped metadata fields
// allowlisted in internal/packagemetadata (see untypedFieldAllowlist) and asserts that SetID and every
// encoder neither panic nor fail. A new allowlist entry should get a case here too.
func TestEncodersHandleUntypedMetadataValues(t *testing.T) {
cases := []struct {
name string
value any
// encoding/json rejects non-finite floats, so syft-json returns an error for them. Producers
// have to replace these before they reach the model (the GGUF cataloger does); the case stays
// here so the expectation flips visibly if syft-json ever learns to encode them.
syftJSONRejects bool
}{
{name: "NaN", value: math.NaN(), syftJSONRejects: true},
{name: "+Inf float32", value: float32(math.Inf(1)), syftJSONRejects: true},
{name: "-Inf", value: math.Inf(-1), syftJSONRejects: true},
{name: "NaN in array", value: []any{1.0, math.NaN()}, syftJSONRejects: true},
{name: "invalid utf-8", value: "\xff\xfe"},
{name: "NUL and control characters", value: "a\x00b\x1b"},
{name: "max uint64", value: uint64(math.MaxUint64)},
{name: "deep array", value: nested(2000, func(v any) any { return []any{v} })},
{name: "deep map", value: nested(2000, func(v any) any { return map[string]any{"k": v} })},
{name: "large array", value: make([]any, 100_000)},
}
for _, tt := range cases {
t.Run(tt.name, func(t *testing.T) {
p := pkg.Package{
Name: "model",
Version: "1.0",
Type: pkg.ModelPkg,
Metadata: pkg.GGUFFileHeader{
RemainingKeyValues: map[string]any{"general.value": tt.value},
},
}
if !noPanic(t, "SetID", p.SetID) {
return
}
s := sbom.SBOM{
Artifacts: sbom.Artifacts{Packages: pkg.NewCollection(p)},
Source: source.Description{Metadata: source.DirectoryMetadata{Path: "/models"}},
}
for _, enc := range Encoders() {
id := fmt.Sprintf("%s@%s", enc.ID(), enc.Version())
var buf bytes.Buffer
noPanic(t, id, func() {
err := enc.Encode(&buf, s)
switch {
case tt.syftJSONRejects && enc.ID() == syftjson.ID:
if err == nil {
t.Errorf("%s: expected an encode error for a non-finite float", id)
}
case err != nil:
t.Errorf("%s: encode failed: %v", id, err)
}
})
}
})
}
}
func nested(depth int, wrap func(any) any) any {
var v any = "leaf"
for range depth {
v = wrap(v)
}
return v
}
func noPanic(t *testing.T, what string, fn func()) (ok bool) {
t.Helper()
defer func() {
if r := recover(); r != nil {
t.Errorf("%s: panic: %v", what, r)
ok = false
}
}()
fn()
return true
}
syft/internal/fileresolver/chroot_context.go
+4 -1
View File
@@ -216,5 +216,8 @@ func (r ChrootContext) ToChrootPath(nativePath string) string {
responsePath = strings.TrimPrefix(responsePath, prefix)
}
return responsePath
// chroot paths are posix regardless of host (these end up as location paths in the SBOM). This uses
// filepath (not path) since responsePath is native here (FromPosix above, then trimmed of a native prefix),
// and only filepath.ToSlash knows the host separator to convert.
return filepath.ToSlash(responsePath)
}
syft/internal/fileresolver/chroot_context_test.go
+7
View File
@@ -3,6 +3,7 @@ package fileresolver
import (
"os"
"path/filepath"
"runtime"
"testing"
"github.com/stretchr/testify/assert"
@@ -10,6 +11,9 @@ import (
)
func Test_ChrootContext_RequestResponse(t *testing.T) {
if runtime.GOOS == "windows" {
t.Skip("fixture relies on symlinks, which git does not check out as links on windows by default")
}
// /
// somewhere/
// outside.txt
@@ -563,6 +567,9 @@ func TestNewChrootContext_BaseValidation(t *testing.T) {
}
func TestToNativeGlob(t *testing.T) {
if runtime.GOOS == "windows" {
t.Skip("chroot context and expectations are posix paths")
}
tests := []struct {
name string
chrootContext ChrootContext
syft/internal/fileresolver/directory_indexer_test.go
+37 -11
View File
@@ -4,7 +4,6 @@ import (
"fmt"
"io/fs"
"os"
"path"
"path/filepath"
"runtime"
"sort"
@@ -18,6 +17,7 @@ import (
"github.com/wagoodman/go-progress"
"github.com/anchore/stereoscope/pkg/file"
"github.com/anchore/syft/syft/internal/windows"
)
type indexerMock struct {
@@ -138,6 +138,9 @@ func TestDirectoryIndexer_handleFileAccessErr(t *testing.T) {
}
func TestDirectoryIndexer_IncludeRootPathInIndex(t *testing.T) {
if runtime.GOOS == "windows" {
t.Skip("indexes the posix filesystem root")
}
filterFn := func(_, path string, _ os.FileInfo, _ error) error {
if path != "/" {
return fs.SkipDir
@@ -206,11 +209,7 @@ func TestDirectoryIndexer_index(t *testing.T) {
info, err := os.Stat(test.path)
assert.NoError(t, err)
// note: the index uses absolute paths, so assertions MUST keep this in mind
cwd, err := os.Getwd()
require.NoError(t, err)
p := file.Path(path.Join(cwd, test.path))
p := indexedPath(t, test.path)
assert.Equal(t, true, tree.HasPath(p))
exists, ref, err := tree.File(p)
assert.Equal(t, true, exists)
@@ -226,6 +225,9 @@ func TestDirectoryIndexer_index(t *testing.T) {
}
func TestDirectoryIndexer_index_for_AncestorSymlinks(t *testing.T) {
if runtime.GOOS == "windows" {
t.Skip("fixture relies on symlinks, which git does not check out as links on windows by default")
}
// note: this test is testing the effects from NewFromDirectory, indexTree, and addPathToIndex
_, filename, _, ok := runtime.Caller(0)
require.True(t, ok)
@@ -256,11 +258,7 @@ func TestDirectoryIndexer_index_for_AncestorSymlinks(t *testing.T) {
info, err := os.Stat(test.path)
assert.NoError(t, err)
// note: the index uses absolute paths, so assertions MUST keep this in mind
cwd, err := os.Getwd()
require.NoError(t, err)
p := file.Path(path.Join(cwd, test.path))
p := indexedPath(t, test.path)
assert.Equal(t, true, tree.HasPath(p))
exists, ref, err := tree.File(p)
assert.Equal(t, true, exists)
@@ -287,6 +285,9 @@ func TestDirectoryIndexer_index_survive_badSymlink(t *testing.T) {
}
func TestDirectoryIndexer_index_survive_inaccessibleSymlinkTarget(t *testing.T) {
if runtime.GOOS == "windows" {
t.Skip("relies on symlinks and posix permissions")
}
// a symlink that resolves into a directory we don't have permission to traverse should be skipped
// with a warning like any other inaccessible path, not abort the entire scan (see #3286).
if os.Geteuid() == 0 {
@@ -323,6 +324,9 @@ func TestDirectoryIndexer_index_survive_inaccessibleSymlinkTarget(t *testing.T)
}
func TestDirectoryIndexer_SkipsAlreadyVisitedLinkDestinations(t *testing.T) {
if runtime.GOOS == "windows" {
t.Skip("fixture relies on symlinks, which git does not check out as links on windows by default")
}
var observedPaths []string
pathObserver := func(_, p string, _ os.FileInfo, _ error) error {
fields := strings.Split(p, "testdata/symlinks-prune-indexing")
@@ -430,6 +434,11 @@ func TestDirectoryIndexer_IndexesAllTypes(t *testing.T) {
}
func Test_allContainedPaths(t *testing.T) {
if runtime.GOOS == "windows" {
// ponytail: production passes native windows paths here, which these posix cases don't represent.
// Ancestor indexing for a root reached through a symlink is likely broken on windows; fix separately.
t.Skip("cases are posix paths")
}
tests := []struct {
name string
@@ -485,6 +494,9 @@ func Test_allContainedPaths(t *testing.T) {
}
func Test_relativePath(t *testing.T) {
if runtime.GOOS == "windows" {
t.Skip("exercises a posix-only test helper")
}
tests := []struct {
name string
basePath string
@@ -548,3 +560,17 @@ func relativePath(basePath, givenPath string) string {
return relPath
}
// indexedPath returns how the given cwd-relative path is keyed in the index (absolute and always posix)
func indexedPath(t *testing.T, rel string) file.Path {
t.Helper()
cwd, err := os.Getwd()
require.NoError(t, err)
// filepath (not path): cwd is a native host path, so it must be joined with the host separator before
// being converted to the posix form the index is keyed on
p := filepath.Join(cwd, rel)
if windows.HostRunningOnWindows() {
p = windows.ToPosix(p)
}
return file.Path(p)
}
syft/internal/fileresolver/directory_roundtrip_test.go
+48
View File
@@ -0,0 +1,48 @@
package fileresolver
import (
"context"
"os"
"path/filepath"
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
stereoscopeFile "github.com/anchore/stereoscope/pkg/file"
)
// a location handed out by the resolver should be resolvable again by its own path. This is what the file
// digest cataloger relies on, and it silently found nothing on windows (see #5325).
func TestDirectory_LocationPathsRoundTrip(t *testing.T) {
// resolve symlinks up front (e.g. /var -> /private/var on macOS) so the cwd prefix matches the indexed paths
root, err := filepath.EvalSymlinks(t.TempDir())
require.NoError(t, err)
require.NoError(t, os.MkdirAll(filepath.Join(root, "sub", "dir"), 0o755))
for _, p := range []string{"top.txt", filepath.Join("sub", "mid.txt"), filepath.Join("sub", "dir", "deep.txt")} {
require.NoError(t, os.WriteFile(filepath.Join(root, p), []byte(p), 0o600))
}
// mirror `syft dir:.` from within the scanned directory
t.Chdir(root)
resolver, err := NewFromDirectory(".", "")
require.NoError(t, err)
var regular int
for loc := range resolver.AllLocations(context.Background()) {
md, err := resolver.FileMetadataByLocation(loc)
require.NoError(t, err)
if md.Type != stereoscopeFile.TypeRegular {
continue
}
regular++
assert.NotContains(t, loc.RealPath, `\`, "location paths should be posix")
found, err := resolver.FilesByPath(loc.RealPath)
require.NoError(t, err)
assert.Len(t, found, 1, "unable to resolve %q by its own path", loc.RealPath)
}
assert.Equal(t, 3, regular)
}
syft/internal/fileresolver/directory_windows_test.go
-126
View File
@@ -1,126 +0,0 @@
package fileresolver
import "testing"
func Test_windowsToPosix(t *testing.T) {
type args struct {
windowsPath string
}
tests := []struct {
name string
args args
wantPosixPath string
}{
{
name: "basic case",
args: args{
windowsPath: `C:\some\windows\place`,
},
wantPosixPath: "/c/some/windows/place",
},
{
name: "escaped case",
args: args{
windowsPath: `C:\\some\\windows\\place`,
},
wantPosixPath: "/c/some/windows/place",
},
{
name: "forward slash",
args: args{
windowsPath: `C:/foo/bar`,
},
wantPosixPath: "/c/foo/bar",
},
{
name: "mix slash",
args: args{
windowsPath: `C:\foo/bar\`,
},
wantPosixPath: "/c/foo/bar",
},
{
name: "case sensitive case",
args: args{
windowsPath: `C:\Foo/bAr\`,
},
wantPosixPath: "/c/Foo/bAr",
},
{
name: "special char case",
args: args{
windowsPath: `C:\ふー\バー`,
},
wantPosixPath: "/c/ふー/バー",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
if gotPosixPath := windowsToPosix(tt.args.windowsPath); gotPosixPath != tt.wantPosixPath {
t.Errorf("windowsToPosix() = %v, want %v", gotPosixPath, tt.wantPosixPath)
}
})
}
}
func Test_posixToWindows(t *testing.T) {
type args struct {
posixPath string
}
tests := []struct {
name string
args args
wantWindowsPath string
}{
{
name: "basic case",
args: args{
posixPath: "/c/some/windows/place",
},
wantWindowsPath: `C:\some\windows\place`,
},
{
name: "escaped case",
args: args{
posixPath: "/c/some/windows/place",
},
wantWindowsPath: `C:\\some\\windows\\place`,
},
{
name: "forward slash",
args: args{
posixPath: "/c/foo/bar",
},
wantWindowsPath: `C:/foo/bar`,
},
{
name: "mix slash",
args: args{
posixPath: "/c/foo/bar",
},
wantWindowsPath: `C:\foo/bar\`,
},
{
name: "case sensitive case",
args: args{
posixPath: "/c/Foo/bAr",
},
wantWindowsPath: `C:\Foo/bAr\`,
},
{
name: "special char case",
args: args{
posixPath: "/c/ふー/バー",
},
wantWindowsPath: `C:\ふー\バー`,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
if gotWindowsPath := posixToWindows(tt.args.posixPath); gotWindowsPath != tt.wantWindowsPath {
t.Errorf("posixToWindows() = %v, want %v", gotWindowsPath, tt.wantWindowsPath)
}
})
}
}
syft/internal/fileresolver/file_indexer.go
+5 -1
View File
@@ -29,11 +29,15 @@ func newFileIndexer(path, base string, visitors ...PathIndexVisitor) *fileIndexe
base: base,
tree: filetree.New(),
index: filetree.NewIndex(),
// Unlike the directory indexer there is no skipPathsByMountTypeAndName here: it exists
// to keep a directory walk out of /proc, /sys and similar, and to do so it reads the
// whole mount table on construction. A file source indexes exactly one file and its
// parent, so there is nothing to walk into, and the mount table read is paid for
// every file source with nothing in return.
pathIndexVisitors: append(
[]PathIndexVisitor{
requireFileInfo,
disallowByFileType,
skipPathsByMountTypeAndName(path),
},
visitors...,
),
syft/internal/fileresolver/file_indexer_bench_test.go
+51
View File
@@ -0,0 +1,51 @@
package fileresolver
import (
"os"
"path/filepath"
"testing"
"github.com/moby/sys/mountinfo"
)
// BenchmarkNewFromFile is the cost of building a resolver for a single-file source, which is
// paid once per file by callers that catalog files individually. The default case is
// NewFromFile as shipped. The with-mount-skipper case attaches skipPathsByMountTypeAndName
// the way the file indexer used to, so the difference is what reading the mount table for
// every file source cost. That cost scales with the mount table, so its size is reported as
// the mounts metric: expect a bigger gap on a container host than on a laptop.
//
// go test ./syft/internal/fileresolver/ -run '^$' -bench NewFromFile -benchmem -count 6 | tee bench.txt
// benchstat -col /variant bench.txt
func BenchmarkNewFromFile(b *testing.B) {
path := filepath.Join(b.TempDir(), "file.txt")
if err := os.WriteFile(path, []byte("hello\n"), 0o644); err != nil {
b.Fatal(err)
}
mounts, err := mountinfo.GetMounts(nil)
if err != nil {
b.Fatal(err)
}
b.Run("variant=default", func(b *testing.B) {
for b.Loop() {
if _, err := NewFromFile(path); err != nil {
b.Fatal(err)
}
}
b.ReportMetric(float64(len(mounts)), "mounts")
})
b.Run("variant=with-mount-skipper", func(b *testing.B) {
for b.Loop() {
r, err := newFromFileWithoutIndex(path, skipPathsByMountTypeAndName(path))
if err != nil {
b.Fatal(err)
}
if err := r.buildIndex(); err != nil {
b.Fatal(err)
}
}
b.ReportMetric(float64(len(mounts)), "mounts")
})
}
syft/internal/fileresolver/file_indexer_test.go
+7 -10
View File
@@ -3,13 +3,11 @@ package fileresolver
import (
"io/fs"
"os"
"path"
"path/filepath"
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/anchore/stereoscope/pkg/file"
)
// - Verify that both the parent and the path are indexed
@@ -38,11 +36,7 @@ func Test_index(t *testing.T) {
info, err := os.Stat(test.path)
assert.NoError(t, err)
// note: the index uses absolute paths, so assertions MUST keep this in mind
cwd, err := os.Getwd()
require.NoError(t, err)
p := file.Path(path.Join(cwd, test.path))
p := indexedPath(t, test.path)
assert.Equal(t, true, tree.HasPath(p))
exists, ref, err := tree.File(p)
assert.Equal(t, true, exists)
@@ -71,7 +65,9 @@ func Test_ignoresPathIfFiltered(t *testing.T) {
testPath := "testdata/system_paths/target/home/place"
cwd, cwdErr := os.Getwd()
require.NoError(t, cwdErr)
ignorePath := path.Join(cwd, testPath)
// filepath (not path): filters see the native absolute path (before posix normalization), so the
// expected value must be joined with the host separator to match on windows
ignorePath := filepath.Join(cwd, testPath)
filterFn := func(_, path string, _ os.FileInfo, _ error) error {
if path == ignorePath {
return ErrSkipPath
@@ -91,7 +87,8 @@ func Test_ignoresPathIfParentFiltered(t *testing.T) {
cwd, cwdErr := os.Getwd()
require.NoError(t, cwdErr)
ignorePath := path.Join(cwd, parentPath)
// filepath (not path): see Test_ignoresPathIfFiltered
ignorePath := filepath.Join(cwd, parentPath)
filterFn := func(_, path string, _ os.FileInfo, _ error) error {
if path == ignorePath {
return fs.SkipDir
syft/internal/fileresolver/filetree_resolver.go
+23 -6
View File
@@ -6,6 +6,7 @@ import (
"fmt"
"io"
"os"
"path/filepath"
stereoscopeFile "github.com/anchore/stereoscope/pkg/file"
"github.com/anchore/stereoscope/pkg/filetree"
@@ -34,8 +35,17 @@ func nativeOSFileOpener(ref stereoscopeFile.Reference) (io.ReadCloser, error) {
return stereoscopeFile.NewLazyReadCloser(filePath), nil
}
// requestPath converts a user (chroot) path into the path the file tree is keyed on. The tree is always posix
// (windows paths are volume-encoded, e.g. /c/some/path), while ToNativePath yields a native host path.
func (r *FiletreeResolver) requestPath(userPath string) (string, error) {
return r.Chroot.ToNativePath(userPath)
nativePath, err := r.Chroot.ToNativePath(userPath)
if err != nil {
return "", err
}
if windows.HostRunningOnWindows() {
return windows.ToPosix(nativePath), nil
}
return nativePath, nil
}
// responsePath takes a path from the underlying fs domain and converts it to a path that is relative to the root of the file resolver.
@@ -85,10 +95,6 @@ func (r FiletreeResolver) FilesByPath(userPaths ...string) ([]file.Location, err
continue
}
if windows.HostRunningOnWindows() {
userStrPath = windows.ToPosix(userStrPath)
}
if ref.HasReference() {
references = append(references,
file.NewVirtualLocationFromDirectory(
@@ -103,8 +109,19 @@ func (r FiletreeResolver) FilesByPath(userPaths ...string) ([]file.Location, err
return references, nil
}
// requestGlob is the glob equivalent of requestPath.
func (r FiletreeResolver) requestGlob(pattern string) (string, error) {
return r.Chroot.ToNativeGlob(pattern)
glob, err := r.Chroot.ToNativeGlob(pattern)
if err != nil {
return "", err
}
// only a glob that was anchored to the root comes back native. Globs that start with a wildcard
// (e.g. **/foo) are returned untouched and are already posix, and ToPosix would root them at "/".
// This uses filepath (not path) since we are asking whether this is a native absolute path.
if windows.HostRunningOnWindows() && filepath.IsAbs(glob) {
return windows.ToPosix(glob), nil
}
return glob, nil
}
// FilesByGlob returns all file.References that match the given path glob pattern from any layer in the image.
syft/internal/parsing/basic_parsing.go
+24 -7
View File
@@ -22,6 +22,15 @@ func SkipWhitespace(data []byte, i *int) {
}
}
// MaxDepth bounds nesting in the recursive hand-written parsers. Real files nest a handful of levels; without a
// bound, deep input overflows the goroutine stack, which is fatal rather than a recoverable panic.
const MaxDepth = 1000
// errorContext is how many bytes of each echoed line PrintError keeps around the error column, so an error in a
// huge single-line input stays small (and does not copy the file into the SBOM as an unknown).
const errorContext = 80
// PrintError describes where in data offset i is, echoing up to two lines of context clipped around the column.
func PrintError(data []byte, i int) string {
line := 1
char := 1
@@ -57,15 +66,23 @@ func PrintError(data []byte, i int) string {
sep := ": "
lines := ""
if len(prev) > 1 {
lines += fmt.Sprintf("%s%s%s\n", l1, sep, prev[len(prev)-2])
}
if len(prev) > 0 {
lines += fmt.Sprintf("%s%s%s\n", l2, sep, prev[len(prev)-1])
start := max(0, char-1-errorContext/2)
clip := func(s string) string {
if start >= len(s) {
return ""
}
return s[start:min(len(s), start+errorContext)]
}
pointer := strings.Repeat(" ", len(l2)+len(sep)+char-1) + "^"
lines := ""
if len(prev) > 1 {
lines += fmt.Sprintf("%s%s%s\n", l1, sep, clip(prev[len(prev)-2]))
}
if len(prev) > 0 {
lines += fmt.Sprintf("%s%s%s\n", l2, sep, clip(prev[len(prev)-1]))
}
pointer := strings.Repeat(" ", len(l2)+len(sep)+char-1-start) + "^"
return fmt.Sprintf("line: %v, char: %v\n%s%s", line, char, lines, pointer)
}
syft/internal/windows/path.go
+8
View File
@@ -35,7 +35,15 @@ func ToPosix(windowsPath string) (posixPath string) {
func FromPosix(posixPath string) (windowsPath string) {
// decode the volume (e.g. /c/<path> --> C:\\) - There should always be a volume name.
// The volume may be a UNC path (e.g. /\\localhost\C$\ --> \\localhost\C$\)
if !strings.HasPrefix(posixPath, "/") {
// not a volume-encoded posix path (e.g. relative or already native), so there is nothing to decode
return posixPath
}
pathFields := strings.Split(posixPath, "/")
if len(pathFields) < 2 {
// no separator means no volume to decode
return filepath.Clean(posixPath)
}
rootPath := strings.ToUpper(pathFields[1])
volumeName := AppendRootTerminator(rootPath)
syft/internal/windows/path_windows_test.go
+35
View File
@@ -0,0 +1,35 @@
package windows
import (
"testing"
"github.com/stretchr/testify/assert"
)
func TestToPosix(t *testing.T) {
tests := map[string]string{
`C:\some\windows\place`: "/c/some/windows/place",
`C:\\some\\windows\\place`: "/c/some/windows/place",
`C:/foo/bar`: "/c/foo/bar",
`C:\foo/bar\`: "/c/foo/bar",
`C:\Foo/bAr\`: "/c/Foo/bAr",
`C:\ふー\バー`: "/c/ふー/バー",
}
for in, want := range tests {
assert.Equal(t, want, ToPosix(in), in)
}
}
func TestFromPosix(t *testing.T) {
tests := map[string]string{
"/c/some/windows/place": `C:\some\windows\place`,
"/c/Foo/bAr": `C:\Foo\bAr`,
"/c/ふー/バー": `C:\ふー\バー`,
// non volume-encoded input is left alone
"relative/path": "relative/path",
`D:\already\native`: `D:\already\native`,
}
for in, want := range tests {
assert.Equal(t, want, FromPosix(in), in)
}
}
syft/linux/identify_release.go
+2 -2
View File
@@ -156,8 +156,8 @@ var busyboxVersionMatcher = regexp.MustCompile(`BusyBox v[\d.]+`)
func parseBusyBox(contents string) (*Release, error) {
matches := busyboxVersionMatcher.FindAllString(contents, -1)
for _, match := range matches {
parts := strings.Split(match, " ")
version := strings.ReplaceAll(parts[1], "v", "")
_, version, _ := strings.Cut(match, " ")
version = strings.ReplaceAll(version, "v", "")
return simpleRelease(match, "busybox", version, ""), nil
}
syft/pkg/cataloger/ai/cataloger.go
+6 -2
View File
@@ -11,8 +11,12 @@ import (
const (
catalogerName = "gguf-cataloger"
ggufLayerMediaType = "application/vnd.docker.ai*"
safeTensorsCatalogerName = "safetensors-cataloger"
// dockerAIGGUFMediaType mirrors ggufLayerMediaType in
// syft/source/ocimodelsource/registry_client.go. It is matched exactly: the
// other application/vnd.docker.ai.* layers are not GGUF.
dockerAIGGUFMediaType = "application/vnd.docker.ai.gguf.v3"
)
// NewGGUFCataloger returns a new cataloger instance for GGUF model files.
@@ -21,7 +25,7 @@ const (
func NewGGUFCataloger() pkg.Cataloger {
return generic.NewCataloger(catalogerName).
WithParserByGlobs(parseGGUFModel, "**/*.gguf").
WithParserByMediaType(parseGGUFModel, ggufLayerMediaType).
WithParserByMediaType(parseGGUFModel, dockerAIGGUFMediaType).
WithProcessors(ggufMergeProcessor)
}
syft/pkg/cataloger/ai/cataloger_test.go
+45
View File
@@ -1,11 +1,17 @@
package ai
import (
"context"
"os"
"path/filepath"
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/anchore/syft/internal/tmpdir"
"github.com/anchore/syft/syft/artifact"
"github.com/anchore/syft/syft/internal/fileresolver"
"github.com/anchore/syft/syft/pkg"
"github.com/anchore/syft/syft/pkg/cataloger/internal/pkgtest"
)
@@ -138,3 +144,42 @@ func TestGGUFCataloger(t *testing.T) {
})
}
}
func TestGGUFCataloger_ociLayers(t *testing.T) {
// a GGUF OCI artifact is merged into one named package, while the other
// application/vnd.docker.ai.* layers (config, model.file, license and
// safetensors weights) never reach the GGUF parser.
named := newTestGGUFBuilder().
withStringKV("general.architecture", "llama").
withStringKV("general.name", "llama3-8b").
build()
part := newTestGGUFBuilder().withStringKV("general.architecture", "llama").build()
dir := t.TempDir()
layers := map[string]fileresolver.LayerInfo{}
for digest, l := range map[string]struct {
mediaType string
content []byte
}{
"sha256:named": {dockerAIGGUFMediaType, named},
"sha256:part": {dockerAIGGUFMediaType, part},
"sha256:config": {dockerAIModelConfigMediaTypes[0], []byte("not a gguf file")},
"sha256:file": {dockerAIModelFileMediaType, []byte("not a gguf file")},
"sha256:license": {dockerAILicenseMediaType, []byte("not a gguf file")},
"sha256:weights": {dockerAISafeTensorsMediaType, []byte("not a gguf file")},
} {
p := filepath.Join(dir, digest[len("sha256:"):])
require.NoError(t, os.WriteFile(p, l.content, 0o600))
layers[digest] = fileresolver.LayerInfo{TempPath: p, MediaType: l.mediaType}
}
resolver := fileresolver.NewContainerImageModel(dir, layers, "example/model:latest")
ctx := tmpdir.WithValue(context.Background(), tmpdir.FromPath(t.TempDir()))
pkgs, _, err := NewGGUFCataloger().Catalog(ctx, resolver)
require.NoError(t, err)
require.Len(t, pkgs, 1)
assert.Equal(t, "llama3-8b", pkgs[0].Name)
header, ok := pkgs[0].Metadata.(pkg.GGUFFileHeader)
require.True(t, ok)
assert.Len(t, header.Parts, 1)
}
syft/pkg/cataloger/ai/gguf_processor.go
+7 -7
View File
@@ -8,17 +8,17 @@ import (
// ggufMergeProcessor consolidates multiple GGUF packages into a single package
// representing the AI model. When scanning OCI images with multiple layers,
// each layer may produce a separate package. This processor finds the package
// with a name and merges metadata from nameless packages into its GGUFFileParts field.
// with a name and merges metadata from nameless packages into its Parts field.
// Only packages with a non-empty name are returned in the final result.
func ggufMergeProcessor(pkgs []pkg.Package, rels []artifact.Relationship, err error) ([]pkg.Package, []artifact.Relationship, error) {
if err != nil {
return pkgs, rels, err
}
// don't early-return on err: it aggregates per-file failures, and skipping the
// merge would drop every nameless layer and the named package's Parts along
// with it (see safeTensorsMergeProcessor).
if len(pkgs) == 0 {
return pkgs, rels, err
}
// Separate packages with names from those without
// separate packages with names from those without
var namedPkgs []pkg.Package
var namelessHeaders []pkg.GGUFFileHeader
@@ -27,14 +27,14 @@ func ggufMergeProcessor(pkgs []pkg.Package, rels []artifact.Relationship, err er
namedPkgs = append(namedPkgs, p)
} else {
if header, ok := p.Metadata.(pkg.GGUFFileHeader); ok {
// We do not want a kv hash for nameless headers
// we do not want a kv hash for nameless headers
header.MetadataKeyValuesHash = ""
namelessHeaders = append(namelessHeaders, header)
}
}
}
// If there are no named packages, return nothing
// if there are no named packages, return nothing
if len(namedPkgs) == 0 {
return nil, rels, err
}
syft/pkg/cataloger/ai/huggingface.go
+60 -20
View File
@@ -3,7 +3,7 @@ package ai
import (
"bytes"
"gopkg.in/yaml.v3"
"go.yaml.in/yaml/v3"
"github.com/anchore/syft/internal/log"
"github.com/anchore/syft/syft/pkg"
@@ -15,6 +15,12 @@ type hfConfig struct {
NameOrPath string `json:"_name_or_path"`
}
// looksLikeHF reports whether the config carries any field we use. Other JSON
// (generation_config.json, unrelated app configs) is ignored.
func (c hfConfig) looksLikeHF() bool {
return len(c.Architectures) > 0 || c.NameOrPath != ""
}
func applyHFConfig(md *pkg.SafeTensorsModelInfo, cfg *hfConfig) {
if md.Architecture == "" && len(cfg.Architectures) > 0 {
md.Architecture = cfg.Architectures[0]
@@ -23,8 +29,8 @@ func applyHFConfig(md *pkg.SafeTensorsModelInfo, cfg *hfConfig) {
// readmeFrontmatter holds the subset of YAML frontmatter fields we extract.
type readmeFrontmatter struct {
License string `yaml:"license"`
BaseModel []string `yaml:"base_model"`
Licenses []string
BaseModel []string
}
type licenseFrontmatter struct {
@@ -32,21 +38,33 @@ type licenseFrontmatter struct {
}
// extractFrontmatterBlock returns the YAML bytes between the first and second
// "---" delimiters of a file
// "---" delimiter lines of a file
func extractFrontmatterBlock(buf []byte) []byte {
trimmed := bytes.TrimLeft(buf, "\xef\xbb\xbf \t\r\n")
if !bytes.HasPrefix(trimmed, []byte("---")) {
first, rest, _ := bytes.Cut(trimmed, []byte("\n"))
if !isFrontmatterDelimiter(first) {
return nil
}
rest := trimmed[3:]
if i := bytes.IndexByte(rest, '\n'); i >= 0 {
rest = rest[i+1:]
for off := 0; off < len(rest); {
line, _, found := bytes.Cut(rest[off:], []byte("\n"))
if isFrontmatterDelimiter(line) {
if off == 0 || off > maxFrontmatterSize {
return nil
}
return rest[:off]
}
if !found {
break
}
off += len(line) + 1
}
block, _, found := bytes.Cut(rest, []byte("\n---"))
if !found {
return nil
}
return block
return nil
}
// isFrontmatterDelimiter reports whether a line is exactly "---", ignoring
// trailing whitespace and \r.
func isFrontmatterDelimiter(line []byte) bool {
return bytes.Equal(bytes.TrimRight(line, " \t\r"), []byte("---"))
}
// parseFrontmatter decodes a Hugging Face model card YAML frontmatter block
@@ -57,8 +75,9 @@ func parseFrontmatter(buf []byte) *readmeFrontmatter {
return nil
}
// both fields may be a scalar or a list, so decode them as nodes
var raw struct {
License string `yaml:"license"`
License yaml.Node `yaml:"license"`
BaseModel yaml.Node `yaml:"base_model"`
}
if err := yaml.Unmarshal(block, &raw); err != nil {
@@ -66,16 +85,37 @@ func parseFrontmatter(buf []byte) *readmeFrontmatter {
return nil
}
fm := readmeFrontmatter{License: raw.License}
switch raw.BaseModel.Kind {
return &readmeFrontmatter{
Licenses: yamlStrings(raw.License),
BaseModel: yamlStrings(raw.BaseModel),
}
}
// yamlStrings returns the non-empty, non-null values of a scalar or a sequence
// of scalars, keeping at most maxFrontmatterValues. Anything else yields nil.
func yamlStrings(n yaml.Node) []string {
switch n.Kind {
case yaml.ScalarNode:
if raw.BaseModel.Value != "" {
fm.BaseModel = []string{raw.BaseModel.Value}
if isYAMLString(&n) {
return []string{n.Value}
}
case yaml.SequenceNode:
_ = raw.BaseModel.Decode(&fm.BaseModel)
var out []string
for _, c := range n.Content {
if len(out) == maxFrontmatterValues {
break
}
if c.Kind == yaml.ScalarNode && isYAMLString(c) {
out = append(out, c.Value)
}
}
return out
}
return &fm
return nil
}
func isYAMLString(n *yaml.Node) bool {
return n.Value != "" && n.ShortTag() != "!!null"
}
// parseLicenseFrontmatter returns the producer-declared SPDX identifier
syft/pkg/cataloger/ai/identity_dir.go
+28 -20
View File
@@ -3,7 +3,6 @@ package ai
import (
"context"
"encoding/json"
"io"
"path"
"github.com/anchore/syft/internal"
@@ -13,42 +12,42 @@ import (
)
// resolveSafeTensorsDirIdentity handles the directory-scan case for safe tensors
// find config.json beside the model files (walking up parent directories to the
// scanned source root if no sibling exists) and a sibling README.md. It returns
// find config.json beside the model files (or one directory up, for the shard
// subdirectory layout) and a sibling README.md. It returns
// the group's name candidates, resolved licenses, and supporting evidence.
func resolveSafeTensorsDirIdentity(ctx context.Context, resolver file.Resolver, dir string, md *pkg.SafeTensorsModelInfo) safeTensorsIdentity {
id := safeTensorsIdentity{fallbackName: safeTensorsDirName(dir)}
if loc, cfg := findDirHFConfig(resolver, dir); cfg != nil {
applyHFConfig(md, cfg)
id.nameOrPath = cfg.NameOrPath
id.nameOrPath = sanitizeModelName(cfg.NameOrPath)
id.supporting = append(id.supporting, *loc)
}
if loc, fm := readDirReadmeFrontmatter(resolver, path.Join(dir, "README.md")); fm != nil {
if fm.License != "" {
id.licenses = pkg.NewLicensesFromValuesWithContext(ctx, fm.License)
if len(fm.Licenses) > 0 {
id.licenses = pkg.NewLicensesFromValuesWithContext(ctx, fm.Licenses...)
}
if id.nameOrPath == "" && len(fm.BaseModel) > 0 {
id.nameOrPath = fm.BaseModel[0]
if id.nameOrPath == "" {
id.nameOrPath = firstModelName(fm.BaseModel)
}
id.supporting = append(id.supporting, *loc)
}
return id
}
// findDirHFConfig looks for a config.json beside the model files
// findDirHFConfig looks for an HF config.json beside the model files, then in
// the parent directory (covers weights kept in a subdirectory of the model). It
// does not walk further up, so an unrelated config.json higher in the tree can't
// name every model below it.
func findDirHFConfig(resolver file.Resolver, dir string) (*file.Location, *hfConfig) {
for {
if loc, cfg := readDirHFConfig(resolver, path.Join(dir, "config.json")); cfg != nil {
return loc, cfg
}
parent := path.Dir(dir)
if parent == dir {
return nil, nil // reached the source root
}
dir = parent
if loc, cfg := readDirHFConfig(resolver, path.Join(dir, "config.json")); cfg != nil {
return loc, cfg
}
if parent := path.Dir(dir); parent != dir {
return readDirHFConfig(resolver, path.Join(parent, "config.json"))
}
return nil, nil
}
func readDirHFConfig(resolver file.Resolver, p string) (*file.Location, *hfConfig) {
@@ -62,11 +61,19 @@ func readDirHFConfig(resolver file.Resolver, p string) (*file.Location, *hfConfi
}
defer internal.CloseAndLogError(rc, p)
buf, err := readBounded(rc, maxHFConfigSize)
if err != nil {
log.Debugf("failed to read %s: %v", p, err)
return nil, nil
}
var cfg hfConfig
if err := json.NewDecoder(rc).Decode(&cfg); err != nil {
if err := json.Unmarshal(buf, &cfg); err != nil {
log.Debugf("failed to decode %s: %v", p, err)
return nil, nil
}
if !cfg.looksLikeHF() {
return nil, nil
}
return &locations[0], &cfg
}
@@ -81,8 +88,9 @@ func readDirReadmeFrontmatter(resolver file.Resolver, p string) (*file.Location,
}
defer internal.CloseAndLogError(rc, p)
buf, err := io.ReadAll(io.LimitReader(rc, 1024*1024))
buf, err := readPrefix(rc, maxReadmePrefixSize)
if err != nil {
log.Debugf("failed to read %s: %v", p, err)
return nil, nil
}
fm := parseFrontmatter(buf)
syft/pkg/cataloger/ai/identity_oci.go
+32 -20
View File
@@ -4,8 +4,8 @@ import (
"bytes"
"context"
"encoding/json"
"io"
"path"
"sort"
gcrname "github.com/google/go-containerregistry/pkg/name"
@@ -28,9 +28,18 @@ func resolveSafeTensorsOCIIdentity(ctx context.Context, resolver file.Resolver,
log.Debugf("failed to list docker AI model-file layers: %v", err)
}
// Collect config / readme candidates separately so the layer-iteration order
// returned by the resolver doesn't decide the precedence.
var configName, readmeName, readmeLicense string
// the resolver returns layers in map order, and the first config / README
// seen wins below, so sort for a stable result across runs. For OCI
// artifacts every RealPath is "/", so in practice this is digest order.
sort.Slice(modelFileLocs, func(i, j int) bool {
if modelFileLocs[i].RealPath != modelFileLocs[j].RealPath {
return modelFileLocs[i].RealPath < modelFileLocs[j].RealPath
}
return modelFileLocs[i].FileSystemID < modelFileLocs[j].FileSystemID
})
var configName, readmeName string
var readmeLicenses []string
var supporting []file.Location
for _, loc := range modelFileLocs {
cfg, fm := classifyOCIModelFileLayer(resolver, loc)
@@ -38,15 +47,15 @@ func resolveSafeTensorsOCIIdentity(ctx context.Context, resolver file.Resolver,
case cfg != nil:
applyHFConfig(md, cfg)
if configName == "" {
configName = cfg.NameOrPath
configName = sanitizeModelName(cfg.NameOrPath)
}
supporting = append(supporting, loc)
case fm != nil:
if readmeLicense == "" {
readmeLicense = fm.License
if len(readmeLicenses) == 0 {
readmeLicenses = fm.Licenses
}
if readmeName == "" && len(fm.BaseModel) > 0 {
readmeName = fm.BaseModel[0]
if readmeName == "" {
readmeName = firstModelName(fm.BaseModel)
}
supporting = append(supporting, loc)
}
@@ -64,7 +73,7 @@ func resolveSafeTensorsOCIIdentity(ctx context.Context, resolver file.Resolver,
supporting: supporting,
}
// License precedence: a dedicated vnd.docker.ai.license layer is a
// License precedence: a dedicated vnd.docker.ai.license layer
// outranks the free-text license field in a model card's README frontmatter.
licLocs, err := ociResolver.FilesByMediaType(dockerAILicenseMediaType)
if err != nil {
@@ -74,8 +83,8 @@ func resolveSafeTensorsOCIIdentity(ctx context.Context, resolver file.Resolver,
case len(licLocs) > 0:
id.licenses = identifyLicenseLayers(ctx, resolver, licLocs)
id.supporting = append(id.supporting, licLocs...)
case readmeLicense != "":
id.licenses = pkg.NewLicensesFromValuesWithContext(ctx, readmeLicense)
case len(readmeLicenses) > 0:
id.licenses = pkg.NewLicensesFromValuesWithContext(ctx, readmeLicenses...)
}
return id
@@ -108,10 +117,11 @@ func ociImageRefBasename(resolver file.Resolver) string {
return path.Base(parsed.Context().RepositoryStr())
}
// classifyOCIModelFileLayer reads up to 4 MiB of a model.file layer and decodes
// it as either an HF config.json or a README model card's YAML frontmatter,
// based on its leading bytes. It returns whichever it recognized; both are nil
// when the layer is neither (or fails to decode). The caller owns precedence and
// classifyOCIModelFileLayer reads a model.file layer (up to maxModelFileLayerSize)
// and decodes it as either an HF config.json or a README model card's YAML
// frontmatter, based on its leading bytes. It returns whichever it recognized;
// both are nil when the layer is neither, fails to decode, or is JSON that
// doesn't look like an HF config (e.g. generation_config.json). The caller owns precedence and
// metadata enrichment.
func classifyOCIModelFileLayer(resolver file.Resolver, loc file.Location) (*hfConfig, *readmeFrontmatter) {
rc, err := resolver.FileContentsByLocation(loc)
@@ -120,17 +130,18 @@ func classifyOCIModelFileLayer(resolver file.Resolver, loc file.Location) (*hfCo
}
defer internal.CloseAndLogError(rc, loc.RealPath)
buf, err := io.ReadAll(io.LimitReader(rc, 4*1024*1024))
buf, err := readBounded(rc, maxModelFileLayerSize)
if err != nil {
log.Debugf("failed to read docker AI model-file layer: %v", err)
return nil, nil
}
trimmed := bytes.TrimLeft(buf, "\xef\xbb\xbf \t\r\n")
switch {
case bytes.HasPrefix(trimmed, []byte("---")):
return nil, parseFrontmatter(buf)
return nil, parseFrontmatter(trimmed)
case bytes.HasPrefix(trimmed, []byte("{")):
var cfg hfConfig
if err := json.Unmarshal(buf, &cfg); err != nil {
if err := json.Unmarshal(trimmed, &cfg); err != nil || !cfg.looksLikeHF() {
return nil, nil
}
return &cfg, nil
@@ -165,8 +176,9 @@ func readLicenseSPDXIDFromFrontmatter(resolver file.Resolver, loc file.Location)
}
defer internal.CloseAndLogError(rc, loc.RealPath)
buf, err := io.ReadAll(io.LimitReader(rc, 64*1024))
buf, err := readPrefix(rc, maxLicensePrefixSize)
if err != nil {
log.Debugf("failed to read docker AI license layer: %v", err)
return ""
}
return parseLicenseFrontmatter(buf)
syft/pkg/cataloger/ai/limits.go
+46
View File
@@ -0,0 +1,46 @@
package ai
import (
"fmt"
"io"
"github.com/anchore/syft/internal"
)
// size limits for model companion files and headers. JSON reads that hit a
// limit fail (see readBounded) instead of parsing a truncated document.
// Frontmatter-only reads (README, license) take a prefix instead, since the
// frontmatter sits at the top and is itself capped by maxFrontmatterSize.
const (
maxHFConfigSize = 4 * 1024 * 1024
maxReadmePrefixSize = 1 * 1024 * 1024 // dir scans; OCI README layers use maxModelFileLayerSize
maxModelFileLayerSize = 4 * 1024 * 1024
maxLicensePrefixSize = 64 * 1024
maxModelConfigBlobSize = 1 * 1024 * 1024
maxFrontmatterSize = 256 * 1024
maxFrontmatterValues = 32
maxModelNameLength = 256
// maxSafeTensorsHeaderSize caps the header JSON (excluding its 8-byte length
// prefix). It is shared with the OCI source so dir and OCI scans accept the
// same headers.
maxSafeTensorsHeaderSize = internal.MaxSafeTensorsHeaderSize
)
// readPrefix reads at most limit bytes. It is for frontmatter-only reads, where
// the rest of the file is never needed.
func readPrefix(r io.Reader, limit int64) ([]byte, error) {
return io.ReadAll(io.LimitReader(r, limit))
}
// readBounded reads at most limit bytes and reports overflow instead of silently truncating.
func readBounded(r io.Reader, limit int64) ([]byte, error) {
b, err := io.ReadAll(io.LimitReader(r, limit+1))
if err != nil {
return nil, err
}
if int64(len(b)) > limit {
return nil, fmt.Errorf("content exceeds %d bytes", limit)
}
return b, nil
}
syft/pkg/cataloger/ai/merge.go
+1 -1
View File
@@ -75,7 +75,7 @@ func mergeShardsInto(merged *pkg.SafeTensorsModelInfo, shards []pkg.SafeTensorsM
seenKV := map[string]bool{}
for _, s := range shards {
shardTensorTotal += s.TensorCount
merged.Parameters += s.Parameters
merged.Parameters = saturatingAdd(merged.Parameters, s.Parameters)
firstNonEmpty(&merged.Quantization, s.Quantization)
for _, kv := range s.UserMetadata {
if seenKV[kv.Key] {
syft/pkg/cataloger/ai/merge_test.go
+8
View File
@@ -1,6 +1,7 @@
package ai
import (
"math"
"testing"
"github.com/stretchr/testify/assert"
@@ -164,3 +165,10 @@ func TestRollupHash(t *testing.T) {
assert.NotEqual(t, "a", ab)
assert.NotEqual(t, "b", ab)
}
func TestMergeShardsInto_parametersSaturate(t *testing.T) {
var merged pkg.SafeTensorsModelInfo
shards := []pkg.SafeTensorsModelInfo{{Parameters: math.MaxUint64 - 1}, {Parameters: 2}}
mergeShardsInto(&merged, shards)
assert.Equal(t, uint64(math.MaxUint64), merged.Parameters)
}
syft/pkg/cataloger/ai/naming.go
+32 -3
View File
@@ -1,15 +1,44 @@
package ai
import "path"
import (
"path"
"strings"
"unicode/utf8"
)
// pickSafeTensorsName implements the documented naming precedence chain:
func pickSafeTensorsName(nameOrPath, fallbackName string) string {
if nameOrPath != "" {
return path.Base(nameOrPath)
if name := sanitizeModelName(nameOrPath); name != "" {
return name
}
return fallbackName
}
// firstModelName returns the first value that sanitizes to a usable name.
func firstModelName(values []string) string {
for _, v := range values {
if name := sanitizeModelName(v); name != "" {
return name
}
}
return ""
}
// sanitizeModelName reduces a producer-supplied name or path (_name_or_path,
// base_model) to its last path element, capped at maxModelNameLength runes. It
// returns "" when nothing usable is left.
func sanitizeModelName(nameOrPath string) string {
base := path.Base(strings.ReplaceAll(nameOrPath, `\`, "/"))
switch base {
case "/", ".", "..", "":
return ""
}
if utf8.RuneCountInString(base) > maxModelNameLength {
base = string([]rune(base)[:maxModelNameLength])
}
return base
}
// safeTensorsDirName returns the directory-scan naming fallback: the base name
// of the group's parent directory (the group key is already that directory).
func safeTensorsDirName(directory string) string {
syft/pkg/cataloger/ai/package.go
+4 -2
View File
@@ -1,17 +1,19 @@
package ai
import (
"context"
"github.com/anchore/syft/syft/file"
"github.com/anchore/syft/syft/pkg"
)
func newGGUFPackage(metadata *pkg.GGUFFileHeader, modelName, version, license string, locations ...file.Location) pkg.Package {
func newGGUFPackage(ctx context.Context, metadata *pkg.GGUFFileHeader, modelName, version, license string, locations ...file.Location) pkg.Package {
p := pkg.Package{
Name: modelName,
Version: version,
Locations: file.NewLocationSet(locations...),
Type: pkg.ModelPkg,
Licenses: pkg.NewLicenseSet(pkg.NewLicensesFromValues(license)...),
Licenses: pkg.NewLicenseSet(pkg.NewLicensesFromValuesWithContext(ctx, license)...),
Metadata: *metadata,
// NOTE: PURL is intentionally not set as the package-url spec
// has not yet finalized support for ML model packages
syft/pkg/cataloger/ai/package_test.go
+2
View File
@@ -1,6 +1,7 @@
package ai
import (
"context"
"testing"
"github.com/stretchr/testify/require"
@@ -103,6 +104,7 @@ func TestNewGGUFPackage(t *testing.T) {
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
actual := newGGUFPackage(
context.Background(),
tt.metadata,
tt.input.modelName,
tt.input.version,
syft/pkg/cataloger/ai/parse_gguf.go
+300 -5
View File
@@ -1,9 +1,14 @@
package ai
import (
"bufio"
"encoding/binary"
"errors"
"fmt"
"io"
"math"
"slices"
"strings"
gguf_parser "github.com/gpustack/gguf-parser-go"
)
@@ -12,12 +17,41 @@ import (
const (
ggufMagicNumber = 0x46554747 // "GGUF" in little-endian
maxHeaderSize = 50 * 1024 * 1024 // 50MB for large tokenizer vocabularies
// maxGGUFArrayDepth allows arrays of arrays but nothing deeper.
maxGGUFArrayDepth = 2
// maxGGUFDecodedItems bounds the items, across the whole header, of arrays that gguf-parser-go
// always decodes in full. Real files hold a few per-layer arrays.
maxGGUFDecodedItems = 65536
// maxGGUFSplitTensors bounds split.tensors.count, which the library preallocates tensor infos for.
// Shards declare the total across all splits, so this can't be the header tensor count.
maxGGUFSplitTensors = 1 << 17
// maxGGUFKeyLength is the key length limit from the GGUF spec.
maxGGUFKeyLength = 65535
// maxGGUFKVCount bounds the metadata KV count. Real files hold hundreds; vocabularies live in arrays.
maxGGUFKVCount = 65536
// maxGGUFReadStringBytes bounds, across the header, the key and string bytes the library reads in
// full and syft then hashes and encodes (chat templates are the largest, at tens of KB).
maxGGUFReadStringBytes = 8 << 20
// maxGGUFSkippedStrings bounds, across the header, the strings the library skips one at a time
// inside arrays (vocabularies run to a few hundred thousand tokens and merges).
maxGGUFSkippedStrings = 1 << 22
)
// copyHeader copies the GGUF header from the reader to the writer.
// ggufDecodedArraySuffixes are the keys gguf-parser-go decodes in full even with SkipLargeMetadata
// (hardcoded in its ReadArray as of v0.26.3; re-check on upgrade).
var ggufDecodedArraySuffixes = []string{
".feed_forward_length", ".attention.head_count", ".attention.head_count_kv", ".attention.sliding_window_pattern",
}
// ggufScalarSizes is the encoded size of each value type, indexed by type (0 for strings and arrays).
var ggufScalarSizes = [...]int{1, 1, 2, 2, 4, 4, 4, 1, 0, 0, 8, 8, 8}
// copyHeader copies at most maxHeaderSize bytes of the GGUF header from the reader to the writer.
// It validates the magic number first, then copies the rest of the data.
// The reader should be wrapped with io.LimitedReader to prevent OOM issues.
func copyHeader(w io.Writer, r io.Reader) error {
r = io.LimitReader(r, maxHeaderSize)
// Read initial chunk to validate magic number
// GGUF format: magic(4) + version(4) + tensor_count(8) + metadata_kv_count(8) + metadata_kvs + tensors_info
initialBuf := make([]byte, 24) // Enough for magic, version, tensor count, and kv count
@@ -37,7 +71,6 @@ func copyHeader(w io.Writer, r io.Reader) error {
}
// Copy the rest of the header from reader to writer
// The LimitedReader will return EOF once maxHeaderSize is reached
if _, err := io.Copy(w, r); err != nil {
return fmt.Errorf("failed to copy GGUF header: %w", err)
}
@@ -45,7 +78,269 @@ func copyHeader(w io.Writer, r io.Reader) error {
return nil
}
// Helper to convert gguf_parser metadata to simpler types
// validateGGUFHeader walks the metadata KV section and rejects values that gguf-parser-go would panic
// on, recurse on without bound, or overallocate for. It reads without storing anything.
// ponytail: tensor infos are not checked here (zero block size types, dimension overflow); those
// panics are recoverable and are left to the per-file panic recovery in the generic cataloger.
func validateGGUFHeader(r io.Reader) error {
v := &ggufHeaderValidator{
r: bufio.NewReader(r),
decodedBudget: maxGGUFDecodedItems,
readStringBudget: maxGGUFReadStringBytes,
skippedStringBudget: maxGGUFSkippedStrings,
}
var prefix struct {
Magic, Version uint32
}
if err := binary.Read(v.r, binary.LittleEndian, &prefix); err != nil {
return fmt.Errorf("reading header prefix: %w", err)
}
v.v1 = prefix.Version <= 1
// the tensor count is validated by the library
if _, err := v.readLen(); err != nil {
return fmt.Errorf("reading tensor count: %w", err)
}
kvCount, err := v.readLen()
if err != nil {
return fmt.Errorf("reading metadata count: %w", err)
}
if kvCount > maxGGUFKVCount {
return fmt.Errorf("metadata count %d exceeds %d", kvCount, maxGGUFKVCount)
}
for i := uint64(0); i < kvCount; i++ {
key, err := v.readKey()
if err != nil {
return fmt.Errorf("metadata key %d: %w", i, err)
}
var vt uint32
if err := binary.Read(v.r, binary.LittleEndian, &vt); err != nil {
return fmt.Errorf("metadata key %q: %w", key, err)
}
if err := v.checkKey(key, gguf_parser.GGUFMetadataValueType(vt)); err != nil {
return fmt.Errorf("metadata key %q: %w", key, err)
}
}
return nil
}
type ggufHeaderValidator struct {
r *bufio.Reader
v1 bool // v1 uses 32-bit lengths
decodedBudget uint64
readStringBudget uint64
skippedStringBudget uint64
}
func (v *ggufHeaderValidator) checkKey(key string, vt gguf_parser.GGUFMetadataValueType) error {
decoded := slices.ContainsFunc(ggufDecodedArraySuffixes, func(s string) bool { return strings.HasSuffix(key, s) })
n, err := v.readValue(vt, decoded, 1)
if err != nil {
return err
}
// these are read with typed accessors that panic on a type mismatch
switch key {
case "general.architecture", "general.name", "general.license", "general.type", "general.author",
"general.url", "general.description", "controlvector.model_hint":
if vt != gguf_parser.GGUFMetadataValueTypeString {
return fmt.Errorf("expected string, got %v", vt)
}
case "general.quantization_version", "general.file_type":
if !vt.IsNumeric() {
return fmt.Errorf("expected number, got %v", vt)
}
case "general.alignment":
if vt != gguf_parser.GGUFMetadataValueTypeUint32 || n == 0 {
return fmt.Errorf("expected non-zero uint32, got %v %v", vt, n)
}
case "split.tensors.count":
if !vt.IsNumeric() {
return fmt.Errorf("expected number, got %v", vt)
}
if math.IsNaN(n) || n < 0 || n > maxGGUFSplitTensors {
return fmt.Errorf("unreasonable split tensor count %.0f", n)
}
}
return nil
}
// readValue consumes one value, returning scalar numbers as float64 for the key checks.
func (v *ggufHeaderValidator) readValue(vt gguf_parser.GGUFMetadataValueType, decoded bool, depth int) (float64, error) {
if uint32(vt) >= uint32(len(ggufScalarSizes)) {
return 0, fmt.Errorf("invalid value type %d", vt)
}
switch vt {
case gguf_parser.GGUFMetadataValueTypeString:
// only strings inside skipped arrays are skipped by the library, the rest are read in full
return 0, v.skipString(depth > 1 && !decoded)
case gguf_parser.GGUFMetadataValueTypeArray:
return 0, v.readArray(decoded, depth)
}
size := ggufScalarSizes[vt]
b, err := v.r.Peek(size)
if err != nil {
return 0, err
}
var buf [8]byte
copy(buf[:], b)
if _, err := v.r.Discard(size); err != nil {
return 0, err
}
u := binary.LittleEndian.Uint64(buf[:])
switch vt {
case gguf_parser.GGUFMetadataValueTypeInt8:
return float64(int8(u)), nil
case gguf_parser.GGUFMetadataValueTypeInt16:
return float64(int16(u)), nil
case gguf_parser.GGUFMetadataValueTypeInt32:
return float64(int32(u)), nil
case gguf_parser.GGUFMetadataValueTypeInt64:
return float64(int64(u)), nil
case gguf_parser.GGUFMetadataValueTypeFloat32:
return float64(math.Float32frombits(uint32(u))), nil
case gguf_parser.GGUFMetadataValueTypeFloat64:
return math.Float64frombits(u), nil
}
return float64(u), nil
}
func (v *ggufHeaderValidator) readArray(decoded bool, depth int) error {
var itemType uint32
if err := binary.Read(v.r, binary.LittleEndian, &itemType); err != nil {
return err
}
n, err := v.readLen()
if err != nil {
return err
}
it := gguf_parser.GGUFMetadataValueType(itemType)
switch {
case itemType >= uint32(len(ggufScalarSizes)):
return fmt.Errorf("invalid array item type %d", itemType)
case decoded && n > v.decodedBudget:
return fmt.Errorf("decoded arrays exceed %d items", maxGGUFDecodedItems)
case it == gguf_parser.GGUFMetadataValueTypeArray && (!decoded || depth >= maxGGUFArrayDepth):
// the library can only skip over arrays of scalars
return errors.New("unsupported nested array")
case n > maxHeaderSize:
return fmt.Errorf("array length %d exceeds header size", n)
case !decoded && ggufScalarSizes[itemType] > 0:
// skipped arrays of fixed size items can be passed over in one go
_, err := v.r.Discard(int(n) * ggufScalarSizes[itemType])
return err
}
if decoded {
v.decodedBudget -= n
}
for i := uint64(0); i < n; i++ {
if _, err := v.readValue(it, decoded, depth+1); err != nil {
return err
}
}
return nil
}
func (v *ggufHeaderValidator) readLen() (uint64, error) {
if v.v1 {
var n uint32
err := binary.Read(v.r, binary.LittleEndian, &n)
return uint64(n), err
}
var n uint64
err := binary.Read(v.r, binary.LittleEndian, &n)
return n, err
}
// readKey reads a key the way gguf-parser-go will see it, whitespace trimmed.
func (v *ggufHeaderValidator) readKey() (string, error) {
n, err := v.readLen()
if err != nil {
return "", err
}
if n == 0 || n > maxGGUFKeyLength {
return "", fmt.Errorf("invalid key length %d", n)
}
if err := v.spendReadString(n); err != nil {
return "", err
}
buf := make([]byte, n)
if _, err := io.ReadFull(v.r, buf); err != nil {
return "", err
}
return strings.TrimSpace(string(buf)), nil
}
func (v *ggufHeaderValidator) skipString(allowEmpty bool) error {
n, err := v.readLen()
if err != nil {
return err
}
// gguf-parser-go reads an empty string as a full pool buffer, so everything after it would be
// parsed out of step with this walk
if n == 0 && !allowEmpty {
return errors.New("empty string")
}
if n > maxHeaderSize {
return fmt.Errorf("string length %d exceeds header size", n)
}
if allowEmpty {
if v.skippedStringBudget == 0 {
return fmt.Errorf("skipped strings exceed %d", maxGGUFSkippedStrings)
}
v.skippedStringBudget--
} else if err := v.spendReadString(n); err != nil {
return err
}
_, err = v.r.Discard(int(n))
return err
}
func (v *ggufHeaderValidator) spendReadString(n uint64) error {
if n > v.readStringBudget {
return fmt.Errorf("strings exceed %d bytes", maxGGUFReadStringBytes)
}
v.readStringBudget -= n
return nil
}
// sanitizeGGUFValue replaces non-finite floats, which JSON can not encode, leaving everything else as is.
func sanitizeGGUFValue(v any) any {
switch t := v.(type) {
case float32:
if f := float64(t); math.IsNaN(f) || math.IsInf(f, 0) {
return nonFiniteString(f)
}
case float64:
if math.IsNaN(t) || math.IsInf(t, 0) {
return nonFiniteString(t)
}
case gguf_parser.GGUFMetadataKVArrayValue:
// keep the library shape as is; items are only present for the few keys the library
// decodes in full, which the pre-scan keeps small and shallow
if len(t.Array) > 0 {
items := make([]any, len(t.Array))
for i, item := range t.Array {
items[i] = sanitizeGGUFValue(item)
}
t.Array = items
}
return t
}
return v
}
func nonFiniteString(f float64) string {
switch {
case math.IsInf(f, 1):
return "+Inf"
case math.IsInf(f, -1):
return "-Inf"
}
return "NaN"
}
// helper to convert gguf_parser metadata to simpler types
func convertGGUFMetadataKVs(kvs gguf_parser.GGUFMetadataKVs) map[string]any {
result := make(map[string]any)
@@ -56,7 +351,7 @@ func convertGGUFMetadataKVs(kvs gguf_parser.GGUFMetadataKVs) map[string]any {
"general.version", "general.parameter_count", "general.quantization":
continue
}
result[kv.Key] = kv.Value
result[kv.Key] = sanitizeGGUFValue(kv.Value)
}
return result
syft/pkg/cataloger/ai/parse_gguf_model.go
+26 -14
View File
@@ -13,9 +13,7 @@ import (
gguf_parser "github.com/gpustack/gguf-parser-go"
"github.com/anchore/syft/internal"
"github.com/anchore/syft/internal/log"
"github.com/anchore/syft/internal/tmpdir"
"github.com/anchore/syft/internal/unknown"
"github.com/anchore/syft/syft/artifact"
"github.com/anchore/syft/syft/file"
"github.com/anchore/syft/syft/pkg"
@@ -38,14 +36,20 @@ func parseGGUFModel(ctx context.Context, _ file.Resolver, _ *generic.Environment
defer cleanup()
tempPath := tempFile.Name()
// Copy and validate the GGUF file header using LimitedReader to prevent OOM
// We use LimitedReader to cap reads at maxHeaderSize (50MB)
limitedReader := &io.LimitedReader{R: reader, N: maxHeaderSize}
if err := copyHeader(tempFile, limitedReader); err != nil {
// copy and validate the GGUF file header (bounded to maxHeaderSize)
if err := copyHeader(tempFile, reader); err != nil {
tempFile.Close()
return nil, nil, fmt.Errorf("failed to copy GGUF header: %w", err)
}
if _, err := tempFile.Seek(0, io.SeekStart); err != nil {
tempFile.Close()
return nil, nil, fmt.Errorf("failed to rewind GGUF header: %w", err)
}
err = validateGGUFHeader(tempFile)
tempFile.Close()
if err != nil {
return nil, nil, fmt.Errorf("invalid GGUF header: %w", err)
}
// Parse using gguf-parser-go with options to skip unnecessary data
ggufFile, err := gguf_parser.ParseGGUFFile(tempPath,
@@ -61,6 +65,11 @@ func parseGGUFModel(ctx context.Context, _ file.Resolver, _ *generic.Environment
// Extract version separately (will be set on Package.Version)
modelVersion := extractVersion(ggufFile.Header.MetadataKV)
metadataHash, err := computeKVMetadataHash(ggufFile.Header.MetadataKV)
if err != nil {
return nil, nil, err
}
// Convert to syft metadata structure
syftMetadata := &pkg.GGUFFileHeader{
Architecture: metadata.Architecture,
@@ -69,7 +78,7 @@ func parseGGUFModel(ctx context.Context, _ file.Resolver, _ *generic.Environment
GGUFVersion: uint32(ggufFile.Header.Version),
TensorCount: ggufFile.Header.TensorCount,
RemainingKeyValues: convertGGUFMetadataKVs(ggufFile.Header.MetadataKV),
MetadataKeyValuesHash: computeKVMetadataHash(ggufFile.Header.MetadataKV),
MetadataKeyValuesHash: metadataHash,
}
// If model name is not in metadata, use filename
@@ -79,6 +88,7 @@ func parseGGUFModel(ctx context.Context, _ file.Resolver, _ *generic.Environment
// Create package from metadata
p := newGGUFPackage(
ctx,
syftMetadata,
metadata.Name,
modelVersion,
@@ -86,14 +96,17 @@ func parseGGUFModel(ctx context.Context, _ file.Resolver, _ *generic.Environment
reader.WithAnnotation(pkg.EvidenceAnnotationKey, pkg.PrimaryEvidenceAnnotation),
)
return []pkg.Package{p}, nil, unknown.IfEmptyf([]pkg.Package{p}, "unable to parse GGUF file")
return []pkg.Package{p}, nil, nil
}
// computeKVMetadataHash computes a stable hash of the KV metadata for use as a global identifier
func computeKVMetadataHash(metadata gguf_parser.GGUFMetadataKVs) string {
// computeKVMetadataHash computes a stable hash of the sanitized KV metadata for use as a global identifier
func computeKVMetadataHash(metadata gguf_parser.GGUFMetadataKVs) (string, error) {
// Sort the KV pairs by key for stable hashing
sortedKVs := make([]gguf_parser.GGUFMetadataKV, len(metadata))
copy(sortedKVs, metadata)
for i, kv := range metadata {
kv.Value = sanitizeGGUFValue(kv.Value)
sortedKVs[i] = kv
}
sort.Slice(sortedKVs, func(i, j int) bool {
return sortedKVs[i].Key < sortedKVs[j].Key
})
@@ -101,13 +114,12 @@ func computeKVMetadataHash(metadata gguf_parser.GGUFMetadataKVs) string {
// Marshal sorted KVs to JSON for stable hashing
jsonBytes, err := json.Marshal(sortedKVs)
if err != nil {
log.Debugf("failed to marshal metadata for hashing: %v", err)
return ""
return "", fmt.Errorf("failed to marshal GGUF metadata for hashing: %w", err)
}
// Compute xxhash
hash := xxhash.Sum64(jsonBytes)
return fmt.Sprintf("%016x", hash) // 16 hex chars (64 bits)
return fmt.Sprintf("%016x", hash), nil // 16 hex chars (64 bits)
}
// extractVersion attempts to extract version from metadata KV pairs
syft/pkg/cataloger/ai/parse_gguf_test.go
+251
View File
@@ -0,0 +1,251 @@
package ai
import (
"bytes"
"context"
"encoding/binary"
"encoding/json"
"io"
"math"
"strings"
"testing"
gguf_parser "github.com/gpustack/gguf-parser-go"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/anchore/syft/internal/tmpdir"
"github.com/anchore/syft/syft/file"
"github.com/anchore/syft/syft/format/syftjson"
"github.com/anchore/syft/syft/pkg"
"github.com/anchore/syft/syft/sbom"
)
func le(v any) []byte {
buf := new(bytes.Buffer)
binary.Write(buf, binary.LittleEndian, v)
return buf.Bytes()
}
func ggufString(s string) []byte {
return append(le(uint64(len(s))), s...)
}
func parseGGUFBytes(t *testing.T, data []byte) ([]pkg.Package, error) {
t.Helper()
ctx, td := tmpdir.Root(context.Background(), "gguf-test")
t.Cleanup(func() { _ = td.Cleanup() })
reader := file.NewLocationReadCloser(file.NewLocation("/model.gguf"), io.NopCloser(bytes.NewReader(data)))
var pkgs []pkg.Package
var err error
require.NotPanics(t, func() {
pkgs, _, err = parseGGUFModel(ctx, nil, nil, reader)
})
return pkgs, err
}
func TestParseGGUFModel_malformedHeader(t *testing.T) {
// nested arrays of arrays, each one level deeper, ending in an empty uint8 array
nested := func(depth int) []byte {
v := ggufArray(ggufTypeUint8, 0, nil)
for i := 1; i < depth; i++ {
v = ggufArray(ggufTypeArray, 1, v)
}
return v
}
// a header that declares more KVs than the cap, with none following
tooManyKVs := append(append([]byte("GGUF"), le(uint32(3))...), le([]uint64{0, maxGGUFKVCount + 1})...)
tests := []struct {
name string
builder *testGGUFBuilder
data []byte // used instead of builder when set
wantErr bool
}{
{name: "too many KVs", data: tooManyKVs, wantErr: true},
{name: "split tensor count is a string", builder: newTestGGUFBuilder().withStringKV("split.tensors.count", "1"), wantErr: true},
{name: "split tensor count over the cap", builder: newTestGGUFBuilder().withUint32KV("split.tensors.count", maxGGUFSplitTensors+1), wantErr: true},
{name: "read strings too long in total", builder: newTestGGUFBuilder().withStringKV("x", strings.Repeat("a", maxGGUFReadStringBytes)), wantErr: true},
{
name: "skipped strings too many in total",
builder: newTestGGUFBuilder().withRawKV("tokenizer.ggml.tokens", ggufTypeArray, ggufArray(ggufTypeString, maxGGUFSkippedStrings+1, make([]byte, 8*(maxGGUFSkippedStrings+1)))),
wantErr: true,
},
{name: "v1 header", builder: newTestGGUFBuilder().withVersion(1).withStringKV("general.name", "m").withUint32KV("llama.context_length", 8)},
{name: "architecture not a string", builder: newTestGGUFBuilder().withUint32KV("general.architecture", 1), wantErr: true},
{name: "name not a string", builder: newTestGGUFBuilder().withUint32KV("general.name", 1), wantErr: true},
{name: "license not a string", builder: newTestGGUFBuilder().withUint32KV("general.license", 1), wantErr: true},
{name: "type not a string", builder: newTestGGUFBuilder().withUint32KV("general.type", 1), wantErr: true},
{name: "author not a string", builder: newTestGGUFBuilder().withUint32KV("general.author", 1), wantErr: true},
{name: "url not a string", builder: newTestGGUFBuilder().withUint32KV("general.url", 1), wantErr: true},
{name: "description not a string", builder: newTestGGUFBuilder().withUint32KV("general.description", 1), wantErr: true},
{name: "model hint not a string", builder: newTestGGUFBuilder().withUint32KV("controlvector.model_hint", 1), wantErr: true},
{name: "quantization version is a string", builder: newTestGGUFBuilder().withStringKV("general.quantization_version", "2"), wantErr: true},
{name: "alignment is a string", builder: newTestGGUFBuilder().withStringKV("general.alignment", "32"), wantErr: true},
{name: "alignment is uint64", builder: newTestGGUFBuilder().withUint64KV("general.alignment", 32), wantErr: true},
{name: "alignment is zero", builder: newTestGGUFBuilder().withUint32KV("general.alignment", 0), wantErr: true},
{
name: "array of arrays on a skipped key",
builder: newTestGGUFBuilder().withRawKV("tokenizer.ggml.merges", ggufTypeArray, ggufArray(ggufTypeArray, 1, ggufArray(ggufTypeUint8, 0, nil))),
wantErr: true,
},
{
name: "invalid array item type",
builder: newTestGGUFBuilder().withRawKV("tokenizer.ggml.scores", ggufTypeArray, ggufArray(99, 0, nil)),
wantErr: true,
},
{name: "split tensor count out of range", builder: newTestGGUFBuilder().withUint64KV("split.tensors.count", 1<<62), wantErr: true},
{name: "split tensor count huge", builder: newTestGGUFBuilder().withUint64KV("split.tensors.count", 1<<40), wantErr: true},
{name: "split tensor count negative", builder: newTestGGUFBuilder().withRawKV("split.tensors.count", ggufTypeInt64, le(int64(-1))), wantErr: true},
{
name: "nesting too deep on a decoded key",
builder: newTestGGUFBuilder().withRawKV("llama.attention.head_count", ggufTypeArray, nested(3)),
wantErr: true,
},
{
name: "decoded array too long",
builder: newTestGGUFBuilder().withRawKV("llama.feed_forward_length", ggufTypeArray, ggufArray(ggufTypeUint8, maxGGUFDecodedItems+1, make([]byte, maxGGUFDecodedItems+1))),
wantErr: true,
},
{name: "empty string value", builder: newTestGGUFBuilder().withStringKV("general.author", ""), wantErr: true},
{name: "padded key", builder: newTestGGUFBuilder().withUint32KV(" general.alignment\t", 0), wantErr: true},
{name: "long padded key", builder: newTestGGUFBuilder().withUint32KV(strings.Repeat(" ", 300)+"general.alignment", 0), wantErr: true},
{name: "file type is a string", builder: newTestGGUFBuilder().withUint32KV("general.file_type", 1).withStringKV("general.file_type", "x"), wantErr: true},
{
name: "decoded arrays too long in total",
builder: newTestGGUFBuilder().
withRawKV("a.feed_forward_length", ggufTypeArray, ggufArray(ggufTypeUint8, maxGGUFDecodedItems, make([]byte, maxGGUFDecodedItems))).
withRawKV("b.feed_forward_length", ggufTypeArray, ggufArray(ggufTypeUint8, 1, make([]byte, 1))),
wantErr: true,
},
{name: "invalid value type", builder: newTestGGUFBuilder().withRawKV("x", 0xFFFFFFFF, nil), wantErr: true},
{name: "empty key", builder: newTestGGUFBuilder().withUint32KV("", 1), wantErr: true},
{
name: "empty string in a decoded array",
builder: newTestGGUFBuilder().withRawKV("llama.attention.head_count", ggufTypeArray, ggufArray(ggufTypeString, 1, le(uint64(0)))),
wantErr: true,
},
{
name: "empty string in a skipped array",
builder: newTestGGUFBuilder().withRawKV("tokenizer.ggml.tokens", ggufTypeArray, ggufArray(ggufTypeString, 1, le(uint64(0)))),
},
{
name: "array of arrays on a decoded key",
builder: newTestGGUFBuilder().withRawKV("llama.attention.head_count", ggufTypeArray, nested(2)),
},
{name: "alignment is a uint32", builder: newTestGGUFBuilder().withUint32KV("general.alignment", 32)},
{
name: "split keys as written by llama.cpp",
builder: newTestGGUFBuilder().
withRawKV("split.no", ggufTypeUint16, le(uint16(0))).
withRawKV("split.count", ggufTypeUint16, le(uint16(3))).
withRawKV("split.tensors.count", ggufTypeInt32, le(int32(3000))),
},
{
name: "split shard declares the total tensor count",
builder: newTestGGUFBuilder().withRawKV("split.tensors.count", ggufTypeInt32, le(int32(3000))),
},
{
name: "skipped scalar array",
builder: newTestGGUFBuilder().withRawKV("tokenizer.ggml.scores", ggufTypeArray, ggufArray(ggufTypeFloat32, 3, make([]byte, 12))),
},
// ponytail: zero block size tensor types and dimension overflow panic inside tensor info parsing;
// those are left to the generic cataloger's panic handling rather than checked here.
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
data := tt.data
if data == nil {
data = tt.builder.withStringKV("general.architecture", "llama").build()
}
pkgs, err := parseGGUFBytes(t, data)
if tt.wantErr {
// the pre-scan has to be what rejects these, before the library sees them
require.ErrorContains(t, err, "invalid GGUF header")
assert.Empty(t, pkgs)
return
}
require.NoError(t, err)
require.Len(t, pkgs, 1)
})
}
}
func TestParseGGUFModel_nonFiniteFloat(t *testing.T) {
data := newTestGGUFBuilder().
withStringKV("general.architecture", "llama").
withRawKV("llama.rope.freq_base", ggufTypeFloat32, le(float32(math.Inf(1)))).
withRawKV("llama.rope.scale", ggufTypeFloat64, le(math.Inf(-1))).
withRawKV("llama.attention.head_count", ggufTypeArray, ggufArray(ggufTypeFloat32, 2, le([]float32{8, float32(math.NaN())}))).
withRawKV("tokenizer.ggml.scores", ggufTypeArray, ggufArray(ggufTypeFloat32, 1, le(float32(0)))).
build()
pkgs, err := parseGGUFBytes(t, data)
require.NoError(t, err)
require.Len(t, pkgs, 1)
meta := pkgs[0].Metadata.(pkg.GGUFFileHeader)
assert.NotEmpty(t, meta.MetadataKeyValuesHash)
assert.Equal(t, "+Inf", meta.RemainingKeyValues["llama.rope.freq_base"])
assert.Equal(t, "-Inf", meta.RemainingKeyValues["llama.rope.scale"])
headCount := meta.RemainingKeyValues["llama.attention.head_count"].(gguf_parser.GGUFMetadataKVArrayValue)
assert.Equal(t, []any{float32(8), "NaN"}, headCount.Array)
var buf bytes.Buffer
s := sbom.SBOM{Artifacts: sbom.Artifacts{Packages: pkg.NewCollection(pkgs...)}}
require.NoError(t, syftjson.NewFormatEncoder().Encode(&buf, s))
assert.NotEmpty(t, buf.Bytes())
}
// TestParseGGUFModel_outputUnchanged pins the header output and hash for ordinary values to what syft
// produced as of f7f3c6efc, before malformed header handling. The expected values were captured by
// running this same fixture on that commit. float32 0.1 catches any widening to float64.
func TestParseGGUFModel_outputUnchanged(t *testing.T) {
data := newTestGGUFBuilder().
withStringKV("general.architecture", "llama").
withRawKV("tokenizer.ggml.tokens", ggufTypeArray, ggufArray(ggufTypeString, 2, append(ggufString("a"), ggufString("b")...))).
withRawKV("llama.attention.head_count", ggufTypeArray, ggufArray(ggufTypeFloat32, 2, le([]float32{0.1, 0.2}))).
withRawKV("llama.feed_forward_length", ggufTypeArray, ggufArray(ggufTypeArray, 2, append(
ggufArray(ggufTypeUint32, 2, le([]uint32{1, 2})),
ggufArray(ggufTypeUint32, 1, le([]uint32{3}))...))).
withRawKV("llama.rope.freq_base", ggufTypeFloat32, le(float32(0.1))).
build()
pkgs, err := parseGGUFBytes(t, data)
require.NoError(t, err)
require.Len(t, pkgs, 1)
meta := pkgs[0].Metadata.(pkg.GGUFFileHeader)
assert.Equal(t, "4f28d55f0985a985", meta.MetadataKeyValuesHash)
header, err := json.Marshal(meta.RemainingKeyValues)
require.NoError(t, err)
assert.JSONEq(t, `{
"llama.attention.head_count": {"type": 6, "len": 2, "array": [0.1, 0.2], "startOffset": 170, "size": 8},
"llama.feed_forward_length": {"type": 9, "len": 2, "array": [
{"type": 4, "len": 2, "array": [1, 2], "startOffset": 239, "size": 8},
{"type": 4, "len": 1, "array": [3], "startOffset": 259, "size": 4}
], "startOffset": 227, "size": 36},
"llama.rope.freq_base": 0.1,
"tokenizer.ggml.tokens": {"type": 8, "len": 2, "startOffset": 102, "size": 18}
}`, string(header))
}
// TestCopyHeader_appliesItsOwnBound pins that copyHeader caps the copy at maxHeaderSize itself,
// whatever reader it is handed.
//
// Bytes retained is asserted directly rather than through testutils.MeasureAlloc: the destination is a
// bytes.Buffer, whose doubling growth costs a multiple of what it holds and lands on a different power
// of two depending on the platform. buf.Len() is the same property without that noise.
func TestCopyHeader_appliesItsOwnBound(t *testing.T) {
const payload = 2*maxHeaderSize + 1
file := make([]byte, 24, 24+payload)
binary.LittleEndian.PutUint32(file[0:4], ggufMagicNumber)
file = append(file, bytes.Repeat([]byte{'x'}, payload)...)
var buf bytes.Buffer
require.NoError(t, copyHeader(&buf, bytes.NewReader(file)))
assert.Equal(t, maxHeaderSize, buf.Len(), "copyHeader must cap the copy at maxHeaderSize")
}
syft/pkg/cataloger/ai/parse_safetensors.go
+52 -32
View File
@@ -5,6 +5,8 @@ import (
"encoding/json"
"fmt"
"io"
"math"
"math/bits"
"sort"
"strings"
@@ -15,9 +17,7 @@ import (
// SafeTensors file format: [8 bytes u64 LE header size] [N bytes JSON header] [tensor data].
// Reference: https://github.com/huggingface/safetensors#format
const (
maxSafeTensorsHeaderSize = 100 * 1024 * 1024 // 100MB ceiling on header JSON to prevent OOM
)
// The header JSON is capped at maxSafeTensorsHeaderSize (see limits.go).
// safeTensorsHeader is the decoded JSON header. Tensor entries live alongside a
// reserved "__metadata__" key holding a string-to-string producer map. We decode
@@ -84,47 +84,67 @@ func readSafeTensorsHeader(r io.Reader) (*safeTensorsHeader, error) {
return h, nil
}
// parameterCount sums the element counts across all tensors in the header.
func (h *safeTensorsHeader) parameterCount() uint64 {
var total uint64
for _, t := range h.tensors {
count := uint64(1)
for _, dim := range t.Shape {
if dim <= 0 {
count = 0
break
}
count *= uint64(dim)
}
total += count
// modelInfo returns the metadata derivable from the header bytes alone. Naming,
// licenses and ShardCount are left to the merge processor.
func (h *safeTensorsHeader) modelInfo() pkg.SafeTensorsModelInfo {
params, dtype := h.parameterStats()
return pkg.SafeTensorsModelInfo{
Format: "safetensors",
TensorCount: uint64(len(h.tensors)),
Parameters: params,
Quantization: normalizeDType(dtype),
UserMetadata: userMetadataKeyValues(h.metadata),
MetadataHash: h.metadataHash(),
}
return total
}
// dominantDType returns the dtype that accounts for the largest fraction of parameters.
// For mixed-precision models the "dominant" dtype is still a useful summary.
func (h *safeTensorsHeader) dominantDType() string {
// parameterStats sums the element counts across all tensors and returns the
// dtype that accounts for the largest share of them. For mixed-precision models
// the "dominant" dtype is still a useful summary. Totals saturate rather than wrap.
func (h *safeTensorsHeader) parameterStats() (total uint64, dominantDType string) {
sizeByDType := make(map[string]uint64)
for _, t := range h.tensors {
count := uint64(1)
for _, dim := range t.Shape {
if dim <= 0 {
count = 0
break
}
count *= uint64(dim)
count, ok := tensorElements(t.Shape)
if !ok {
continue
}
sizeByDType[t.DType] += count
total = saturatingAdd(total, count)
sizeByDType[t.DType] = saturatingAdd(sizeByDType[t.DType], count)
}
var best string
var bestSize uint64
for dtype, size := range sizeByDType {
if size > bestSize || (size == bestSize && dtype < best) {
best = dtype
if size > bestSize || (size == bestSize && dtype < dominantDType) {
dominantDType = dtype
bestSize = size
}
}
return best
return total, dominantDType
}
// tensorElements returns the element count for a shape, or ok=false on a
// non-positive dim or when the product overflows uint64.
func tensorElements(shape []int64) (uint64, bool) {
n := uint64(1)
for _, d := range shape {
if d <= 0 {
return 0, false
}
hi, lo := bits.Mul64(n, uint64(d))
if hi != 0 {
return 0, false
}
n = lo
}
return n, true
}
// saturatingAdd returns a+b, clamped to math.MaxUint64 instead of wrapping.
func saturatingAdd(a, b uint64) uint64 {
sum, carry := bits.Add64(a, b, 0)
if carry != 0 {
return math.MaxUint64
}
return sum
}
// metadataHash returns a stable xxhash64 over the logical tensor content
syft/pkg/cataloger/ai/parse_safetensors_model.go
+2 -12
View File
@@ -6,7 +6,6 @@ import (
"io"
"github.com/anchore/syft/internal"
"github.com/anchore/syft/internal/unknown"
"github.com/anchore/syft/syft/artifact"
"github.com/anchore/syft/syft/file"
"github.com/anchore/syft/syft/pkg"
@@ -26,22 +25,13 @@ func parseSafeTensorsFile(_ context.Context, _ file.Resolver, _ *generic.Environ
return nil, nil, fmt.Errorf("failed to read safetensors header: %w", err)
}
// ShardCount is intentionally not set here: the merge processor is the single
// owner of ShardCount and derives it from the number of shards in the group.
md := pkg.SafeTensorsModelInfo{
Format: "safetensors",
TensorCount: uint64(len(header.tensors)),
Parameters: header.parameterCount(),
Quantization: normalizeDType(header.dominantDType()),
UserMetadata: userMetadataKeyValues(header.metadata),
MetadataHash: header.metadataHash(),
}
md := header.modelInfo()
p := newSafeTensorsPackage(
&md,
reader.WithAnnotation(pkg.EvidenceAnnotationKey, pkg.PrimaryEvidenceAnnotation),
)
return []pkg.Package{p}, nil, unknown.IfEmptyf([]pkg.Package{p}, "unable to parse safetensors file")
return []pkg.Package{p}, nil, nil
}
// integrity check
syft/pkg/cataloger/ai/parse_safetensors_oci.go
+3 -11
View File
@@ -8,7 +8,6 @@ import (
"strings"
"github.com/anchore/syft/internal"
"github.com/anchore/syft/internal/unknown"
"github.com/anchore/syft/syft/artifact"
"github.com/anchore/syft/syft/file"
"github.com/anchore/syft/syft/pkg"
@@ -50,7 +49,7 @@ type dockerAIModelConfig struct {
func parseSafeTensorsOCIConfig(_ context.Context, _ file.Resolver, _ *generic.Environment, reader file.LocationReadCloser) ([]pkg.Package, []artifact.Relationship, error) {
defer internal.CloseAndLogError(reader, reader.Path())
body, err := io.ReadAll(io.LimitReader(reader, 1024*1024))
body, err := readBounded(reader, maxModelConfigBlobSize)
if err != nil {
return nil, nil, fmt.Errorf("failed to read docker AI model config: %w", err)
}
@@ -81,7 +80,7 @@ func parseSafeTensorsOCIConfig(_ context.Context, _ file.Resolver, _ *generic.En
&md,
reader.WithAnnotation(pkg.EvidenceAnnotationKey, pkg.PrimaryEvidenceAnnotation),
)
return []pkg.Package{p}, nil, unknown.IfEmptyf([]pkg.Package{p}, "unable to parse docker AI safetensors config")
return []pkg.Package{p}, nil, nil
}
// parseSafeTensorsOCILayer decodes the JSON header of a SafeTensors weight
@@ -94,14 +93,7 @@ func parseSafeTensorsOCILayer(_ context.Context, _ file.Resolver, _ *generic.Env
return nil, nil, fmt.Errorf("failed to read safetensors layer header: %w", err)
}
md := pkg.SafeTensorsModelInfo{
Format: "safetensors",
TensorCount: uint64(len(header.tensors)),
Parameters: header.parameterCount(),
Quantization: normalizeDType(header.dominantDType()),
UserMetadata: userMetadataKeyValues(header.metadata),
MetadataHash: header.metadataHash(),
}
md := header.modelInfo()
p := newSafeTensorsPackage(
&md,
syft/pkg/cataloger/ai/parse_safetensors_test.go
+268 -41
View File
@@ -6,8 +6,11 @@ import (
"encoding/binary"
"encoding/json"
"io"
"math"
"os"
"path/filepath"
"slices"
"strings"
"testing"
"github.com/stretchr/testify/assert"
@@ -48,7 +51,7 @@ func buildSafeTensorsFile(t *testing.T, metadata map[string]string, tensors map[
//
// The naming precedence (owned by the merge processor's pickSafeTensorsName) is:
// 1. config.json _name_or_path (path.Base applied), found beside the model or
// by walking up parent directories to the scan root
// in its parent directory
// 2. otherwise the model's immediate parent directory base name
// → drop (no package) when neither yields a usable name
//
@@ -231,9 +234,9 @@ func TestSafeTensorsCataloger(t *testing.T) {
expectedPackages: nil,
},
{
// rung 1 via parent-walk: findDirHFConfig walks up from the model
// directory, so a config.json in an ancestor names a nested model.
name: "config.json in an ancestor directory names a nested model",
// rung 1 via the parent: findDirHFConfig also checks the model
// directory's parent, so a config.json there names a nested model.
name: "config.json in the parent directory names a nested model",
setup: func(t *testing.T) string {
dir := t.TempDir()
model(t, filepath.Join(dir, "dir", "someothertensor"))
@@ -248,6 +251,87 @@ func TestSafeTensorsCataloger(t *testing.T) {
},
},
},
{
// the walk stops at the parent: a config.json two levels up is not used.
name: "config.json two levels up is not used",
setup: func(t *testing.T) string {
dir := t.TempDir()
model(t, filepath.Join(dir, "a", "b", "someothertensor"))
writeFile(t, filepath.Join(dir, "a", "config.json"), `{"_name_or_path":"org/TooFar"}`)
return dir
},
expectedPackages: []pkg.Package{
{
Name: "someothertensor",
Type: pkg.ModelPkg,
Metadata: wantMetadata(""),
},
},
},
{
// JSON without any HF field is not treated as a model config, so the
// lookup keeps going to the parent.
name: "non-HF config.json is skipped",
setup: func(t *testing.T) string {
dir := t.TempDir()
modelDir := filepath.Join(dir, "app", "sometensor")
model(t, modelDir)
writeFile(t, filepath.Join(modelDir, "config.json"), `{"port":8080}`)
writeFile(t, filepath.Join(dir, "app", "config.json"), `{"_name_or_path":"org/Parent"}`)
return dir
},
expectedPackages: []pkg.Package{
{
Name: "Parent",
Type: pkg.ModelPkg,
Metadata: wantMetadata(""),
},
},
},
{
// an oversized config.json is skipped entirely, while a large README
// still yields its frontmatter (only a prefix is read).
name: "oversized config.json is ignored and a large README still names the model",
setup: func(t *testing.T) string {
dir := t.TempDir()
modelDir := filepath.Join(dir, "sometensor")
model(t, modelDir)
writeFile(t, filepath.Join(modelDir, "config.json"),
`{"architectures":["X"],"_name_or_path":"`+strings.Repeat("A", maxHFConfigSize)+`"}`)
writeFile(t, filepath.Join(modelDir, "README.md"),
"---\nlicense: MIT\nbase_model: org/Readme\n---\n"+strings.Repeat("A", maxReadmePrefixSize))
return dir
},
expectedPackages: []pkg.Package{
{
Name: "Readme",
Type: pkg.ModelPkg,
Licenses: pkg.NewLicenseSet(
pkg.NewLicenseFromFields("MIT", "", nil),
),
Metadata: wantMetadata(""),
},
},
},
{
// a _name_or_path that sanitizes to nothing doesn't block README base_model.
name: "degenerate _name_or_path falls through to README base_model",
setup: func(t *testing.T) string {
dir := t.TempDir()
modelDir := filepath.Join(dir, "sometensor")
model(t, modelDir)
writeFile(t, filepath.Join(modelDir, "config.json"), `{"_name_or_path":"."}`)
writeFile(t, filepath.Join(modelDir, "README.md"), "---\nbase_model: org/Readme\n---\n")
return dir
},
expectedPackages: []pkg.Package{
{
Name: "Readme",
Type: pkg.ModelPkg,
Metadata: wantMetadata(""),
},
},
},
{
// grouping: independent models in one scan are grouped by their own
// parent directory and each named from it.
@@ -601,6 +685,60 @@ spdx-id: Apache-2.0
assertHasLicense(t, out[0], "Apache-2.0")
})
t.Run("OCI: config precedence is stable across layer order", func(t *testing.T) {
dir := t.TempDir()
a := filepath.Join(dir, "a.json")
b := filepath.Join(dir, "b.json")
gen := filepath.Join(dir, "generation_config.json")
require.NoError(t, os.WriteFile(a, []byte(`{"_name_or_path":"org/first"}`), 0o644))
require.NoError(t, os.WriteFile(b, []byte(`{"_name_or_path":"org/second"}`), 0o644))
require.NoError(t, os.WriteFile(gen, []byte(`{"max_new_tokens":10}`), 0o644))
configMd := pkg.SafeTensorsModelInfo{Format: "safetensors", TensorCount: 1}
for _, order := range [][]string{{a, b, gen}, {gen, b, a}} {
var locs []file.Location
for _, p := range order {
locs = append(locs, file.NewLocation(p))
}
resolver := file.NewMockResolverForMediaTypes(map[string][]file.Location{
dockerAIModelFileMediaType: locs,
})
out, _, err := safeTensorsMergeProcessor(context.Background(), resolver, []pkg.Package{ociPkg(configMd)}, nil, nil)
require.NoError(t, err)
require.Len(t, out, 1)
assert.Equal(t, "first", out[0].Name)
// the non-HF generation config is not recorded as supporting evidence
assert.Len(t, out[0].Locations.ToSlice(), 3)
}
})
t.Run("dir scan: non-HF config.json in the parent is not supporting evidence", func(t *testing.T) {
dir := t.TempDir()
cfgPath := filepath.Join(dir, "config.json")
require.NoError(t, os.WriteFile(cfgPath, []byte(`{"port":8080}`), 0o644))
loc, cfg := findDirHFConfig(file.NewMockResolverForPaths(cfgPath), filepath.Join(dir, "model"))
assert.Nil(t, loc)
assert.Nil(t, cfg)
})
t.Run("OCI: a large license layer still yields its frontmatter spdx-id", func(t *testing.T) {
dir := t.TempDir()
licensePath := filepath.Join(dir, "LICENSE")
require.NoError(t, os.WriteFile(licensePath,
[]byte("---\nspdx-id: Apache-2.0\n---\n"+strings.Repeat("A", maxLicensePrefixSize)), 0o644))
assert.Equal(t, "Apache-2.0", readLicenseSPDXIDFromFrontmatter(file.NewMockResolverForPaths(licensePath), file.NewLocation(licensePath)))
})
t.Run("OCI: oversized model.file layer is rejected", func(t *testing.T) {
dir := t.TempDir()
cfgPath := filepath.Join(dir, "config.json")
require.NoError(t, os.WriteFile(cfgPath,
[]byte(`{"_name_or_path":"org/x","pad":"`+strings.Repeat("A", maxModelFileLayerSize)+`"}`), 0o644))
cfg, fm := classifyOCIModelFileLayer(file.NewMockResolverForPaths(cfgPath), file.NewLocation(cfgPath))
assert.Nil(t, cfg)
assert.Nil(t, fm)
})
t.Run("names packages and still propagates an upstream error", func(t *testing.T) {
// A non-nil upstream error (e.g. another file in the run failed to parse)
// must NOT short-circuit naming. The package is still nameless at this
@@ -622,7 +760,7 @@ spdx-id: Apache-2.0
// (or neither, asserting the drop path).
//
// Precedence (highest → lowest):
// 1. config.json _name_or_path (path.Base applied; both dir-scan and OCI)
// 1. config.json _name_or_path (sanitized to its last path element; both dir-scan and OCI)
// 2. fallback name — OCI image-ref last segment, or dir-scan parent directory
// base name (the merge processor computes the right one per group)
// → drop (empty name) when nothing matches
@@ -663,6 +801,31 @@ func TestSafeTensorsNamingPrecedence(t *testing.T) {
want: "tiny-llama",
},
// sanitizing
{
name: "windows-style path uses the last element",
nameOrPath: `C:\models\win-model`,
fallbackName: "fallback-name",
want: "win-model",
},
{
name: "degenerate path falls back",
nameOrPath: "..",
fallbackName: "fallback-name",
want: "fallback-name",
},
{
name: "root path falls back",
nameOrPath: "/",
fallbackName: "fallback-name",
want: "fallback-name",
},
{
name: "long names are truncated",
nameOrPath: "org/" + strings.Repeat("x", maxModelNameLength+10),
want: strings.Repeat("x", maxModelNameLength),
},
// drops
{
name: "drop: both rungs empty",
@@ -984,8 +1147,9 @@ func TestSafeTensorsHeader_parameterCountAndDType(t *testing.T) {
"small": {DType: "F32", Shape: []int64{16, 16}},
"scalar": {DType: "F32", Shape: []int64{}}, // empty shape contributes 1
}}
assert.Equal(t, uint64(1000*16+16*16+1), h.parameterCount())
assert.Equal(t, "BF16", h.dominantDType())
params, dtype := h.parameterStats()
assert.Equal(t, uint64(1000*16+16*16+1), params)
assert.Equal(t, "BF16", dtype)
}
func TestSafeTensorsHeader_parameterCount_nonPositiveDims(t *testing.T) {
@@ -997,7 +1161,29 @@ func TestSafeTensorsHeader_parameterCount_nonPositiveDims(t *testing.T) {
"zero": {DType: "F32", Shape: []int64{4, 0}},
"negative": {DType: "F32", Shape: []int64{-1, 8}},
}}
assert.Equal(t, uint64(16), h.parameterCount())
params, _ := h.parameterStats()
assert.Equal(t, uint64(16), params)
}
func TestSafeTensorsHeader_parameterStats_overflow(t *testing.T) {
// a shape whose product overflows uint64 is skipped instead of wrapping (the
// first tensor would wrap to 0) and can't flip the dominant dtype; the total
// saturates instead of wrapping.
h := &safeTensorsHeader{tensors: map[string]safeTensorsEntry{
"good": {DType: "BF16", Shape: []int64{4, 4}},
"wraps": {DType: "F32", Shape: []int64{4294967296, 4294967296}},
"overflow": {DType: "F32", Shape: []int64{math.MaxInt64, 3}},
}}
params, dtype := h.parameterStats()
assert.Equal(t, uint64(16), params)
assert.Equal(t, "BF16", dtype)
h = &safeTensorsHeader{tensors: map[string]safeTensorsEntry{
"a": {DType: "F32", Shape: []int64{math.MaxInt64, 2}},
"b": {DType: "F32", Shape: []int64{math.MaxInt64, 2}},
}}
params, _ = h.parameterStats()
assert.Equal(t, uint64(math.MaxUint64), params)
}
func TestSafeTensorsHeader_dominantDType_tieBreak(t *testing.T) {
@@ -1007,7 +1193,8 @@ func TestSafeTensorsHeader_dominantDType_tieBreak(t *testing.T) {
"a": {DType: "F32", Shape: []int64{4, 4}},
"b": {DType: "BF16", Shape: []int64{4, 4}},
}}
assert.Equal(t, "BF16", h.dominantDType())
_, dtype := h.parameterStats()
assert.Equal(t, "BF16", dtype)
}
func TestNormalizeDType(t *testing.T) {
@@ -1036,25 +1223,78 @@ func TestParseFrontmatter(t *testing.T) {
name string
input string
wantNil bool
wantLicense string
wantLicenses []string
wantBaseModel []string
}{
{
name: "list base_model",
input: "---\nlicense: mit\nbase_model:\n - org/Model\n---\nbody",
wantLicense: "mit",
wantLicenses: []string{"mit"},
wantBaseModel: []string{"org/Model"},
},
{
name: "scalar base_model",
input: "---\nlicense: apache-2.0\nbase_model: org/Model\n---\n",
wantLicense: "apache-2.0",
wantLicenses: []string{"apache-2.0"},
wantBaseModel: []string{"org/Model"},
},
{
name: "leading BOM",
input: "\xef\xbb\xbf---\nlicense: mit\n---\n",
wantLicense: "mit",
name: "list license keeps base_model",
input: "---\nlicense: [mit, apache-2.0]\nbase_model: org/Model\n---\n",
wantLicenses: []string{"mit", "apache-2.0"},
wantBaseModel: []string{"org/Model"},
},
{
name: "mapping-valued fields are ignored, not fatal",
input: "---\nlicense: {name: mit}\nbase_model: org/Model\n---\n",
wantBaseModel: []string{"org/Model"},
},
{
name: "leading BOM",
input: "\xef\xbb\xbf---\nlicense: mit\n---\n",
wantLicenses: []string{"mit"},
},
{
name: "null license is ignored",
input: "---\nlicense: null\nbase_model: ~\n---\n",
wantLicenses: nil,
wantBaseModel: nil,
},
{
name: "empty block",
input: "---\n---\n",
wantNil: true,
},
{
name: "oversized block is rejected",
input: "---\nlicense: mit\n" + strings.Repeat("# pad\n", maxFrontmatterSize/6+1) + "---\n",
wantNil: true,
},
{
name: "closing delimiter at EOF with trailing whitespace",
input: "---\nlicense: mit\n--- ",
wantLicenses: []string{"mit"},
},
{
name: "list values are capped",
input: "---\nlicense: [" + strings.Repeat("mit,", maxFrontmatterValues+5) + "mit]\n---\n",
wantLicenses: slices.Repeat([]string{"mit"}, maxFrontmatterValues),
},
{
name: "CRLF delimiters",
input: "---\r\nlicense: mit\r\n---\r\nbody",
wantLicenses: []string{"mit"},
},
{
name: "opening delimiter must be exactly ---",
input: "----\nlicense: mit\n---\n",
wantNil: true,
},
{
name: "a ---x line inside the block does not close it",
input: "---\nbase_model: org/Model\n---x: 1\nlicense: mit\n---\n",
wantLicenses: []string{"mit"},
wantBaseModel: []string{"org/Model"},
},
{
name: "no frontmatter",
@@ -1066,6 +1306,12 @@ func TestParseFrontmatter(t *testing.T) {
input: "---\nlicense: mit\n",
wantNil: true,
},
{
// https://github.com/anchore/syft/issues/5327
name: "malformed frontmatter that panics the yaml decoder",
input: "---\n<<:\r? 0:\n---\nbody",
wantNil: true,
},
}
for _, tt := range tests {
@@ -1076,7 +1322,7 @@ func TestParseFrontmatter(t *testing.T) {
return
}
require.NotNil(t, fm)
assert.Equal(t, tt.wantLicense, fm.License)
assert.Equal(t, tt.wantLicenses, fm.Licenses)
if tt.wantBaseModel != nil {
assert.Equal(t, tt.wantBaseModel, fm.BaseModel)
}
@@ -1157,6 +1403,12 @@ limitations:
input: "---\nspdx-id: MIT\n(never closes)\n",
want: "",
},
{
// https://github.com/anchore/syft/issues/5327
name: "malformed frontmatter that panics the yaml decoder",
input: "---\n<<:\r? 0:\n---\nbody\n",
want: "",
},
}
for _, tt := range tests {
@@ -1233,28 +1485,3 @@ func TestReadSafeTensorsHeader_capBoundsHeap(t *testing.T) {
assert.Less(t, bounded, uint64(8*1024*1024),
"the declared-size check has to reject before the body is read, not after")
}
// TestCopyHeader_boundIsTheCallersLimitedReader pins where the GGUF bound actually lives. copyHeader
// io.Copy's whatever it is handed, so maxHeaderSize only holds because every caller wraps the reader
// first. A caller that stops doing that reads the whole file, and nothing in copyHeader would object.
//
// Bytes retained is asserted directly rather than through testutils.MeasureAlloc: the destination is a
// bytes.Buffer, whose doubling growth costs a multiple of what it holds and lands on a different power
// of two depending on the platform. buf.Len() is the same property without that noise.
func TestCopyHeader_boundIsTheCallersLimitedReader(t *testing.T) {
const payload = 2*maxHeaderSize + 1
file := make([]byte, 24, 24+payload)
binary.LittleEndian.PutUint32(file[0:4], ggufMagicNumber)
file = append(file, bytes.Repeat([]byte{'x'}, payload)...)
var unbounded bytes.Buffer
require.NoError(t, copyHeader(&unbounded, bytes.NewReader(file)))
require.Equal(t, len(file), unbounded.Len(),
"an unwrapped reader is copied whole; copyHeader applies no limit of its own")
var bounded bytes.Buffer
require.NoError(t, copyHeader(&bounded, &io.LimitedReader{R: bytes.NewReader(file), N: maxHeaderSize}))
assert.Equal(t, maxHeaderSize, bounded.Len(),
"a wrapped reader must cap the copy at maxHeaderSize")
}
syft/pkg/cataloger/ai/processor.go
+2 -2
View File
@@ -104,10 +104,10 @@ func mergeDirModels(ctx context.Context, resolver file.Resolver, pkgs []pkg.Pack
// groupByParentDir buckets filesystem-scanned models by the directory their
// primary-evidence file lives in.
//
// This encodes a deliberate assumption: a directory holds one logical
// model, so every .safetensors file in a directory is treated as a shard of the
// same modeland merged into one package.
// same model and merged into one package.
// The trade-off is that if a directory happens to contain several unrelated models,
// they are merged into one package rather than reported separately.
// We accept that because the conventional on-disk layout gives each model
syft/pkg/cataloger/ai/processor_test.go
+18
View File
@@ -1,6 +1,7 @@
package ai
import (
"errors"
"testing"
"github.com/stretchr/testify/assert"
@@ -61,3 +62,20 @@ func Test_ggufMergeProcessor(t *testing.T) {
})
}
}
func Test_ggufMergeProcessor_mergesDespiteError(t *testing.T) {
// one layer failing to parse must not drop the rest of the model
parseErr := errors.New("one layer failed to parse")
pkgs := []pkg.Package{
{Name: "model", Metadata: pkg.GGUFFileHeader{MetadataKeyValuesHash: "abc"}},
{Name: "", Metadata: pkg.GGUFFileHeader{MetadataKeyValuesHash: "part1"}},
}
got, _, err := ggufMergeProcessor(pkgs, nil, parseErr)
require.ErrorIs(t, err, parseErr)
require.Len(t, got, 1)
assert.Equal(t, "model", got[0].Name)
header, ok := got[0].Metadata.(pkg.GGUFFileHeader)
require.True(t, ok)
assert.Len(t, header.Parts, 1)
}
syft/pkg/cataloger/ai/test_helpers_test.go
+31 -3
View File
@@ -72,8 +72,32 @@ func (b *testGGUFBuilder) withUint32KV(key string, value uint32) *testGGUFBuilde
return b
}
// withRawKV adds a KV whose value bytes are written verbatim, for arrays and malformed values.
func (b *testGGUFBuilder) withRawKV(key string, valueType uint32, raw []byte) *testGGUFBuilder {
b.kvPairs = append(b.kvPairs, testKVPair{key: key, valueType: valueType, value: raw})
return b
}
// ggufArray encodes an array value: item type, length, then the already encoded items.
func ggufArray(itemType uint32, n uint64, items []byte) []byte {
buf := new(bytes.Buffer)
binary.Write(buf, binary.LittleEndian, itemType)
binary.Write(buf, binary.LittleEndian, n)
buf.Write(items)
return buf.Bytes()
}
// writeLen writes a count or length, which is 32-bit in GGUF v1 and 64-bit after.
func (b *testGGUFBuilder) writeLen(n uint64) {
if b.version <= 1 {
binary.Write(b.buf, binary.LittleEndian, uint32(n))
return
}
binary.Write(b.buf, binary.LittleEndian, n)
}
func (b *testGGUFBuilder) writeString(s string) {
binary.Write(b.buf, binary.LittleEndian, uint64(len(s)))
b.writeLen(uint64(len(s)))
b.buf.WriteString(s)
}
@@ -85,10 +109,10 @@ func (b *testGGUFBuilder) build() []byte {
binary.Write(b.buf, binary.LittleEndian, b.version)
// Write tensor count
binary.Write(b.buf, binary.LittleEndian, b.tensorCount)
b.writeLen(b.tensorCount)
// Write KV count
binary.Write(b.buf, binary.LittleEndian, uint64(len(b.kvPairs)))
b.writeLen(uint64(len(b.kvPairs)))
// Write KV pairs
for _, kv := range b.kvPairs {
@@ -97,6 +121,10 @@ func (b *testGGUFBuilder) build() []byte {
// Write value type
binary.Write(b.buf, binary.LittleEndian, kv.valueType)
// Write value based on type
if raw, ok := kv.value.([]byte); ok {
b.buf.Write(raw)
continue
}
switch kv.valueType {
case ggufTypeString:
b.writeString(kv.value.(string))
syft/pkg/cataloger/ai/testdata/safetensors/extract_header.go
Vendored
+3 -3
View File
@@ -28,9 +28,9 @@ import (
const (
safetensorsLayerMediaType = "application/vnd.docker.ai.safetensors"
// 8 MB matches maxHeaderBytes in the OCI model source. Real model headers
// are well under 1 MB; the extra slack covers outliers.
fetchBytes = 8 * 1024 * 1024
// matches maxSafeTensorsHeaderBytes in the OCI model source (8 MB header
// cap plus the 8-byte length prefix). Real model headers are well under 1 MB.
fetchBytes = 8*1024*1024 + 8
)
func main() {
syft/pkg/cataloger/alpine/parse_apk_db.go
+12
View File
@@ -270,6 +270,10 @@ func (f apkField) apply(p *parsedData, ctx *apkFileParsingContext) {
ctx.indexOfLatestDirectory = len(ctx.files) - 1
case "M":
i := ctx.indexOfLatestDirectory
if i < 0 {
log.Debugf("APK field %q appears before any directory entry", f.name)
return
}
latest := ctx.files[i]
var ok bool
@@ -296,6 +300,10 @@ func (f apkField) apply(p *parsedData, ctx *apkFileParsingContext) {
ctx.indexOfLatestRegularFile = len(ctx.files) - 1
case "a":
i := ctx.indexOfLatestRegularFile
if i < 0 {
log.Debugf("APK field %q appears before any file entry", f.name)
return
}
latest := ctx.files[i]
var ok bool
@@ -309,6 +317,10 @@ func (f apkField) apply(p *parsedData, ctx *apkFileParsingContext) {
ctx.files[i] = latest
case "Z":
i := ctx.indexOfLatestRegularFile
if i < 0 {
log.Debugf("APK field %q appears before any file entry", f.name)
return
}
latest := ctx.files[i]
latest.Digest = processChecksum(f.value)
syft/pkg/cataloger/alpine/parse_apk_db_test.go
+26
View File
@@ -869,3 +869,29 @@ https://foo.them.org/alpine/v3.14/community`,
})
}
}
func TestParseApkDB_attributeBeforeItsEntry(t *testing.T) {
// ACL (M:, a:) and checksum (Z:) lines describe the most recent directory or file entry, and used to index
// files[-1] when no entry had been seen yet
tests := []struct {
name string
line string
}{
{name: "directory ACL before any directory", line: "M:0:0:755"},
{name: "file ACL before any file", line: "a:0:0:755"},
{name: "checksum before any file", line: "Z:Q1Kz6TXnPVJeDQUfO0IJVxD3HnYmU="},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
contents := strings.Join([]string{"P:foo", "V:1.0-r0", tt.line, ""}, "\n")
reader := file.NewLocationReadCloser(file.NewLocation("installed"), io.NopCloser(strings.NewReader(contents)))
pkgs, _, err := parseApkDB(context.Background(), nil, new(generic.Environment), reader)
require.NoError(t, err)
require.Len(t, pkgs, 1)
assert.Equal(t, "foo", pkgs[0].Name)
assert.Empty(t, pkgs[0].Metadata.(pkg.ApkDBEntry).Files, "an orphaned attribute line should not invent a file")
})
}
}
syft/pkg/cataloger/arch/parse_alpm_db.go
+1 -6
View File
@@ -148,12 +148,7 @@ func parseAlpmDBEntry(reader io.Reader) (*parsedData, error) {
}
func newScanner(reader io.Reader) *bufio.Scanner {
// This is taken from the apk parser
// https://github.com/anchore/syft/blob/v0.47.0/syft/pkg/cataloger/apkdb/parse_apk_db.go#L37
const maxScannerCapacity = 1024 * 1024
bufScan := make([]byte, maxScannerCapacity)
scanner := bufio.NewScanner(reader)
scanner.Buffer(bufScan, maxScannerCapacity)
scanner := internal.NewLineScanner(reader)
onDoubleLF := func(data []byte, atEOF bool) (advance int, token []byte, err error) {
for i := range data {
if i > 0 && data[i-1] == '\n' && data[i] == '\n' {
syft/pkg/cataloger/arch/parse_alpm_db_test.go
+38 -19
View File
@@ -492,12 +492,16 @@ func Test_parseMtree_malformedInput(t *testing.T) {
// bytes still costs multiples of its own size in heap. Asserting the error would keep passing if the
// line cap were removed, because the byte cap would still reject a large enough input eventually.
func Test_parseMtree_lineCapBoundsHeap(t *testing.T) {
// newline-only lines are the worst case: minimum bytes per entry, so the most entries per byte
const payloadSize = 16 * intFile.MB
spec := bytes.Repeat([]byte{'\n'}, payloadSize)
// the property is linear in the cap, so it is measured at a small cap and scaled to the shipped
// one. Running it at 300k lines means an uncapped parse costing ~16GB, which under -race is most
// of a CI minute for a test with no concurrency in it.
const (
lineCap = 4096
payloadSize = 64 * lineCap
)
// well inside the byte cap, so only the line cap can stop this
require.Less(t, int64(payloadSize), int64(maxMtreeSize))
// newline-only lines are the worst case: minimum bytes per entry, so the most entries per byte
spec := bytes.Repeat([]byte{'\n'}, payloadSize)
unbounded := testutils.MeasureAlloc(t, func() {
// the same listing with the line cap lifted, to prove the fixture is really a bomb
@@ -507,19 +511,29 @@ func Test_parseMtree_lineCapBoundsHeap(t *testing.T) {
require.Greater(t, unbounded, uint64(payloadSize),
"fixture did not actually cost more than its own size; it is no longer a bomb")
bounded := testutils.MeasureAlloc(t, func() {
_, err := parseMtreeWithLimits(gzipOf(t, spec), maxMtreeSize, maxMtreeLines)
require.ErrorIs(t, err, errTooManyMtreeLines)
})
boundedAt := func(limit int) uint64 {
gz := gzipOf(t, spec)
return testutils.MeasureAlloc(t, func() {
_, err := parseMtreeWithLimits(gz, maxMtreeSize, limit)
require.ErrorIs(t, err, errTooManyMtreeLines)
})
}
bounded := boundedAt(lineCap)
t.Logf("unbounded allocated %d bytes, bounded allocated %d bytes", unbounded, bounded)
// the marginal cost between two caps, so fixed setup (gzip reader, parser) does not get scaled
perLine := (boundedAt(2*lineCap) - bounded) / lineCap
atShippedCap := perLine * maxMtreeLines
t.Logf("unbounded allocated %d bytes, bounded allocated %d bytes (%d per line, ~%d at the shipped cap)",
unbounded, bounded, perLine, atShippedCap)
// measured at ~238MB against ~16.5GB uncapped, so the cap is doing its job, but note what it
// actually buys: 300k entries still cost a few hundred MB, and that is per concurrent cataloger.
// The budget is set to catch the cap being removed or raised by an order of magnitude, not to pin
// the exact figure.
assert.Less(t, bounded, uint64(512*intFile.MB),
assert.Less(t, bounded, unbounded/10,
"the line cap has to stop the parse before it builds an entry per line")
// measured at ~238MB for the shipped 300k lines, so the cap is doing its job, but note what it
// actually buys: that is still a few hundred MB, per concurrent cataloger. The budget is set to
// catch the cap being raised by an order of magnitude, not to pin the exact figure.
assert.Less(t, atShippedCap, uint64(512*intFile.MB),
"the shipped line cap allows more heap than budgeted")
}
// Test_parseMtree_sizeCapBoundsHeap is the byte-cap counterpart to the line-cap budget above.
@@ -532,9 +546,12 @@ func Test_parseMtree_lineCapBoundsHeap(t *testing.T) {
// be ordinary for the byte cap to be the thing under test, and there have to be few enough of them
// that the line cap does not fire first.
func Test_parseMtree_sizeCapBoundsHeap(t *testing.T) {
// same reasoning as the line cap: measured at a small byte cap, since the shipped 64MB cap means
// gzipping and parsing ~200MB of fixture, which is slow under -race and proves nothing more.
const (
sizeCap = intFile.MB
lineLen = 1024
numLines = 96 * 1024 // ~96MB across ~98k lines: past the 64MB cap, well under the 300k line cap
numLines = 2 * sizeCap / lineLen // twice the cap, and far under the line cap
)
var raw bytes.Buffer
@@ -542,7 +559,7 @@ func Test_parseMtree_sizeCapBoundsHeap(t *testing.T) {
for range numLines {
raw.Write(line)
}
require.Greater(t, int64(raw.Len()), int64(maxMtreeSize), "fixture must exceed the byte cap")
require.Greater(t, int64(raw.Len()), int64(sizeCap), "fixture must exceed the byte cap")
require.Less(t, numLines, maxMtreeLines, "and must not trip the line cap first")
unbounded := testutils.MeasureAlloc(t, func() {
@@ -554,11 +571,13 @@ func Test_parseMtree_sizeCapBoundsHeap(t *testing.T) {
"fixture did not actually cost more than its own size; it is no longer a bomb")
bounded := testutils.MeasureAlloc(t, func() {
_, err := parseMtreeWithLimits(gzipOf(t, raw.Bytes()), maxMtreeSize, maxMtreeLines)
_, err := parseMtreeWithLimits(gzipOf(t, raw.Bytes()), sizeCap, maxMtreeLines)
require.ErrorIs(t, err, errMtreeTooLarge)
})
t.Logf("unbounded allocated %d bytes, bounded allocated %d bytes", unbounded, bounded)
assert.Less(t, bounded, uint64(512*intFile.MB),
// the old budget was 512MB against the 64MB cap, so 8x the cap
assert.Less(t, bounded, uint64(8*sizeCap),
"the size cap has to stop the read at the cap, not after the member is drained")
}
syft/pkg/cataloger/binary/classifier_cataloger_test.go
+24
View File
@@ -407,6 +407,30 @@ func Test_Cataloger_PositiveCases(t *testing.T) {
Metadata: metadata("traefik-binary"),
},
},
{
// regression test: big-endian architectures (e.g. s390x) previously matched an unrelated
// single-NUL-terminated numeric string elsewhere in the binary instead of the real version
logicalFixture: "traefik/3.6.8/linux-s390x",
expected: pkg.Package{
Name: "traefik",
Version: "3.6.8",
Type: "binary",
PURL: "pkg:generic/traefik@3.6.8",
Locations: locations("traefik"),
Metadata: metadata("traefik-binary"),
},
},
{
logicalFixture: "traefik/2.11.1/linux-arm",
expected: pkg.Package{
Name: "traefik",
Version: "2.11.1",
Type: "binary",
PURL: "pkg:generic/traefik@2.11.1",
Locations: locations("traefik"),
Metadata: metadata("traefik-binary"),
},
},
{
logicalFixture: "memcached/1.6.18/linux-amd64",
expected: pkg.Package{
syft/pkg/cataloger/binary/classifiers.go
+9 -2
View File
@@ -284,11 +284,18 @@ func DefaultClassifiers() []binutils.Classifier {
{
Class: "traefik-binary",
FileGlob: "**/traefik",
EvidenceMatcher: m.FileContentsVersionMatcher(
EvidenceMatcher: binutils.MatchAny(
// on some architectures (e.g. s390x) the real version string isn't NUL-prefixed
// and is instead terminated by two NUL bytes; checking this pattern first avoids
// matching an unrelated single-NUL-terminated numeric string elsewhere in the binary
m.FileContentsVersionMatcher(
`(?m)(?P<version>[0-9]+\.[0-9]+\.[0-9]+(-alpha[0-9]|-beta[0-9]|-rc[0-9])?)\x00\x00`),
// [NUL]v1.7.34[NUL]
// [NUL]2.9.6[NUL]
// 3.0.4[NUL]
`(?m)(\x00v?|\x{FFFD}.?)(?P<version>[0-9]+\.[0-9]+\.[0-9]+(-alpha[0-9]|-beta[0-9]|-rc[0-9])?)\x00`),
m.FileContentsVersionMatcher(
`(?m)(\x00v?|\x{FFFD}.?)(?P<version>[0-9]+\.[0-9]+\.[0-9]+(-alpha[0-9]|-beta[0-9]|-rc[0-9])?)\x00`),
),
Package: "traefik",
PURL: mustPURL("pkg:generic/traefik@version"),
CPEs: singleCPE("cpe:2.3:a:traefik:traefik:*:*:*:*:*:*:*:*", cpe.NVDDictionaryLookupSource),
syft/pkg/cataloger/binary/internal/manager/internal/config/application.go
+4 -4
View File
@@ -102,10 +102,10 @@ func (c Application) Validate() error {
}
func (c Application) GetBinaryFromImage(name, version string) *BinaryFromImage {
if strings.Contains(name, "@") && version == "" {
parts := strings.Split(name, "@")
name = parts[0]
version = parts[1]
if version == "" {
if n, v, ok := strings.Cut(name, "@"); ok {
name, version = n, v
}
}
for _, entry := range c.FromImages {
if entry.Name() == name && entry.Version == version {
syft/pkg/cataloger/binary/testdata/classifiers/snippets/traefik/2.11.1/linux-arm/traefik
Vendored
BIN
View File
Binary file not shown.
syft/pkg/cataloger/binary/testdata/classifiers/snippets/traefik/3.6.8/linux-s390x/traefik
Vendored
BIN
View File
Binary file not shown.
syft/pkg/cataloger/debian/parse_copyright.go
+5 -2
View File
@@ -1,7 +1,6 @@
package debian
import (
"bufio"
"io"
"regexp"
"sort"
@@ -10,6 +9,7 @@ import (
"github.com/scylladb/go-set/strset"
"github.com/anchore/syft/internal"
"github.com/anchore/syft/internal/log"
)
// For more information see: https://www.debian.org/doc/packaging-manuals/copyright-format/1.0/#license-syntax
@@ -34,7 +34,7 @@ const (
func parseLicensesFromCopyright(reader io.Reader) []string {
findings := strset.New()
scanner := bufio.NewScanner(reader)
scanner := internal.NewLineScanner(reader)
// per the DEP-5 spec, machine-readable copyright files MUST have a
// Format field whose value is a URI for the specification. Only files
@@ -69,6 +69,9 @@ func parseLicensesFromCopyright(reader io.Reader) []string {
findings.Add(found)
}
}
if err := scanner.Err(); err != nil {
log.WithFields("error", err).Debug("failed to fully read debian copyright file")
}
if !formatVerified {
return nil
syft/pkg/cataloger/debian/parse_deb_archive_test.go
+7 -3
View File
@@ -373,9 +373,11 @@ func Test_processDataTar_capsCopyrightFiles(t *testing.T) {
// than after the fact. The existing bounds test uses a payload small enough that buffering the whole
// thing would pass anyway; this one is large enough that the difference shows up as bytes.
func Test_decompressionStream_failsEarlyInsteadOfBuffering(t *testing.T) {
const expanded = 512 * intFile.MB
// 4x the 8MB budget below is plenty to tell bounded from unbounded. Going bigger only makes the
// fixture slower to build and inflate, which is most of this test's time under -race.
const expanded = 32 * intFile.MB
// a few KB of input expanding to 512MB, so heap is what separates a streaming bound from a
// well under 1% of input expanding to 32MB, so heap is what separates a streaming bound from a
// bound checked after the read completes
var gz bytes.Buffer
gw := gzip.NewWriter(&gz)
@@ -386,7 +388,9 @@ func Test_decompressionStream_failsEarlyInsteadOfBuffering(t *testing.T) {
remaining -= int64(n)
}
require.NoError(t, gw.Close())
require.Less(t, gz.Len(), 1024*1024, "compressed payload should be tiny relative to what it expands to")
// a ratio rather than a fixed size: deflate output for zeros is toolchain dependent (about 510KB on
// go1.26, about 1MB on go1.27), and only the ratio matters to the test
require.Less(t, gz.Len(), expanded/100, "compressed payload should be tiny relative to what it expands to")
// both arms use io.ReadAll on purpose. The stream itself never holds much whatever the cap says,
// so copying to io.Discard measures nothing; what the cap actually protects is a consumer that
syft/pkg/cataloger/erlang/erlang_parser.go
+25 -6
View File
@@ -15,6 +15,10 @@ type erlangNode struct {
var errSkipComments = errors.New("")
var errUnexpectedEnd = errors.New("unexpected end of input")
var errTooDeep = fmt.Errorf("nesting deeper than %d levels", parsing.MaxDepth)
func (e erlangNode) Slice() []erlangNode {
out, ok := e.value.([]erlangNode)
if ok {
@@ -79,7 +83,7 @@ func parseErlang(reader io.Reader) (erlangNode, error) {
}
func parseErlangBlock(data []byte, i *int) (erlangNode, error) {
block, err := parseErlangNode(data, i)
block, err := parseErlangNode(data, i, 0)
if err != nil {
return node(nil), err
}
@@ -89,13 +93,19 @@ func parseErlangBlock(data []byte, i *int) (erlangNode, error) {
return block, nil
}
func parseErlangNode(data []byte, i *int) (erlangNode, error) {
func parseErlangNode(data []byte, i *int, depth int) (erlangNode, error) {
parsing.SkipWhitespace(data, i)
if *i >= len(data) {
return node(nil), errUnexpectedEnd
}
c := data[*i]
switch c {
case '[', '{':
offset := *i + 1
parsing.SkipWhitespace(data, &offset)
if offset >= len(data) {
return node(nil), errUnexpectedEnd
}
c2 := data[offset]
// Add support for empty lists
@@ -104,7 +114,7 @@ func parseErlangNode(data []byte, i *int) (erlangNode, error) {
return node(nil), nil
}
return parseErlangList(data, i)
return parseErlangList(data, i, depth+1)
case '"':
fallthrough
case '\'':
@@ -145,6 +155,9 @@ func parseErlangAngleString(data []byte, i *int) (erlangNode, error) {
}
func parseErlangString(data []byte, i *int) (erlangNode, error) {
if *i >= len(data) {
return node(nil), errUnexpectedEnd
}
delim := data[*i]
*i++
var buf bytes.Buffer
@@ -167,13 +180,16 @@ func parseErlangString(data []byte, i *int) (erlangNode, error) {
return node(nil), fmt.Errorf("unterminated string at %d", *i)
}
func parseErlangList(data []byte, i *int) (erlangNode, error) {
func parseErlangList(data []byte, i *int, depth int) (erlangNode, error) {
if depth > parsing.MaxDepth {
return node(nil), errTooDeep
}
*i++
out := erlangNode{
value: []erlangNode{},
}
for *i < len(data) {
item, err := parseErlangNode(data, i)
item, err := parseErlangNode(data, i, depth)
if err != nil {
if err == errSkipComments {
parsing.SkipWhitespace(data, i)
@@ -183,6 +199,9 @@ func parseErlangList(data []byte, i *int) (erlangNode, error) {
}
out.value = append(out.value.([]erlangNode), item)
parsing.SkipWhitespace(data, i)
if *i >= len(data) {
return node(nil), errUnexpectedEnd
}
c := data[*i]
switch c {
case ',':
@@ -210,7 +229,7 @@ func parseErlangComment(data []byte, i *int) {
// Rest of a line is a comment. Deals with CR, LF and CR/LF
if c == '\n' {
break
} else if c == '\r' && data[*i] == '\n' {
} else if c == '\r' && *i < len(data) && data[*i] == '\n' {
*i++
break
}
syft/pkg/cataloger/erlang/erlang_parser_test.go
+33
View File
@@ -2,6 +2,7 @@ package erlang
import (
"bytes"
"strings"
"testing"
"github.com/stretchr/testify/assert"
@@ -128,3 +129,35 @@ func Test_parseErlangString_escapedByte(t *testing.T) {
require.NoError(t, err)
assert.Equal(t, `a"b`, got.String())
}
func TestParseErlang_deepNesting(t *testing.T) {
// 4MB of '[' overflowed the goroutine stack (fatal, not recoverable) before nesting was bounded
_, err := parseErlang(strings.NewReader(strings.Repeat("[a,", 4<<20)))
require.ErrorIs(t, err, errTooDeep)
// the error echoes a clipped window of the single-line input, not the whole thing
assert.Less(t, len(err.Error()), 1024)
}
func Test_parseErlang_truncated(t *testing.T) {
// each of these read past the end of the input before end-of-input was checked
tests := []struct {
name string
content string
wantErr require.ErrorAssertionFunc
}{
{name: "whitespace only", content: " ", wantErr: require.Error},
{name: "unterminated list", content: "[a", wantErr: require.Error},
{name: "unterminated empty list", content: "[ ", wantErr: require.Error},
{name: "lone binary string opener", content: "<", wantErr: require.Error},
{name: "comment ending in CR", content: "%\r", wantErr: require.NoError},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
var err error
require.NotPanics(t, func() {
_, err = parseErlang(strings.NewReader(tt.content))
})
tt.wantErr(t, err)
})
}
}
syft/pkg/cataloger/generic/cataloger.go
+25 -2
View File
@@ -2,6 +2,8 @@ package generic
import (
"context"
"fmt"
"runtime/debug"
"github.com/anchore/go-logger"
"github.com/anchore/go-sync"
@@ -221,12 +223,24 @@ func (c *Cataloger) Catalog(ctx context.Context, resolver file.Resolver) ([]pkg.
func (c *Cataloger) process(ctx context.Context, resolver file.Resolver, pkgs []pkg.Package, rels []artifact.Relationship, err error) ([]pkg.Package, []artifact.Relationship, error) {
for _, p := range c.processors {
pkgs, rels, err = p.process(ctx, resolver, pkgs, rels, err)
pkgs, rels, err = c.processSafely(ctx, p, resolver, pkgs, rels, err)
}
return pkgs, rels, err
}
func invokeParser(ctx context.Context, resolver file.Resolver, location file.Location, logger logger.Logger, parser Parser, env *Environment) ([]pkg.Package, []artifact.Relationship, error) {
func (c *Cataloger) processSafely(ctx context.Context, p processExecutor, resolver file.Resolver, pkgs []pkg.Package, rels []artifact.Relationship, err error) (outPkgs []pkg.Package, outRels []artifact.Relationship, outErr error) {
defer func() {
if r := recover(); r != nil {
// keep the unprocessed results; there is no single location to attach an unknown to
log.WithFields("cataloger", c.upstreamCataloger, "panic", r).Warn("recovered from panic in package processor")
log.Tracef("processor panic stack:\n%s", debug.Stack())
outPkgs, outRels, outErr = pkgs, rels, err
}
}()
return p.process(ctx, resolver, pkgs, rels, err)
}
func invokeParser(ctx context.Context, resolver file.Resolver, location file.Location, logger logger.Logger, parser Parser, env *Environment) (pkgs []pkg.Package, rels []artifact.Relationship, err error) {
contentReader, err := resolver.FileContentsByLocation(location)
if err != nil {
logger.WithFields("location", location.RealPath, "error", err).Debug("unable to fetch contents")
@@ -234,6 +248,15 @@ func invokeParser(ctx context.Context, resolver file.Resolver, location file.Loc
}
defer internal.CloseAndLogError(contentReader, location.AccessPath)
defer func() {
if r := recover(); r != nil {
// a malformed file must not take down the whole SBOM; the caller records this as an unknown
logger.WithFields("location", location.RealPath, "panic", r).Debug("recovered from panic while parsing file")
logger.Tracef("parser panic stack:\n%s", debug.Stack())
pkgs, rels, err = nil, nil, fmt.Errorf("recovered from panic while parsing file: %v", r)
}
}()
discoveredPackages, discoveredRelationships, err := parser(ctx, resolver, env, file.NewLocationReadCloser(location, contentReader))
if err != nil {
// these errors are propagated up, and are likely to be coordinate errors

Some files were not shown because too many files have changed in this diff Show More