oss/syft
2
0
Fork 0
mirror of https://github.com/anchore/syft.git synced 2026-11-06 01:16:32 +01:00
Code Issues Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'upstream/main' into add-nested-archive-cataloging

Browse Source 
Signed-off-by: Keith Zantow <keith.zantow@anchore.com>
This commit is contained in:
Keith Zantow
2026-09-30 14:55:38 -04:00
parent 8e2cdf2429 d61c2cada3
commit 12d3097741
188 changed files with 5325 additions and 1234 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.binny.yaml
+1 -1
View File
@@ -16,7 +16,7 @@ tools:
# used to upload test fixture cache # used to upload test fixture cache
- name: oras - name: oras
version: version:
want: v1.3.3 want: v1.3.4
method: github-release method: github-release
with: with:
repo: oras-project/oras repo: oras-project/oras
.gitattributes
+7
View File
@@ -0,0 +1,7 @@
# check out text files with lf on every platform (regardless of core.autocrlf), letting git detect binary files
* text=auto eol=lf
# fixtures that must keep their crlf line endings byte for byte
syft/linux/testdata/unprintable -text
syft/pkg/cataloger/bitnami/testdata/no-rel/opt/bitnami/redis/.spdx-redis.spdx -text
syft/pkg/cataloger/java/testdata/jar-metadata/**/META-INF/MANIFEST.MF -text
.github/workflows/release.yaml
+1 -1
View File
@@ -86,7 +86,7 @@ jobs:
# dockers_v2 builds multi-platform manifests in a single buildx invocation, which requires a # dockers_v2 builds multi-platform manifests in a single buildx invocation, which requires a
# docker-container driver builder (the default builder on the runner cannot build multi-platform). # docker-container driver builder (the default builder on the runner cannot build multi-platform).
- name: Set up Docker Buildx - name: Set up Docker Buildx
uses: docker/setup-buildx-action@37fe631027851001ddb9b187196cc803df7f5f0e # v4.3.0 uses: docker/setup-buildx-action@f87e5991a6d7451dcb8d9637bfbc97413f497069 # v4.4.1
- name: Login to Docker Hub - name: Login to Docker Hub
uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f #v4.6.0 uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f #v4.6.0
.github/workflows/validations.yaml
+29 -1
View File
@@ -27,7 +27,7 @@ jobs:
# enough; a 4 vCPU instance can't pull much more than that anyway. # enough; a 4 vCPU instance can't pull much more than that anyway.
# note: s3-cache intentionally omitted -- PR runs are untrusted and must not write to the # note: s3-cache intentionally omitted -- PR runs are untrusted and must not write to the
# shared cache backend that the trusted release workflow reads from (cache poisoning). # shared cache backend that the trusted release workflow reads from (cache poisoning).
runs-on: &test-runner "runs-on=${{ github.run_id }}/cpu=4+8/ram=16+32/family=m5+m6+m7+m8+c5+c6+c7+c8/spot=price-capacity-optimized/volume=100gb" runs-on: &test-runner "runs-on=${{ github.run_id }}/cpu=4+8/ram=16+32/family=m6+m7+m8+c6+c7+c8/spot=price-capacity-optimized/volume=100gb"
permissions: permissions:
contents: read contents: read
steps: steps:
@@ -49,6 +49,13 @@ jobs:
Unit-Test: Unit-Test:
# Note: changing this job name requires making the same update in the .github/workflows/release.yaml pipeline # Note: changing this job name requires making the same update in the .github/workflows/release.yaml pipeline
name: "Unit tests" name: "Unit tests"
runs-on: ubuntu-latest
needs: [Unit-Test-Linux, Unit-Test-Windows]
steps:
- run: echo All unit tests passed!
Unit-Test-Linux:
name: "Unit tests (Linux)"
runs-on: *test-runner runs-on: *test-runner
permissions: permissions:
contents: read contents: read
@@ -68,6 +75,27 @@ jobs:
- name: Check for capability drift - name: Check for capability drift
run: make check-capability-drift run: make check-capability-drift
Unit-Test-Windows:
name: "Unit tests (Windows)"
runs-on: windows-latest
permissions:
contents: read
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 #v7.0.1
with:
persist-credentials: false
# go-make's setup only (not ./.github/actions/bootstrap): no apt, binny tools, or fixture cache are needed
- name: Setup go + go-make tooling
uses: anchore/go-make/.github/actions/setup@fa5421b3bf24c9d18f11a736c5eb4172f2b7d683 # v0.8.1
with:
# keep in sync with the go-version default in ./.github/actions/bootstrap
go-version: "1.26.2"
# the full unit suite leans on linux docker fixtures, so this runs the windows-relevant subset
- name: Run unit tests
run: make unit:windows
Integration-Test: Integration-Test:
# Note: changing this job name requires making the same update in the .github/workflows/release.yaml pipeline # Note: changing this job name requires making the same update in the .github/workflows/release.yaml pipeline
name: "Integration tests" name: "Integration tests"
.golangci.yaml
+8 -1
View File
@@ -7,6 +7,7 @@ linters:
- asciicheck - asciicheck
- bodyclose - bodyclose
- copyloopvar - copyloopvar
- depguard
- dogsled - dogsled
- dupl - dupl
- errcheck - errcheck
@@ -29,6 +30,13 @@ linters:
- unused - unused
- whitespace - whitespace
settings: settings:
depguard:
rules:
yaml:
deny:
# gopkg.in/yaml.v3 is unmaintained and panics on some malformed input (see #5327)
- pkg: gopkg.in/yaml.v3
desc: use go.yaml.in/yaml/v3 instead
funlen: funlen:
lines: 70 lines: 70
statements: 50 statements: 50
@@ -71,7 +79,6 @@ linters:
# do not enable... # do not enable...
# - deadcode # The owner seems to have abandoned the linter. Replaced by "unused". # - deadcode # The owner seems to have abandoned the linter. Replaced by "unused".
# - depguard # We don't have a configuration for this yet
# - goprintffuncname # does not catch all cases and there are exceptions # - goprintffuncname # does not catch all cases and there are exceptions
# - nakedret # does not catch all cases and should not fail a build # - nakedret # does not catch all cases and should not fail a build
# - gochecknoglobals # - gochecknoglobals
.make/main.go
+19 -1
View File
@@ -1,6 +1,7 @@
package main package main
import ( import (
"os"
"path/filepath" "path/filepath"
"runtime" "runtime"
"strconv" "strconv"
@@ -52,7 +53,7 @@ func main() {
raceFlag = " -race" raceFlag = " -race"
} }
Run( Run(
"go test -count=1 -timeout=30m -v"+raceFlag+" ./cmd/syft/internal/test/integration/...", "go test -count=1 -timeout=30m"+raceFlag+" ./cmd/syft/internal/test/integration/...",
run.Env("GODEBUG", "dontfreezetheworld=1"), run.Env("GODEBUG", "dontfreezetheworld=1"),
) )
}, },
@@ -96,6 +97,23 @@ func main() {
}, },
}, },
// windows unit tests: deliberately not hooked into "test" (no RunsOn). The full unit suite leans on
// linux docker fixture images and posix-only assumptions, so this covers the packages that own
// host path handling (where windows bugs live) and skips the tests that need docker image fixtures.
// ponytail: curated package list, widen it as more of the suite is made windows-safe.
Task{
Name: "unit:windows",
Description: "run the windows-relevant subset of unit tests",
Run: func() {
Run("go test -count=1"+
` -skip "Image|Squash|AllLayers|MixFileTypes|GivenCoordinates"`+
" ./syft/internal/windows/... ./syft/internal/fileresolver/... ./syft/source/directorysource/... ./syft/source/filesource/..."+
" ./syft/file ./syft/file/cataloger/filedigest/... ./syft/file/cataloger/filecontent/...",
run.Stdout(os.Stderr),
)
},
},
// default validation pipeline (replaces Taskfile `default`/`pr-validations`/`validations`). // default validation pipeline (replaces Taskfile `default`/`pr-validations`/`validations`).
Task{ Task{
Name: "default", Name: "default",
cmd/syft/cli/ui/handle_attestation.go
+5
View File
@@ -158,6 +158,11 @@ func (l *backgroundLineReader) read(reader io.Reader, stage *progress.Stage) {
l.lock.Unlock() l.lock.Unlock()
} }
if err := s.Err(); err != nil {
log.WithFields("error", err).Debug("unable to read attestation output")
// keep draining so the writer never blocks on a full pipe
_, _ = io.Copy(io.Discard, reader)
}
} }
func (l backgroundLineReader) Lines() []string { func (l backgroundLineReader) Lines() []string {
cmd/syft/internal/test/integration/multiplatform_oci_test.go
+11 -13
View File
@@ -16,19 +16,21 @@ import (
"github.com/anchore/syft/syft/source" "github.com/anchore/syft/syft/source"
) )
// multiplatformImage is busybox:1.38.0 pinned by index digest, since docker hub periodically rebuilds and
// re-pushes official image tags, which changes the per-platform digests asserted below.
const multiplatformImage = "docker.io/library/busybox@sha256:fd7dc98638c8e305f4dc34e979f1c0fdfdcaeb0fbf8fcff77ae834b6da3d7e6e"
// TestMultiPlatformOCIImageSelection verifies that syft can load a single platform out of a // TestMultiPlatformOCIImageSelection verifies that syft can load a single platform out of a
// multi-platform OCI image on disk (both --from oci-dir and --from oci-archive) by selecting the // multi-platform OCI image on disk (both --from oci-dir and --from oci-archive) by selecting the
// image that matches the requested platform. // image that matches the requested platform.
func TestMultiPlatformOCIImageSelection(t *testing.T) { func TestMultiPlatformOCIImageSelection(t *testing.T) {
remoteImage := "docker.io/library/busybox:1.38.0"
// Per-platform image config digests within the multi-platform index, obtained from the OCI layout // Per-platform image config digests within the multi-platform index, obtained from the OCI layout
// (see anchore/stereoscope integration tests: TestPlatformSelectionWithOciLocalSources). // (see anchore/stereoscope integration tests: TestPlatformSelectionWithOciLocalSources).
expectedDigest := map[string]string{ expectedDigest := map[string]string{
"arm64": "sha256:e0e8b3cbfed68a90084781e2962f9c0deead51c5a3f11a488eef0283a4284bc2", "arm64": "sha256:d2482869a6b838d3b4c9cad0c8085c06277909482771eb525d298fc3a7d07927",
"s390x": "sha256:0cf160e720a8e4f20883b72276a6eaded83b79539f3f1d39e35a3336154b9960", "s390x": "sha256:e55a0fa55032cceac2d983f4b4a25b28e42303b46a2f1915fc0fa45051e6ca70",
"amd64": "sha256:c6348fa86ba0fb2108c9334f5fe913ddc6d853313e655891f133a0127c30099f", "amd64": "sha256:aaef90e065235eb0b2d9938be85d1b81add2902cf89affe2cdfa19be33476ac0",
"ppc64le": "sha256:b144fc0e06537d07956cde340b878a81a717e394edb736d3110858a12a6635cb", "ppc64le": "sha256:8158829b015d1c5285d63d54d37e95998ef0af4177e3f416678decbcaff3170c",
} }
// syft --from source tag -> stereoscope OCI source used to prepare the local fixture // syft --from source tag -> stereoscope OCI source used to prepare the local fixture
@@ -39,7 +41,7 @@ func TestMultiPlatformOCIImageSelection(t *testing.T) {
for from, imageSource := range sources { for from, imageSource := range sources {
t.Run(from, func(t *testing.T) { t.Run(from, func(t *testing.T) {
localPath := imagetest.PrepareMultiplatformFixtureImage(t, imageSource, remoteImage) localPath := imagetest.PrepareMultiplatformFixtureImage(t, imageSource, multiplatformImage)
for _, arch := range []string{"amd64", "arm64", "s390x", "ppc64le"} { for _, arch := range []string{"amd64", "arm64", "s390x", "ppc64le"} {
t.Run(fmt.Sprintf("linux/%s", arch), func(t *testing.T) { t.Run(fmt.Sprintf("linux/%s", arch), func(t *testing.T) {
platform, err := image.NewPlatform("linux/" + arch) platform, err := image.NewPlatform("linux/" + arch)
@@ -73,8 +75,6 @@ func TestMultiPlatformOCIImageSelection(t *testing.T) {
// TestMultiPlatformOCIImageSelection_UnavailablePlatform verifies that requesting a platform not present // TestMultiPlatformOCIImageSelection_UnavailablePlatform verifies that requesting a platform not present
// in the multi-platform OCI image results in an error rather than silently selecting the wrong image. // in the multi-platform OCI image results in an error rather than silently selecting the wrong image.
func TestMultiPlatformOCIImageSelection_UnavailablePlatform(t *testing.T) { func TestMultiPlatformOCIImageSelection_UnavailablePlatform(t *testing.T) {
remoteImage := "docker.io/library/busybox:1.38.0"
// windows/amd64 is not present in this linux-only multi-platform image // windows/amd64 is not present in this linux-only multi-platform image
platform, err := image.NewPlatform("windows/amd64") platform, err := image.NewPlatform("windows/amd64")
require.NoError(t, err) require.NoError(t, err)
@@ -86,7 +86,7 @@ func TestMultiPlatformOCIImageSelection_UnavailablePlatform(t *testing.T) {
for from, imageSource := range sources { for from, imageSource := range sources {
t.Run(from, func(t *testing.T) { t.Run(from, func(t *testing.T) {
localPath := imagetest.PrepareMultiplatformFixtureImage(t, imageSource, remoteImage) localPath := imagetest.PrepareMultiplatformFixtureImage(t, imageSource, multiplatformImage)
_, err := syft.GetSource( _, err := syft.GetSource(
context.Background(), context.Background(),
@@ -101,8 +101,6 @@ func TestMultiPlatformOCIImageSelection_UnavailablePlatform(t *testing.T) {
// TestMultiPlatformOCIImageSelection_DefaultPlatform verifies that not specifying a platform results // TestMultiPlatformOCIImageSelection_DefaultPlatform verifies that not specifying a platform results
// in the current platform being selected. // in the current platform being selected.
func TestMultiPlatformOCIImageSelection_DefaultPlatform(t *testing.T) { func TestMultiPlatformOCIImageSelection_DefaultPlatform(t *testing.T) {
remoteImage := "docker.io/library/busybox:1.38.0"
sources := map[string]image.Source{ sources := map[string]image.Source{
"oci-dir": image.OciDirectorySource, "oci-dir": image.OciDirectorySource,
"oci-archive": image.OciTarballSource, "oci-archive": image.OciTarballSource,
@@ -110,7 +108,7 @@ func TestMultiPlatformOCIImageSelection_DefaultPlatform(t *testing.T) {
for from, imageSource := range sources { for from, imageSource := range sources {
t.Run(from, func(t *testing.T) { t.Run(from, func(t *testing.T) {
localPath := imagetest.PrepareMultiplatformFixtureImage(t, imageSource, remoteImage) localPath := imagetest.PrepareMultiplatformFixtureImage(t, imageSource, multiplatformImage)
src, err := syft.GetSource( src, err := syft.GetSource(
context.Background(), context.Background(),
cmd/syft/internal/ui/ui.go
+45 -1
View File
@@ -1,6 +1,7 @@
package ui package ui
import ( import (
"bytes"
"fmt" "fmt"
"io" "io"
"os" "os"
@@ -9,6 +10,7 @@ import (
"time" "time"
tea "github.com/charmbracelet/bubbletea" tea "github.com/charmbracelet/bubbletea"
"github.com/charmbracelet/lipgloss"
"github.com/wagoodman/go-partybus" "github.com/wagoodman/go-partybus"
"github.com/anchore/bubbly" "github.com/anchore/bubbly"
@@ -53,7 +55,7 @@ func New(out io.Writer, quiet bool, handlers ...bubbly.EventHandler) *UI {
func (m *UI) Setup(subscription partybus.Unsubscribable) error { func (m *UI) Setup(subscription partybus.Unsubscribable) error {
// we still want to collect log messages, however, we also the logger shouldn't write to the screen directly // we still want to collect log messages, however, we also the logger shouldn't write to the screen directly
if logWrapper, ok := log.Get().(logger.Controller); ok { if logWrapper, ok := log.Get().(logger.Controller); ok {
logWrapper.SetOutput(m.frame.(*frame.Frame).Footer()) logWrapper.SetOutput(&logFooter{w: m.frame.(*frame.Frame).Footer()})
} }
m.subscription = subscription m.subscription = subscription
@@ -193,6 +195,48 @@ func (m UI) View() string {
return v return v
} }
var (
logTitleStyle = lipgloss.NewStyle().Foreground(lipgloss.Color("8")) // dark grey
// the first log line gets a branch pointing at it from the title, the rest are indented to match
logBranch = logTitleStyle.Render("└─▶") + " "
logIndent = " "
)
// logFooter decorates log entries drawn below the TUI with a one-time title (written when the first entry arrives)
// and a branch from the title to the first log line. Writes are serialized by the logger.
type logFooter struct {
w io.Writer
started bool
}
func (l *logFooter) Write(p []byte) (int, error) {
var buf bytes.Buffer
first := !l.started
if first {
buf.WriteString(logTitleStyle.Render("Logs (non-fatal, see -v for more):") + "\n")
l.started = true
}
for i, line := range bytes.SplitAfter(p, []byte("\n")) {
if len(line) == 0 {
continue
}
if first && i == 0 {
buf.WriteString(logBranch)
} else {
buf.WriteString(logIndent)
}
buf.Write(line)
}
if _, err := l.w.Write(buf.Bytes()); err != nil {
return 0, err
}
return len(p), nil
}
// IsTerminal marks the footer as terminal-bound (it is drawn by the TUI) so the logger keeps colors. NO_COLOR is
// still honored by the formatter.
func (*logFooter) IsTerminal() bool { return true }
func runWithTimeout(timeout time.Duration, fn func() error) (err error) { func runWithTimeout(timeout time.Duration, fn func() error) (err error) {
c := make(chan struct{}, 1) c := make(chan struct{}, 1)
go func() { go func() {
go.mod
+13 -13
View File
@@ -1,6 +1,6 @@
module github.com/anchore/syft module github.com/anchore/syft
go 1.26.3 go 1.26.8
require ( require (
github.com/BurntSushi/toml v1.6.0 github.com/BurntSushi/toml v1.6.0
@@ -11,12 +11,12 @@ require (
github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d
github.com/acobaugh/osrelease v0.1.0 github.com/acobaugh/osrelease v0.1.0
github.com/adrg/xdg v0.5.3 github.com/adrg/xdg v0.5.3
github.com/anchore/bubbly v0.2.1 github.com/anchore/bubbly v0.2.2
github.com/anchore/clio v0.1.1 github.com/anchore/clio v0.1.1
github.com/anchore/fangs v0.1.1 github.com/anchore/fangs v0.1.1
github.com/anchore/go-collections v0.1.1 github.com/anchore/go-collections v0.1.1
github.com/anchore/go-homedir v0.1.1 github.com/anchore/go-homedir v0.1.1
github.com/anchore/go-logger v0.1.1 github.com/anchore/go-logger v0.2.0
github.com/anchore/go-macholibre v0.1.1 github.com/anchore/go-macholibre v0.1.1
github.com/anchore/go-rpmdb v0.2.0 github.com/anchore/go-rpmdb v0.2.0
github.com/anchore/go-sync v0.1.2 github.com/anchore/go-sync v0.1.2
@@ -36,7 +36,7 @@ require (
github.com/deitch/magic v0.0.0-20230404182410-1ff89d7342da github.com/deitch/magic v0.0.0-20230404182410-1ff89d7342da
github.com/diskfs/go-diskfs v1.9.4 github.com/diskfs/go-diskfs v1.9.4
github.com/distribution/reference v0.6.0 github.com/distribution/reference v0.6.0
github.com/dustin/go-humanize v1.0.1 github.com/dustin/go-humanize v1.1.0
github.com/elliotchance/phpserialize v1.4.0 github.com/elliotchance/phpserialize v1.4.0
github.com/facebookincubator/nvdtools v0.1.5 github.com/facebookincubator/nvdtools v0.1.5
github.com/github/go-spdx/v2 v2.7.0 github.com/github/go-spdx/v2 v2.7.0
@@ -56,18 +56,18 @@ require (
github.com/hashicorp/go-cleanhttp v0.5.2 github.com/hashicorp/go-cleanhttp v0.5.2
github.com/hashicorp/go-getter v1.8.9 github.com/hashicorp/go-getter v1.8.9
github.com/hashicorp/go-multierror v1.1.1 github.com/hashicorp/go-multierror v1.1.1
github.com/hashicorp/hcl/v2 v2.24.0 github.com/hashicorp/hcl/v2 v2.25.0
github.com/iancoleman/strcase v0.3.0 github.com/iancoleman/strcase v0.3.0
github.com/invopop/jsonschema v0.14.0 github.com/invopop/jsonschema v0.14.0
github.com/jedib0t/go-pretty/v6 v6.8.3 github.com/jedib0t/go-pretty/v6 v6.8.3
github.com/jinzhu/copier v0.4.0 github.com/jinzhu/copier v0.4.0
github.com/kastenhq/goversion v0.0.0-20230811215019-93b2f8823953 github.com/kastenhq/goversion v0.0.0-20230811215019-93b2f8823953
github.com/klauspost/compress v1.20.0 github.com/klauspost/compress v1.20.0
github.com/magiconair/properties v1.18.11 github.com/magiconair/properties v1.18.12
github.com/mholt/archives v0.1.5 github.com/mholt/archives v0.1.5
github.com/moby/sys/mountinfo v0.7.2 github.com/moby/sys/mountinfo v0.7.2
github.com/nix-community/go-nix v0.0.0-20250101154619-4bdde671e0a1 github.com/nix-community/go-nix v0.0.0-20250101154619-4bdde671e0a1
github.com/olekukonko/tablewriter v1.1.4 github.com/olekukonko/tablewriter v1.1.5
github.com/opencontainers/go-digest v1.0.0 github.com/opencontainers/go-digest v1.0.0
github.com/pelletier/go-toml v1.9.5 github.com/pelletier/go-toml v1.9.5
github.com/quasilyte/go-ruleguard/dsl v0.3.23 github.com/quasilyte/go-ruleguard/dsl v0.3.23
@@ -97,8 +97,7 @@ require (
golang.org/x/net v0.59.0 golang.org/x/net v0.59.0
golang.org/x/time v0.16.0 golang.org/x/time v0.16.0
golang.org/x/tools v0.50.0 golang.org/x/tools v0.50.0
gopkg.in/yaml.v3 v3.0.1 modernc.org/sqlite v1.59.0
modernc.org/sqlite v1.58.0
) )
require ( require (
@@ -131,6 +130,7 @@ require (
github.com/anchore/go-struct-converter v0.2.0-rc2 // indirect github.com/anchore/go-struct-converter v0.2.0-rc2 // indirect
github.com/andybalholm/brotli v1.2.0 // indirect github.com/andybalholm/brotli v1.2.0 // indirect
github.com/apparentlymart/go-textseg/v15 v15.0.0 // indirect github.com/apparentlymart/go-textseg/v15 v15.0.0 // indirect
github.com/apparentlymart/go-textseg/v17 v17.0.1 // indirect
github.com/aquasecurity/go-version v0.0.1 // indirect github.com/aquasecurity/go-version v0.0.1 // indirect
github.com/atotto/clipboard v0.1.4 // indirect github.com/atotto/clipboard v0.1.4 // indirect
github.com/aws/aws-sdk-go-v2 v1.44.0 // indirect github.com/aws/aws-sdk-go-v2 v1.44.0 // indirect
@@ -170,7 +170,7 @@ require (
github.com/cncf/xds/go v0.0.0-20260202195803-dba9d589def2 // indirect github.com/cncf/xds/go v0.0.0-20260202195803-dba9d589def2 // indirect
github.com/containerd/cgroups/v3 v3.1.3 // indirect github.com/containerd/cgroups/v3 v3.1.3 // indirect
github.com/containerd/containerd/api v1.11.1 // indirect github.com/containerd/containerd/api v1.11.1 // indirect
github.com/containerd/containerd/v2 v2.3.5 // indirect github.com/containerd/containerd/v2 v2.3.6 // indirect
github.com/containerd/continuity v0.5.0 // indirect github.com/containerd/continuity v0.5.0 // indirect
github.com/containerd/errdefs v1.0.0 // indirect github.com/containerd/errdefs v1.0.0 // indirect
github.com/containerd/errdefs/pkg v0.3.0 // indirect github.com/containerd/errdefs/pkg v0.3.0 // indirect
@@ -248,7 +248,7 @@ require (
github.com/muesli/cancelreader v0.2.2 // indirect github.com/muesli/cancelreader v0.2.2 // indirect
github.com/muesli/termenv v0.16.0 // indirect github.com/muesli/termenv v0.16.0 // indirect
github.com/ncruces/go-strftime v1.0.0 // indirect github.com/ncruces/go-strftime v1.0.0 // indirect
github.com/nwaples/rardecode/v2 v2.2.0 // indirect github.com/nwaples/rardecode/v2 v2.4.1 // indirect
github.com/olekukonko/cat v0.0.0-20250911104152-50322a0618f6 // indirect github.com/olekukonko/cat v0.0.0-20250911104152-50322a0618f6 // indirect
github.com/olekukonko/errors v1.2.0 // indirect github.com/olekukonko/errors v1.2.0 // indirect
github.com/olekukonko/ll v0.1.6 // indirect github.com/olekukonko/ll v0.1.6 // indirect
@@ -293,7 +293,7 @@ require (
github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 // indirect github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 // indirect
github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8 // indirect github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8 // indirect
github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e // indirect github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e // indirect
github.com/zclconf/go-cty v1.16.3 // indirect github.com/zclconf/go-cty v1.19.0 // indirect
go.opencensus.io v0.24.0 // indirect go.opencensus.io v0.24.0 // indirect
go.opentelemetry.io/auto/sdk v1.2.1 // indirect go.opentelemetry.io/auto/sdk v1.2.1 // indirect
go.opentelemetry.io/contrib/detectors/gcp v1.44.0 // indirect go.opentelemetry.io/contrib/detectors/gcp v1.44.0 // indirect
@@ -321,7 +321,7 @@ require (
google.golang.org/grpc v1.83.2 // indirect google.golang.org/grpc v1.83.2 // indirect
google.golang.org/protobuf v1.36.12 // indirect google.golang.org/protobuf v1.36.12 // indirect
gopkg.in/warnings.v0 v0.1.2 // indirect gopkg.in/warnings.v0 v0.1.2 // indirect
modernc.org/libc v1.75.6 // indirect modernc.org/libc v1.75.7 // indirect
modernc.org/mathutil v1.7.1 // indirect modernc.org/mathutil v1.7.1 // indirect
modernc.org/memory v1.12.1 // indirect modernc.org/memory v1.12.1 // indirect
) )
go.sum
+24 -22
View File
@@ -120,8 +120,8 @@ github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuy
github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
github.com/anchore/bubbly v0.2.1 h1:b3ZNBefY8ks5DI4SQ7BXSbBIsN6eoaTD6s1su7NuCBo= github.com/anchore/bubbly v0.2.2 h1:53JHRlpaNZjMcUYdz7n1U7+kOhy0zsJv9gpGEXvTcz8=
github.com/anchore/bubbly v0.2.1/go.mod h1:o6a9aH3/fCn+mb5g1T1tBbQZ4hGR0yO5/uD1d2Vtago= github.com/anchore/bubbly v0.2.2/go.mod h1:M4twoFkczlfuNdU8QV7oQjVJwPbxdUWRMfKydcLba0U=
github.com/anchore/clio v0.1.1 h1:9bkYnaFbpT0J3G0qSHyQqa/lpK5zV0C9Gtgh82H/jj0= github.com/anchore/clio v0.1.1 h1:9bkYnaFbpT0J3G0qSHyQqa/lpK5zV0C9Gtgh82H/jj0=
github.com/anchore/clio v0.1.1/go.mod h1:CFPIopJlakJbBpKtAL/EPpj4W9hgBO7PfFc8N2FAtlE= github.com/anchore/clio v0.1.1/go.mod h1:CFPIopJlakJbBpKtAL/EPpj4W9hgBO7PfFc8N2FAtlE=
github.com/anchore/fangs v0.1.1 h1:NBTHocmWM2/0gNhebpoE/wEB0Z0nxh2lESww2o/Uf5M= github.com/anchore/fangs v0.1.1 h1:NBTHocmWM2/0gNhebpoE/wEB0Z0nxh2lESww2o/Uf5M=
@@ -130,8 +130,8 @@ github.com/anchore/go-collections v0.1.1 h1:kGlmSehs+ctTBZr+98F2+taHeolHy+XDx1Nc
github.com/anchore/go-collections v0.1.1/go.mod h1:zgDbPSNDpfU47tVo7aW+2suG7+vXB1Sq1qtDhRJknQ0= github.com/anchore/go-collections v0.1.1/go.mod h1:zgDbPSNDpfU47tVo7aW+2suG7+vXB1Sq1qtDhRJknQ0=
github.com/anchore/go-homedir v0.1.1 h1:KpaO5BPrVT7cVMcEZ9KfFfEi0iuo3DtagPGl0C744X0= github.com/anchore/go-homedir v0.1.1 h1:KpaO5BPrVT7cVMcEZ9KfFfEi0iuo3DtagPGl0C744X0=
github.com/anchore/go-homedir v0.1.1/go.mod h1:9B0DGhbmMAJVGGEbrlb+PkORM5eDFCEOtZ3xQ22qaLA= github.com/anchore/go-homedir v0.1.1/go.mod h1:9B0DGhbmMAJVGGEbrlb+PkORM5eDFCEOtZ3xQ22qaLA=
github.com/anchore/go-logger v0.1.1 h1:HPBNmQVBdYliwaFVUL7oJ3UvSlaoXAMj/ZagAIYaCpA= github.com/anchore/go-logger v0.2.0 h1:V8Gazz+F8xwWEWz+whqt8/5163IkO4XrAie+1qbK10k=
github.com/anchore/go-logger v0.1.1/go.mod h1:ekWuh5BkZVwyXnyEd4fwL29N4pNLssgRsqh2+kFN/b8= github.com/anchore/go-logger v0.2.0/go.mod h1:omqLWBwT/B5/aTwIH6pl5WBylBzeH3BGHUTS6OFeFvY=
github.com/anchore/go-lzo v0.1.1 h1:IwL/fvkdtlIrYIXck6WxZ3nb8WjjHziYYmGxlooyOnM= github.com/anchore/go-lzo v0.1.1 h1:IwL/fvkdtlIrYIXck6WxZ3nb8WjjHziYYmGxlooyOnM=
github.com/anchore/go-lzo v0.1.1/go.mod h1:3kLx0bve2oN1iDwgM1U5zGku1Tfbdb0No5qp1eL1fIk= github.com/anchore/go-lzo v0.1.1/go.mod h1:3kLx0bve2oN1iDwgM1U5zGku1Tfbdb0No5qp1eL1fIk=
github.com/anchore/go-macholibre v0.1.1 h1:sTBHi3S1WVIOCamc6NyT8Sn9mBmD30DCTfjfVm4k6/M= github.com/anchore/go-macholibre v0.1.1 h1:sTBHi3S1WVIOCamc6NyT8Sn9mBmD30DCTfjfVm4k6/M=
@@ -156,6 +156,8 @@ github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be/go.mod h1:ySMOLuW
github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY=
github.com/apparentlymart/go-textseg/v15 v15.0.0 h1:uYvfpb3DyLSCGWnctWKGj857c6ew1u1fNQOlOtuGxQY= github.com/apparentlymart/go-textseg/v15 v15.0.0 h1:uYvfpb3DyLSCGWnctWKGj857c6ew1u1fNQOlOtuGxQY=
github.com/apparentlymart/go-textseg/v15 v15.0.0/go.mod h1:K8XmNZdhEBkdlyDdvbmmsvpAG721bKi0joRfFdHIWJ4= github.com/apparentlymart/go-textseg/v15 v15.0.0/go.mod h1:K8XmNZdhEBkdlyDdvbmmsvpAG721bKi0joRfFdHIWJ4=
github.com/apparentlymart/go-textseg/v17 v17.0.1 h1:bpMXRgQ5cEoRNuQke1a80/Nl6w3G5eoIbWo9f3gXkAs=
github.com/apparentlymart/go-textseg/v17 v17.0.1/go.mod h1:fa8X4jgGeevslICIY6LcdjkSecWnXmYd9Lk34z/VxZs=
github.com/aquasecurity/go-pep440-version v0.0.1 h1:8VKKQtH2aV61+0hovZS3T//rUF+6GDn18paFTVS0h0M= github.com/aquasecurity/go-pep440-version v0.0.1 h1:8VKKQtH2aV61+0hovZS3T//rUF+6GDn18paFTVS0h0M=
github.com/aquasecurity/go-pep440-version v0.0.1/go.mod h1:3naPe+Bp6wi3n4l5iBFCZgS0JG8vY6FT0H4NGhFJ+i4= github.com/aquasecurity/go-pep440-version v0.0.1/go.mod h1:3naPe+Bp6wi3n4l5iBFCZgS0JG8vY6FT0H4NGhFJ+i4=
github.com/aquasecurity/go-version v0.0.1 h1:4cNl516agK0TCn5F7mmYN+xVs1E3S45LkgZk3cbaW2E= github.com/aquasecurity/go-version v0.0.1 h1:4cNl516agK0TCn5F7mmYN+xVs1E3S45LkgZk3cbaW2E=
@@ -294,8 +296,8 @@ github.com/containerd/cgroups/v3 v3.1.3 h1:eUNflyMddm18+yrDmZPn3jI7C5hJ9ahABE5q6
github.com/containerd/cgroups/v3 v3.1.3/go.mod h1:PKZ2AcWmSBsY/tJUVhtS/rluX0b1uq1GmPO1ElCmbOw= github.com/containerd/cgroups/v3 v3.1.3/go.mod h1:PKZ2AcWmSBsY/tJUVhtS/rluX0b1uq1GmPO1ElCmbOw=
github.com/containerd/containerd/api v1.11.1 h1:h8nfoDW9+fNsC/9TwiAHj8B1GzXKtR4eFtkhi/X5RLU= github.com/containerd/containerd/api v1.11.1 h1:h8nfoDW9+fNsC/9TwiAHj8B1GzXKtR4eFtkhi/X5RLU=
github.com/containerd/containerd/api v1.11.1/go.mod h1:CaQFRu+N1MtbgL6JDOJLUB1hCKESU1lD6MuTJhgtdlw= github.com/containerd/containerd/api v1.11.1/go.mod h1:CaQFRu+N1MtbgL6JDOJLUB1hCKESU1lD6MuTJhgtdlw=
github.com/containerd/containerd/v2 v2.3.5 h1:9MYlI81gUcOZ0WsCkSMtvOU7rTR3hqAoa2eCzhoLlkA= github.com/containerd/containerd/v2 v2.3.6 h1:huoqZXaDW1x1kgPvFC4rMwxv92vUJnqgq7I5b1rSddM=
github.com/containerd/containerd/v2 v2.3.5/go.mod h1:RXDyLPaI3zoO7dFdAW9/54W4cix+z3A6larieufC9mg= github.com/containerd/containerd/v2 v2.3.6/go.mod h1:qaUSWKk9EchqbudSbhv1MxI8dXKGYHLpZtMQdkEkVZ4=
github.com/containerd/continuity v0.5.0 h1:7a85HZpCSs+1Zps0Ee3DPSuAWY+0SJM1JNM51nlEVDg= github.com/containerd/continuity v0.5.0 h1:7a85HZpCSs+1Zps0Ee3DPSuAWY+0SJM1JNM51nlEVDg=
github.com/containerd/continuity v0.5.0/go.mod h1:/lNJvtJKUQStBzpVQ1+rasXO1LAWtUQssk28EZvJ3nE= github.com/containerd/continuity v0.5.0/go.mod h1:/lNJvtJKUQStBzpVQ1+rasXO1LAWtUQssk28EZvJ3nE=
github.com/containerd/errdefs v1.0.0 h1:tg5yIfIlQIrxYtu9ajqY42W3lpS19XqdxRQeEwYG8PI= github.com/containerd/errdefs v1.0.0 h1:tg5yIfIlQIrxYtu9ajqY42W3lpS19XqdxRQeEwYG8PI=
@@ -348,8 +350,8 @@ github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDD
github.com/dsnet/compress v0.0.2-0.20230904184137-39efe44ab707 h1:2tV76y6Q9BB+NEBasnqvs7e49aEBFI8ejC89PSnWH+4= github.com/dsnet/compress v0.0.2-0.20230904184137-39efe44ab707 h1:2tV76y6Q9BB+NEBasnqvs7e49aEBFI8ejC89PSnWH+4=
github.com/dsnet/compress v0.0.2-0.20230904184137-39efe44ab707/go.mod h1:qssHWj60/X5sZFNxpG4HBPDHVqxNm4DfnCKgrbZOT+s= github.com/dsnet/compress v0.0.2-0.20230904184137-39efe44ab707/go.mod h1:qssHWj60/X5sZFNxpG4HBPDHVqxNm4DfnCKgrbZOT+s=
github.com/dsnet/golib v0.0.0-20171103203638-1ea166775780/go.mod h1:Lj+Z9rebOhdfkVLjJ8T6VcRQv3SXugXy999NBtR9aFY= github.com/dsnet/golib v0.0.0-20171103203638-1ea166775780/go.mod h1:Lj+Z9rebOhdfkVLjJ8T6VcRQv3SXugXy999NBtR9aFY=
github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= github.com/dustin/go-humanize v1.1.0 h1:dbKTrvD0klcbBV/h4AWJdMuZogJACoMlvWIWZ5b2xWg=
github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= github.com/dustin/go-humanize v1.1.0/go.mod h1:hc1CvRkJMsgxqjmjMQF3QNRAZBwY8AXBAzKYoSX9sFI=
github.com/elazarl/goproxy v1.7.2 h1:Y2o6urb7Eule09PjlhQRGNsqRfPmYI3KKQLFpCAV3+o= github.com/elazarl/goproxy v1.7.2 h1:Y2o6urb7Eule09PjlhQRGNsqRfPmYI3KKQLFpCAV3+o=
github.com/elazarl/goproxy v1.7.2/go.mod h1:82vkLNir0ALaW14Rc399OTTjyNREgmdL2cVoIbS6XaE= github.com/elazarl/goproxy v1.7.2/go.mod h1:82vkLNir0ALaW14Rc399OTTjyNREgmdL2cVoIbS6XaE=
github.com/elliotchance/phpserialize v1.4.0 h1:cAp/9+KSnEbUC8oYCE32n2n84BeW8HOY3HMDI8hG2OY= github.com/elliotchance/phpserialize v1.4.0 h1:cAp/9+KSnEbUC8oYCE32n2n84BeW8HOY3HMDI8hG2OY=
@@ -595,8 +597,8 @@ github.com/hashicorp/golang-lru v0.5.4/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uG
github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k= github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k=
github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM= github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM=
github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ=
github.com/hashicorp/hcl/v2 v2.24.0 h1:2QJdZ454DSsYGoaE6QheQZjtKZSUs9Nh2izTWiwQxvE= github.com/hashicorp/hcl/v2 v2.25.0 h1:HmmQVYRny4MaBo4b20TjmL46wyuUxpnMWkPZ4+NTbWk=
github.com/hashicorp/hcl/v2 v2.24.0/go.mod h1:oGoO1FIQYfn/AgyOhlg9qLC6/nOJPX3qGbkZpYAcqfM= github.com/hashicorp/hcl/v2 v2.25.0/go.mod h1:vR+FKETxoZAmRlHgFfKmuqivj+C4Izm/c66XkmZ3r7M=
github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO+LraFDTW64= github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO+LraFDTW64=
github.com/hashicorp/mdns v1.0.1/go.mod h1:4gW7WsVCke5TE7EPeYliwHlRUyBtfCwuFwuMg2DmyNY= github.com/hashicorp/mdns v1.0.1/go.mod h1:4gW7WsVCke5TE7EPeYliwHlRUyBtfCwuFwuMg2DmyNY=
github.com/hashicorp/mdns v1.0.4/go.mod h1:mtBihi+LeNXGtG8L9dX59gAEa12BDtBQSp4v/YAJqrc= github.com/hashicorp/mdns v1.0.4/go.mod h1:mtBihi+LeNXGtG8L9dX59gAEa12BDtBQSp4v/YAJqrc=
@@ -668,8 +670,8 @@ github.com/lucasb-eyer/go-colorful v1.3.0 h1:2/yBRLdWBZKrf7gB40FoiKfAWYQ0lqNcbuQ
github.com/lucasb-eyer/go-colorful v1.3.0/go.mod h1:R4dSotOR9KMtayYi1e77YzuveK+i7ruzyGqttikkLy0= github.com/lucasb-eyer/go-colorful v1.3.0/go.mod h1:R4dSotOR9KMtayYi1e77YzuveK+i7ruzyGqttikkLy0=
github.com/lyft/protoc-gen-star v0.5.3/go.mod h1:V0xaHgaf5oCCqmcxYcWiDfTiKsZsRc87/1qhoTACD8w= github.com/lyft/protoc-gen-star v0.5.3/go.mod h1:V0xaHgaf5oCCqmcxYcWiDfTiKsZsRc87/1qhoTACD8w=
github.com/magiconair/properties v1.8.5/go.mod h1:y3VJvCyxH9uVvJTWEGAELF3aiYNyPKd5NZ3oSwXrF60= github.com/magiconair/properties v1.8.5/go.mod h1:y3VJvCyxH9uVvJTWEGAELF3aiYNyPKd5NZ3oSwXrF60=
github.com/magiconair/properties v1.18.11 h1:j5ozYZl0zCjG7ahMDH0GWIobOvvUzT0BdAguG0ViKy0= github.com/magiconair/properties v1.18.12 h1:sT9zQpvTB3B4gzrX0tmZNTEaGyg8Zw55MFYRE32Mr9I=
github.com/magiconair/properties v1.18.11/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3vdS329zhj2hYo0= github.com/magiconair/properties v1.18.12/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3vdS329zhj2hYo0=
github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc=
github.com/maruel/natural v1.3.0 h1:VsmCsBmEyrR46RomtgHs5hbKADGRVtliHTyCOLFBpsg= github.com/maruel/natural v1.3.0 h1:VsmCsBmEyrR46RomtgHs5hbKADGRVtliHTyCOLFBpsg=
github.com/maruel/natural v1.3.0/go.mod h1:v+Rfd79xlw1AgVBjbO0BEQmptqb5HvL/k9GRHB7ZKEg= github.com/maruel/natural v1.3.0/go.mod h1:v+Rfd79xlw1AgVBjbO0BEQmptqb5HvL/k9GRHB7ZKEg=
@@ -756,16 +758,16 @@ github.com/nix-community/go-nix v0.0.0-20250101154619-4bdde671e0a1 h1:kpt9ZfKcm+
github.com/nix-community/go-nix v0.0.0-20250101154619-4bdde671e0a1/go.mod h1:qgCw4bBKZX8qMgGeEZzGFVT3notl42dBjNqO2jut0M0= github.com/nix-community/go-nix v0.0.0-20250101154619-4bdde671e0a1/go.mod h1:qgCw4bBKZX8qMgGeEZzGFVT3notl42dBjNqO2jut0M0=
github.com/nsf/jsondiff v0.0.0-20210926074059-1e845ec5d249 h1:NHrXEjTNQY7P0Zfx1aMrNhpgxHmow66XQtm0aQLY0AE= github.com/nsf/jsondiff v0.0.0-20210926074059-1e845ec5d249 h1:NHrXEjTNQY7P0Zfx1aMrNhpgxHmow66XQtm0aQLY0AE=
github.com/nsf/jsondiff v0.0.0-20210926074059-1e845ec5d249/go.mod h1:mpRZBD8SJ55OIICQ3iWH0Yz3cjzA61JdqMLoWXeB2+8= github.com/nsf/jsondiff v0.0.0-20210926074059-1e845ec5d249/go.mod h1:mpRZBD8SJ55OIICQ3iWH0Yz3cjzA61JdqMLoWXeB2+8=
github.com/nwaples/rardecode/v2 v2.2.0 h1:4ufPGHiNe1rYJxYfehALLjup4Ls3ck42CWwjKiOqu0A= github.com/nwaples/rardecode/v2 v2.4.1 h1:F7zNW2LdAuuBThHWXQaiFUGVD/sef299NfWSB1nHAl4=
github.com/nwaples/rardecode/v2 v2.2.0/go.mod h1:7uz379lSxPe6j9nvzxUZ+n7mnJNgjsRNb6IbvGVHRmw= github.com/nwaples/rardecode/v2 v2.4.1/go.mod h1:7uz379lSxPe6j9nvzxUZ+n7mnJNgjsRNb6IbvGVHRmw=
github.com/olekukonko/cat v0.0.0-20250911104152-50322a0618f6 h1:zrbMGy9YXpIeTnGj4EljqMiZsIcE09mmF8XsD5AYOJc= github.com/olekukonko/cat v0.0.0-20250911104152-50322a0618f6 h1:zrbMGy9YXpIeTnGj4EljqMiZsIcE09mmF8XsD5AYOJc=
github.com/olekukonko/cat v0.0.0-20250911104152-50322a0618f6/go.mod h1:rEKTHC9roVVicUIfZK7DYrdIoM0EOr8mK1Hj5s3JjH0= github.com/olekukonko/cat v0.0.0-20250911104152-50322a0618f6/go.mod h1:rEKTHC9roVVicUIfZK7DYrdIoM0EOr8mK1Hj5s3JjH0=
github.com/olekukonko/errors v1.2.0 h1:10Zcn4GeV59t/EGqJc8fUjtFT/FuUh5bTMzZ1XwmCRo= github.com/olekukonko/errors v1.2.0 h1:10Zcn4GeV59t/EGqJc8fUjtFT/FuUh5bTMzZ1XwmCRo=
github.com/olekukonko/errors v1.2.0/go.mod h1:ppzxA5jBKcO1vIpCXQ9ZqgDh8iwODz6OXIGKU8r5m4Y= github.com/olekukonko/errors v1.2.0/go.mod h1:ppzxA5jBKcO1vIpCXQ9ZqgDh8iwODz6OXIGKU8r5m4Y=
github.com/olekukonko/ll v0.1.6 h1:lGVTHO+Qc4Qm+fce/2h2m5y9LvqaW+DCN7xW9hsU3uA= github.com/olekukonko/ll v0.1.6 h1:lGVTHO+Qc4Qm+fce/2h2m5y9LvqaW+DCN7xW9hsU3uA=
github.com/olekukonko/ll v0.1.6/go.mod h1:NVUmjBb/aCtUpjKk75BhWrOlARz3dqsM+OtszpY4o88= github.com/olekukonko/ll v0.1.6/go.mod h1:NVUmjBb/aCtUpjKk75BhWrOlARz3dqsM+OtszpY4o88=
github.com/olekukonko/tablewriter v1.1.4 h1:ORUMI3dXbMnRlRggJX3+q7OzQFDdvgbN9nVWj1drm6I= github.com/olekukonko/tablewriter v1.1.5 h1:4LoZSfMySpMQY3PT8RWJsJeuEuMIoo9xGRgvmqjg6IQ=
github.com/olekukonko/tablewriter v1.1.4/go.mod h1:+kedxuyTtgoZLwif3P1Em4hARJs+mVnzKxmsCL/C5RY= github.com/olekukonko/tablewriter v1.1.5/go.mod h1:+kedxuyTtgoZLwif3P1Em4hARJs+mVnzKxmsCL/C5RY=
github.com/onsi/gomega v1.34.1 h1:EUMJIKUjM8sKjYbtxQI9A4z2o+rruxnzNvpknOXie6k= github.com/onsi/gomega v1.34.1 h1:EUMJIKUjM8sKjYbtxQI9A4z2o+rruxnzNvpknOXie6k=
github.com/onsi/gomega v1.34.1/go.mod h1:kU1QgUvBDLXBJq618Xvm2LUX6rSAfRaFRTcdOeDLwwY= github.com/onsi/gomega v1.34.1/go.mod h1:kU1QgUvBDLXBJq618Xvm2LUX6rSAfRaFRTcdOeDLwwY=
github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U=
@@ -976,8 +978,8 @@ github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9de
github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
github.com/zclconf/go-cty v1.16.3 h1:osr++gw2T61A8KVYHoQiFbFd1Lh3JOCXc/jFLJXKTxk= github.com/zclconf/go-cty v1.19.0 h1:IV8WdqYZc2c5rLX9bEoLNXKojBAp0MZPBHMIrCoa/s4=
github.com/zclconf/go-cty v1.16.3/go.mod h1:VvMs5i0vgZdhYawQNq5kePSpLAoz8u1xvZgrPIxfnZE= github.com/zclconf/go-cty v1.19.0/go.mod h1:12W89jGn3JCOIQi7infWr9m80rOkb5RNYJqXMZcN4c8=
github.com/zclconf/go-cty-debug v0.0.0-20240509010212-0d6042c53940 h1:4r45xpDWB6ZMSMNJFMOjqrGHynW3DIBuR2H9j0ug+Mo= github.com/zclconf/go-cty-debug v0.0.0-20240509010212-0d6042c53940 h1:4r45xpDWB6ZMSMNJFMOjqrGHynW3DIBuR2H9j0ug+Mo=
github.com/zclconf/go-cty-debug v0.0.0-20240509010212-0d6042c53940/go.mod h1:CmBdvvj3nqzfzJ6nTCIwDTPZ56aVGvDrmztiO5g3qrM= github.com/zclconf/go-cty-debug v0.0.0-20240509010212-0d6042c53940/go.mod h1:CmBdvvj3nqzfzJ6nTCIwDTPZ56aVGvDrmztiO5g3qrM=
github.com/zyedidia/generic v1.2.2-0.20230320175451-4410d2372cb1 h1:V+UsotZpAVvfj3X/LMoEytoLzSiP6Lg0F7wdVyu9gGg= github.com/zyedidia/generic v1.2.2-0.20230320175451-4410d2372cb1 h1:V+UsotZpAVvfj3X/LMoEytoLzSiP6Lg0F7wdVyu9gGg=
@@ -1526,8 +1528,8 @@ modernc.org/gc/v3 v3.1.5 h1:21ldfPfRYE31Tb7B3mwAK8gy1AxP4+dKjrOQPfqakoc=
modernc.org/gc/v3 v3.1.5/go.mod h1:HFK/6AGESC7Ex+EZJhJ2Gni6cTaYpSMmU/cT9RmlfYY= modernc.org/gc/v3 v3.1.5/go.mod h1:HFK/6AGESC7Ex+EZJhJ2Gni6cTaYpSMmU/cT9RmlfYY=
modernc.org/goabi0 v0.2.0 h1:HvEowk7LxcPd0eq6mVOAEMai46V+i7Jrj13t4AzuNks= modernc.org/goabi0 v0.2.0 h1:HvEowk7LxcPd0eq6mVOAEMai46V+i7Jrj13t4AzuNks=
modernc.org/goabi0 v0.2.0/go.mod h1:CEFRnnJhKvWT1c1JTI3Avm+tgOWbkOu5oPA8eH8LnMI= modernc.org/goabi0 v0.2.0/go.mod h1:CEFRnnJhKvWT1c1JTI3Avm+tgOWbkOu5oPA8eH8LnMI=
modernc.org/libc v1.75.6 h1:yKk8qo+Di4gkmvRboK8ocCqH22FiUCR6jRy2OwtCRus= modernc.org/libc v1.75.7 h1:o3DTP9/0p9pKmY2WCKQaySW6wIiZhNM7wc2lUoyhfew=
modernc.org/libc v1.75.6/go.mod h1:bO5o2ztHxBb2rjz0PgdHN0sSMw57CgxGFLZ3Qd/QpVQ= modernc.org/libc v1.75.7/go.mod h1:bO5o2ztHxBb2rjz0PgdHN0sSMw57CgxGFLZ3Qd/QpVQ=
modernc.org/mathutil v1.7.1 h1:GCZVGXdaN8gTqB1Mf/usp1Y/hSqgI2vAGGP4jZMCxOU= modernc.org/mathutil v1.7.1 h1:GCZVGXdaN8gTqB1Mf/usp1Y/hSqgI2vAGGP4jZMCxOU=
modernc.org/mathutil v1.7.1/go.mod h1:4p5IwJITfppl0G4sUEDtCr4DthTaT47/N3aT6MhfgJg= modernc.org/mathutil v1.7.1/go.mod h1:4p5IwJITfppl0G4sUEDtCr4DthTaT47/N3aT6MhfgJg=
modernc.org/memory v1.12.1 h1:nFMiWrpStgZczNl6XI9GnIk/rWhYIyHGUaR04pGbp9g= modernc.org/memory v1.12.1 h1:nFMiWrpStgZczNl6XI9GnIk/rWhYIyHGUaR04pGbp9g=
@@ -1536,8 +1538,8 @@ modernc.org/opt v0.2.0 h1:tGyef5ApycA7FSEOMraay9SaTk5zmbx7Tu+cJs4QKZg=
modernc.org/opt v0.2.0/go.mod h1:03fq9lsNfvkYSfxrfUhZCWPk1lm4cq4N+Bh//bEtgns= modernc.org/opt v0.2.0/go.mod h1:03fq9lsNfvkYSfxrfUhZCWPk1lm4cq4N+Bh//bEtgns=
modernc.org/sortutil v1.2.1 h1:+xyoGf15mM3NMlPDnFqrteY07klSFxLElE2PVuWIJ7w= modernc.org/sortutil v1.2.1 h1:+xyoGf15mM3NMlPDnFqrteY07klSFxLElE2PVuWIJ7w=
modernc.org/sortutil v1.2.1/go.mod h1:7ZI3a3REbai7gzCLcotuw9AC4VZVpYMjDzETGsSMqJE= modernc.org/sortutil v1.2.1/go.mod h1:7ZI3a3REbai7gzCLcotuw9AC4VZVpYMjDzETGsSMqJE=
modernc.org/sqlite v1.58.0 h1:38u40/bwkfM7f0Myhosl+SEMltSDxnGdQf8o6Kjmys0= modernc.org/sqlite v1.59.0 h1:X1es1GpqBlS/5T+vbM4HLUdaa8OtQx468DF2vrx+38A=
modernc.org/sqlite v1.58.0/go.mod h1:rsD2CckafgObKC4DhBlGBf+RiHxkc3hINGt1Xw32tVY= modernc.org/sqlite v1.59.0/go.mod h1:+paeT2A3iPRHkQDwG7oA6Tk0zQd5woMEI8q7orfry8k=
modernc.org/strutil v1.2.1 h1:UneZBkQA+DX2Rp35KcM69cSsNES9ly8mQWD71HKlOA0= modernc.org/strutil v1.2.1 h1:UneZBkQA+DX2Rp35KcM69cSsNES9ly8mQWD71HKlOA0=
modernc.org/strutil v1.2.1/go.mod h1:EHkiggD70koQxjVdSBM3JKM7k6L0FbGE5eymy9i3B9A= modernc.org/strutil v1.2.1/go.mod h1:EHkiggD70koQxjVdSBM3JKM7k6L0FbGE5eymy9i3B9A=
modernc.org/token v1.1.0 h1:Xl7Ap9dKaEs5kLoOQeQmPWevfnk/DM5qcLcYlA8ys6Y= modernc.org/token v1.1.0 h1:Xl7Ap9dKaEs5kLoOQeQmPWevfnk/DM5qcLcYlA8ys6Y=
internal/capabilities/capabilities.go
+1 -1
View File
@@ -9,7 +9,7 @@ import (
"sort" "sort"
"github.com/scylladb/go-set/strset" "github.com/scylladb/go-set/strset"
"gopkg.in/yaml.v3" "go.yaml.in/yaml/v3"
"github.com/anchore/syft/internal/task" "github.com/anchore/syft/internal/task"
) )
internal/capabilities/generate/io.go
+2 -2
View File
@@ -1,4 +1,4 @@
// this file handles YAML file reading and writing with comment preservation, using gopkg.in/yaml.v3's node tree to maintain all existing comments during regeneration. // this file handles YAML file reading and writing with comment preservation, using go.yaml.in/yaml/v3's node tree to maintain all existing comments during regeneration.
package main package main
import ( import (
@@ -6,7 +6,7 @@ import (
"os" "os"
"regexp" "regexp"
"gopkg.in/yaml.v3" "go.yaml.in/yaml/v3"
"github.com/anchore/syft/internal/capabilities" "github.com/anchore/syft/internal/capabilities"
"github.com/anchore/syft/internal/capabilities/internal" "github.com/anchore/syft/internal/capabilities/internal"
internal/capabilities/generate/io_test.go
+1 -1
View File
@@ -4,7 +4,7 @@ import (
"testing" "testing"
"github.com/stretchr/testify/require" "github.com/stretchr/testify/require"
"gopkg.in/yaml.v3" "go.yaml.in/yaml/v3"
) )
func TestFindFieldValue(t *testing.T) { func TestFindFieldValue(t *testing.T) {
internal/capabilities/internal/load_capabilities.go
+1 -1
View File
@@ -5,7 +5,7 @@ import (
"os" "os"
"path/filepath" "path/filepath"
"gopkg.in/yaml.v3" "go.yaml.in/yaml/v3"
"github.com/anchore/syft/internal/capabilities" "github.com/anchore/syft/internal/capabilities"
) )
internal/capabilities/model_test.go
+1 -1
View File
@@ -5,7 +5,7 @@ import (
"github.com/google/go-cmp/cmp" "github.com/google/go-cmp/cmp"
"github.com/stretchr/testify/require" "github.com/stretchr/testify/require"
"gopkg.in/yaml.v3" "go.yaml.in/yaml/v3"
) )
func TestConfigSchemaExtensions(t *testing.T) { func TestConfigSchemaExtensions(t *testing.T) {
internal/line_scanner.go
+19
View File
@@ -0,0 +1,19 @@
package internal
import (
"bufio"
"io"
)
// maxScannedLineSize is the longest line NewLineScanner will return before Scan stops with bufio.ErrTooLong.
// The buffer only grows to this size when a line needs it.
const maxScannedLineSize = 1024 * 1024
// NewLineScanner returns a bufio.Scanner that accepts lines up to 1MB rather than the 64KB default, since
// package metadata files carry long single-line fields (descriptions, license text, dependency lists). Callers
// must still check Err() after scanning.
func NewLineScanner(r io.Reader) *bufio.Scanner {
s := bufio.NewScanner(r)
s.Buffer(nil, maxScannedLineSize)
return s
}
internal/line_scanner_test.go
+24
View File
@@ -0,0 +1,24 @@
package internal
import (
"bufio"
"strings"
"testing"
"github.com/stretchr/testify/require"
)
func TestNewLineScanner(t *testing.T) {
long := strings.Repeat("a", 100*1024)
s := NewLineScanner(strings.NewReader(long + "\nb\n"))
require.True(t, s.Scan())
require.Equal(t, long, s.Text())
require.True(t, s.Scan())
require.Equal(t, "b", s.Text())
require.False(t, s.Scan())
require.NoError(t, s.Err())
s = NewLineScanner(strings.NewReader(strings.Repeat("a", maxScannedLineSize+1)))
require.False(t, s.Scan())
require.ErrorIs(t, s.Err(), bufio.ErrTooLong)
}
internal/os/feature_detection.go
+4 -6
View File
@@ -6,6 +6,7 @@ import (
"errors" "errors"
"fmt" "fmt"
"regexp" "regexp"
"slices"
"github.com/anchore/syft/internal" "github.com/anchore/syft/internal"
"github.com/anchore/syft/internal/log" "github.com/anchore/syft/internal/log"
@@ -191,12 +192,9 @@ func hasInstalledESMPackage(packages *pkg.Collection) bool {
if packages == nil { if packages == nil {
return false return false
} }
for p := range packages.Enumerate(pkg.DebPkg) { return slices.ContainsFunc(packages.Sorted(pkg.DebPkg), func(p pkg.Package) bool {
if esmVersionPattern.MatchString(p.Version) { return esmVersionPattern.MatchString(p.Version)
return true })
}
}
return false
} }
func fileMatchesAny(resolver file.Resolver, location file.Location, patterns ...*regexp.Regexp) (bool, error) { func fileMatchesAny(resolver file.Resolver, location file.Location, patterns ...*regexp.Regexp) (bool, error) {
internal/os/feature_detection_test.go
+28
View File
@@ -8,6 +8,7 @@ import (
"path/filepath" "path/filepath"
"slices" "slices"
"testing" "testing"
"time"
"github.com/stretchr/testify/require" "github.com/stretchr/testify/require"
@@ -232,6 +233,33 @@ func Test_findUbuntuFeatures(t *testing.T) {
} }
} }
// returning early from Collection.Enumerate leaks its goroutine and read lock, so any later write blocks forever.
func Test_findUbuntuFeatures_releasesPackageCollectionLock(t *testing.T) {
resolver := fixtureResolverForDir(t, "testdata/ubuntu_plain")
s := sbom.SBOM{}
s.Artifacts.LinuxDistribution = &linux.Release{ID: "ubuntu"}
s.Artifacts.Packages = pkg.NewCollection(
pkg.Package{Name: "openssl", Version: "1.1.1f-1ubuntu2.19+esm1", Type: pkg.DebPkg},
pkg.Package{Name: "libcap2", Version: "1:2.32-1ubuntu0.1~esm1", Type: pkg.DebPkg},
)
require.NoError(t, os.DetectFeatures(context.Background(), resolver, sbomsync.NewBuilder(&s)))
require.True(t, s.Artifacts.LinuxDistribution.ExtendedSupport)
added := make(chan struct{})
go func() {
s.Artifacts.Packages.Add(pkg.Package{Name: "zlib1g", Version: "1:1.2.11.dfsg-2ubuntu1", Type: pkg.DebPkg})
close(added)
}()
select {
case <-added:
case <-time.After(5 * time.Second):
t.Fatal("package collection is still read-locked after feature detection")
}
}
// a read error in the apt-source signal must not suppress the ubuntu-advantage status signal; the three // a read error in the apt-source signal must not suppress the ubuntu-advantage status signal; the three
// signals are meant to be independent fallbacks. // signals are meant to be independent fallbacks.
func Test_findUbuntuFeatures_signalErrorDoesNotSuppressOthers(t *testing.T) { func Test_findUbuntuFeatures_signalErrorDoesNotSuppressOthers(t *testing.T) {
internal/packagemetadata/untyped_fields_test.go
+74
View File
@@ -0,0 +1,74 @@
package packagemetadata
import (
"reflect"
"strings"
"testing"
)
// untypedFieldAllowlist holds package metadata fields that are allowed to carry untyped (interface) values.
// Values in these fields come straight from whatever a parser decoded, so they are not shaped by syft
// and can hold anything an encoder or hasher chokes on. Every entry is part of existing SBOM output and
// cannot be retyped without breaking it; new entries need a reason and a review.
var untypedFieldAllowlist = map[string]string{
"GGUFFileHeader.RemainingKeyValues": "raw GGUF header key-values; retyping changes the header output and metadataHash",
}
// TestNoUntypedMetadataFields fails when a package metadata type gains a field typed any, []any,
// map[string]any (or any other interface) that is not on the allowlist.
func TestNoUntypedMetadataFields(t *testing.T) {
var hits []string
for _, ty := range AllTypes() {
rt := reflect.TypeOf(ty)
collectUntyped(rt, rt.Name(), map[reflect.Type]bool{}, &hits)
}
seen := map[string]bool{}
for _, h := range hits {
seen[h] = true
if _, ok := untypedFieldAllowlist[h]; !ok {
t.Errorf("untyped field %s: use a concrete type, or add it to untypedFieldAllowlist with a reason", h)
}
}
for k := range untypedFieldAllowlist {
if !seen[k] {
t.Errorf("stale untypedFieldAllowlist entry %s: field no longer exists or is now typed", k)
}
}
}
// collectUntyped records "Struct.Field" for every field whose type reaches an interface, descending
// through pointers, slices, arrays, maps and nested syft-owned structs.
func collectUntyped(rt reflect.Type, path string, visited map[reflect.Type]bool, hits *[]string) {
switch rt.Kind() {
case reflect.Interface:
*hits = append(*hits, path)
case reflect.Pointer, reflect.Slice, reflect.Array:
collectUntyped(rt.Elem(), path, visited, hits)
case reflect.Map:
collectUntyped(rt.Key(), path, visited, hits)
collectUntyped(rt.Elem(), path, visited, hits)
case reflect.Struct:
if visited[rt] {
return
}
if rt.Name() != "" && !strings.HasPrefix(rt.PkgPath(), "github.com/anchore/syft/") {
// a third-party type in the model carries whatever shape the library gives it (the same leak as an
// interface field), so it counts as untyped. The standard library is left alone.
if strings.Contains(strings.Split(rt.PkgPath(), "/")[0], ".") {
*hits = append(*hits, path)
}
return
}
visited[rt] = true
for i := 0; i < rt.NumField(); i++ {
f := rt.Field(i)
// unexported embedded structs still promote their exported fields to the encoders and the hasher. A
// field is only out of reach when it is neither encoded nor hashed by SetID.
if (!f.IsExported() && !f.Anonymous) || (f.Tag.Get("json") == "-" && f.Tag.Get("hash") == "ignore") {
continue
}
collectUntyped(f.Type, rt.Name()+"."+f.Name, visited, hits)
}
}
}
internal/safetensors.go
+7
View File
@@ -0,0 +1,7 @@
package internal
// MaxSafeTensorsHeaderSize caps a safetensors header's JSON (excluding its
// 8-byte length prefix). Shared by the ai cataloger and the OCI model source so
// dir and OCI scans accept the same headers. Real-world shard headers are well
// under 1 MB.
const MaxSafeTensorsHeaderSize = 8 * 1024 * 1024
internal/spdxlicense/license.go
+43
View File
@@ -44,6 +44,7 @@ type LicenseInfo struct {
// LicenseByURL returns the license ID and name for a given URL from the SPDX license list. // LicenseByURL returns the license ID and name for a given URL from the SPDX license list.
// The URL should match one of the URLs in the seeAlso field of an SPDX license. // The URL should match one of the URLs in the seeAlso field of an SPDX license.
// The scheme (http:// or https://) is stripped before lookup, so both schemes match. // The scheme (http:// or https://) is stripped before lookup, so both schemes match.
// opensource.org URLs additionally match regardless of the form used (see normalizeOpenSourceOrgURL).
func LicenseByURL(url string) (LicenseInfo, bool) { func LicenseByURL(url string) (LicenseInfo, bool) {
url = strings.TrimSpace(url) url = strings.TrimSpace(url)
url = stripScheme(url) url = stripScheme(url)
@@ -52,9 +53,51 @@ func LicenseByURL(url string) (LicenseInfo, bool) {
ID: id, ID: id,
}, true }, true
} }
if normalized, ok := normalizeOpenSourceOrgURL(url); ok {
if id, exists := openSourceOrgURLToLicense[normalized]; exists {
return LicenseInfo{
ID: id,
}, true
}
}
return LicenseInfo{}, false return LicenseInfo{}, false
} }
// openSourceOrgURLToLicense indexes the opensource.org entries of urlToLicense by normalized URL.
var openSourceOrgURLToLicense = buildOpenSourceOrgURLToLicense(urlToLicense)
func buildOpenSourceOrgURLToLicense(urls map[string]string) map[string]string {
index := make(map[string]string)
ambiguous := make(map[string]bool)
for url, id := range urls {
normalized, ok := normalizeOpenSourceOrgURL(url)
if !ok || ambiguous[normalized] {
continue
}
if existing, exists := index[normalized]; exists && existing != id {
// don't guess between licenses that only differ by URL form
delete(index, normalized)
ambiguous[normalized] = true
continue
}
index[normalized] = id
}
return index
}
// normalizeOpenSourceOrgURL reduces a scheme-less opensource.org URL to a canonical form.
// SPDX has changed these URLs over time (e.g. 3.29.0 replaced opensource.org/licenses/MIT and
// opensource.org/license/mit/ with opensource.org/license/MIT), while package metadata in the wild
// still uses the older forms. The www. prefix, /licenses/ vs /license/, case, and trailing slash are ignored.
func normalizeOpenSourceOrgURL(url string) (string, bool) {
url = strings.TrimPrefix(strings.ToLower(url), "www.")
if !strings.HasPrefix(url, "opensource.org/") {
return "", false
}
url = strings.Replace(url, "opensource.org/licenses/", "opensource.org/license/", 1)
return strings.TrimSuffix(url, "/"), true
}
// stripScheme removes http:// or https:// prefix from a URL. // stripScheme removes http:// or https:// prefix from a URL.
// This allows a single map entry to match both schemes. // This allows a single map entry to match both schemes.
func stripScheme(url string) string { func stripScheme(url string) string {
internal/spdxlicense/license_list.go
+254 -199
View File
@@ -2,7 +2,7 @@
// This file was generated using data from https://spdx.org/licenses/licenses.json // This file was generated using data from https://spdx.org/licenses/licenses.json
package spdxlicense package spdxlicense
const Version = "3.28.0" const Version = "3.29.0"
var licenseIDs = map[string]string{ var licenseIDs = map[string]string{
"0bsd": "0BSD", "0bsd": "0BSD",
@@ -116,6 +116,7 @@ var licenseIDs = map[string]string{
"aswfdigitalassets1.0.0": "ASWF-Digital-Assets-1.0", "aswfdigitalassets1.0.0": "ASWF-Digital-Assets-1.0",
"aswfdigitalassets1.1": "ASWF-Digital-Assets-1.1", "aswfdigitalassets1.1": "ASWF-Digital-Assets-1.1",
"aswfdigitalassets1.1.0": "ASWF-Digital-Assets-1.1", "aswfdigitalassets1.1.0": "ASWF-Digital-Assets-1.1",
"atcgame": "atc-game",
"baekmuk": "Baekmuk", "baekmuk": "Baekmuk",
"bahyph": "Bahyph", "bahyph": "Bahyph",
"barr": "Barr", "barr": "Barr",
@@ -142,8 +143,11 @@ var licenseIDs = map[string]string{
"briangladman2.0clause": "Brian-Gladman-2-Clause", "briangladman2.0clause": "Brian-Gladman-2-Clause",
"briangladman2clause": "Brian-Gladman-2-Clause", "briangladman2clause": "Brian-Gladman-2-Clause",
"briangladman3.0.0clause": "Brian-Gladman-3-Clause", "briangladman3.0.0clause": "Brian-Gladman-3-Clause",
"briangladman3.0.0clausenoconversion": "Brian-Gladman-3-Clause-no-conversion",
"briangladman3.0clause": "Brian-Gladman-3-Clause", "briangladman3.0clause": "Brian-Gladman-3-Clause",
"briangladman3.0clausenoconversion": "Brian-Gladman-3-Clause-no-conversion",
"briangladman3clause": "Brian-Gladman-3-Clause", "briangladman3clause": "Brian-Gladman-3-Clause",
"briangladman3clausenoconversion": "Brian-Gladman-3-Clause-no-conversion",
"bsd1.0.0clause": "BSD-1-Clause", "bsd1.0.0clause": "BSD-1-Clause",
"bsd1.0clause": "BSD-1-Clause", "bsd1.0clause": "BSD-1-Clause",
"bsd1clause": "BSD-1-Clause", "bsd1clause": "BSD-1-Clause",
@@ -154,6 +158,7 @@ var licenseIDs = map[string]string{
"bsd2.0.0clausenetbsd": "BSD-2-Clause", "bsd2.0.0clausenetbsd": "BSD-2-Clause",
"bsd2.0.0clausepatent": "BSD-2-Clause-Patent", "bsd2.0.0clausepatent": "BSD-2-Clause-Patent",
"bsd2.0.0clausepkgconfdisclaimer": "BSD-2-Clause-pkgconf-disclaimer", "bsd2.0.0clausepkgconfdisclaimer": "BSD-2-Clause-pkgconf-disclaimer",
"bsd2.0.0clauseposunchanged": "BSD-2-Clause-pos-unchanged",
"bsd2.0.0clauseviews": "BSD-2-Clause-Views", "bsd2.0.0clauseviews": "BSD-2-Clause-Views",
"bsd2.0clause": "BSD-2-Clause", "bsd2.0clause": "BSD-2-Clause",
"bsd2.0clausedarwin": "BSD-2-Clause-Darwin", "bsd2.0clausedarwin": "BSD-2-Clause-Darwin",
@@ -162,6 +167,7 @@ var licenseIDs = map[string]string{
"bsd2.0clausenetbsd": "BSD-2-Clause", "bsd2.0clausenetbsd": "BSD-2-Clause",
"bsd2.0clausepatent": "BSD-2-Clause-Patent", "bsd2.0clausepatent": "BSD-2-Clause-Patent",
"bsd2.0clausepkgconfdisclaimer": "BSD-2-Clause-pkgconf-disclaimer", "bsd2.0clausepkgconfdisclaimer": "BSD-2-Clause-pkgconf-disclaimer",
"bsd2.0clauseposunchanged": "BSD-2-Clause-pos-unchanged",
"bsd2.0clauseviews": "BSD-2-Clause-Views", "bsd2.0clauseviews": "BSD-2-Clause-Views",
"bsd2clause": "BSD-2-Clause", "bsd2clause": "BSD-2-Clause",
"bsd2clausedarwin": "BSD-2-Clause-Darwin", "bsd2clausedarwin": "BSD-2-Clause-Darwin",
@@ -170,6 +176,7 @@ var licenseIDs = map[string]string{
"bsd2clausenetbsd": "BSD-2-Clause", "bsd2clausenetbsd": "BSD-2-Clause",
"bsd2clausepatent": "BSD-2-Clause-Patent", "bsd2clausepatent": "BSD-2-Clause-Patent",
"bsd2clausepkgconfdisclaimer": "BSD-2-Clause-pkgconf-disclaimer", "bsd2clausepkgconfdisclaimer": "BSD-2-Clause-pkgconf-disclaimer",
"bsd2clauseposunchanged": "BSD-2-Clause-pos-unchanged",
"bsd2clauseviews": "BSD-2-Clause-Views", "bsd2clauseviews": "BSD-2-Clause-Views",
"bsd3.0.0clause": "BSD-3-Clause", "bsd3.0.0clause": "BSD-3-Clause",
"bsd3.0.0clauseacpica": "BSD-3-Clause-acpica", "bsd3.0.0clauseacpica": "BSD-3-Clause-acpica",
@@ -184,6 +191,7 @@ var licenseIDs = map[string]string{
"bsd3.0.0clausenonuclearlicense2014": "BSD-3-Clause-No-Nuclear-License-2014", "bsd3.0.0clausenonuclearlicense2014": "BSD-3-Clause-No-Nuclear-License-2014",
"bsd3.0.0clausenonuclearwarranty": "BSD-3-Clause-No-Nuclear-Warranty", "bsd3.0.0clausenonuclearwarranty": "BSD-3-Clause-No-Nuclear-Warranty",
"bsd3.0.0clauseopenmpi": "BSD-3-Clause-Open-MPI", "bsd3.0.0clauseopenmpi": "BSD-3-Clause-Open-MPI",
"bsd3.0.0clauseopenwebui": "BSD-3-Clause-OpenWebUI",
"bsd3.0.0clausesun": "BSD-3-Clause-Sun", "bsd3.0.0clausesun": "BSD-3-Clause-Sun",
"bsd3.0.0clausetso": "BSD-3-Clause-Tso", "bsd3.0.0clausetso": "BSD-3-Clause-Tso",
"bsd3.0clause": "BSD-3-Clause", "bsd3.0clause": "BSD-3-Clause",
@@ -199,6 +207,7 @@ var licenseIDs = map[string]string{
"bsd3.0clausenonuclearlicense2014": "BSD-3-Clause-No-Nuclear-License-2014", "bsd3.0clausenonuclearlicense2014": "BSD-3-Clause-No-Nuclear-License-2014",
"bsd3.0clausenonuclearwarranty": "BSD-3-Clause-No-Nuclear-Warranty", "bsd3.0clausenonuclearwarranty": "BSD-3-Clause-No-Nuclear-Warranty",
"bsd3.0clauseopenmpi": "BSD-3-Clause-Open-MPI", "bsd3.0clauseopenmpi": "BSD-3-Clause-Open-MPI",
"bsd3.0clauseopenwebui": "BSD-3-Clause-OpenWebUI",
"bsd3.0clausesun": "BSD-3-Clause-Sun", "bsd3.0clausesun": "BSD-3-Clause-Sun",
"bsd3.0clausetso": "BSD-3-Clause-Tso", "bsd3.0clausetso": "BSD-3-Clause-Tso",
"bsd3clause": "BSD-3-Clause", "bsd3clause": "BSD-3-Clause",
@@ -214,6 +223,7 @@ var licenseIDs = map[string]string{
"bsd3clausenonuclearlicense2014": "BSD-3-Clause-No-Nuclear-License-2014", "bsd3clausenonuclearlicense2014": "BSD-3-Clause-No-Nuclear-License-2014",
"bsd3clausenonuclearwarranty": "BSD-3-Clause-No-Nuclear-Warranty", "bsd3clausenonuclearwarranty": "BSD-3-Clause-No-Nuclear-Warranty",
"bsd3clauseopenmpi": "BSD-3-Clause-Open-MPI", "bsd3clauseopenmpi": "BSD-3-Clause-Open-MPI",
"bsd3clauseopenwebui": "BSD-3-Clause-OpenWebUI",
"bsd3clausesun": "BSD-3-Clause-Sun", "bsd3clausesun": "BSD-3-Clause-Sun",
"bsd3clausetso": "BSD-3-Clause-Tso", "bsd3clausetso": "BSD-3-Clause-Tso",
"bsd4.0.0clause": "BSD-4-Clause", "bsd4.0.0clause": "BSD-4-Clause",
@@ -232,12 +242,15 @@ var licenseIDs = map[string]string{
"bsd4reno": "BSD-4.3RENO", "bsd4reno": "BSD-4.3RENO",
"bsd4tahoe": "BSD-4.3TAHOE", "bsd4tahoe": "BSD-4.3TAHOE",
"bsdadvertisingacknowledgement": "BSD-Advertising-Acknowledgement", "bsdadvertisingacknowledgement": "BSD-Advertising-Acknowledgement",
"bsdasktoendorse": "BSD-ask-to-endorse",
"bsdattributionhpnddisclaimer": "BSD-Attribution-HPND-disclaimer", "bsdattributionhpnddisclaimer": "BSD-Attribution-HPND-disclaimer",
"bsdinfernonettverk": "BSD-Inferno-Nettverk", "bsdinfernonettverk": "BSD-Inferno-Nettverk",
"bsdmarkmodifications": "BSD-Mark-Modifications", "bsdmarkmodifications": "BSD-Mark-Modifications",
"bsdprotection": "BSD-Protection", "bsdprotection": "BSD-Protection",
"bsdsourcealtgpl": "BSD-Source-alt-GPL",
"bsdsourcebeginningfile": "BSD-Source-beginning-file", "bsdsourcebeginningfile": "BSD-Source-beginning-file",
"bsdsourcecode": "BSD-Source-Code", "bsdsourcecode": "BSD-Source-Code",
"bsdsourcecodenodisclaimer": "BSD-Source-Code-no-disclaimer",
"bsdsystemics": "BSD-Systemics", "bsdsystemics": "BSD-Systemics",
"bsdsystemicsw3.0.0works": "BSD-Systemics-W3Works", "bsdsystemicsw3.0.0works": "BSD-Systemics-W3Works",
"bsdsystemicsw3.0works": "BSD-Systemics-W3Works", "bsdsystemicsw3.0works": "BSD-Systemics-W3Works",
@@ -246,6 +259,7 @@ var licenseIDs = map[string]string{
"bsl1.0": "BSL-1.0", "bsl1.0": "BSL-1.0",
"bsl1.0.0": "BSL-1.0", "bsl1.0.0": "BSL-1.0",
"buddy": "Buddy", "buddy": "Buddy",
"bugroff": "Bugroff",
"busl1": "BUSL-1.1", "busl1": "BUSL-1.1",
"busl1.1": "BUSL-1.1", "busl1.1": "BUSL-1.1",
"busl1.1.0": "BUSL-1.1", "busl1.1.0": "BUSL-1.1",
@@ -316,8 +330,11 @@ var licenseIDs = map[string]string{
"ccbync3.0": "CC-BY-NC-3.0", "ccbync3.0": "CC-BY-NC-3.0",
"ccbync3.0.0": "CC-BY-NC-3.0", "ccbync3.0.0": "CC-BY-NC-3.0",
"ccbync3.0.0de": "CC-BY-NC-3.0-DE", "ccbync3.0.0de": "CC-BY-NC-3.0-DE",
"ccbync3.0.0igo": "CC-BY-NC-3.0-IGO",
"ccbync3.0de": "CC-BY-NC-3.0-DE", "ccbync3.0de": "CC-BY-NC-3.0-DE",
"ccbync3.0igo": "CC-BY-NC-3.0-IGO",
"ccbync3de": "CC-BY-NC-3.0-DE", "ccbync3de": "CC-BY-NC-3.0-DE",
"ccbync3igo": "CC-BY-NC-3.0-IGO",
"ccbync4": "CC-BY-NC-4.0", "ccbync4": "CC-BY-NC-4.0",
"ccbync4.0": "CC-BY-NC-4.0", "ccbync4.0": "CC-BY-NC-4.0",
"ccbync4.0.0": "CC-BY-NC-4.0", "ccbync4.0.0": "CC-BY-NC-4.0",
@@ -592,6 +609,7 @@ var licenseIDs = map[string]string{
"fair": "Fair", "fair": "Fair",
"fbm": "FBM", "fbm": "FBM",
"fdkaac": "FDK-AAC", "fdkaac": "FDK-AAC",
"fdkmpegh": "FDK-MPEG-H",
"fergusontwofish": "Ferguson-Twofish", "fergusontwofish": "Ferguson-Twofish",
"frameworx1": "Frameworx-1.0", "frameworx1": "Frameworx-1.0",
"frameworx1.0": "Frameworx-1.0", "frameworx1.0": "Frameworx-1.0",
@@ -767,6 +785,9 @@ var licenseIDs = map[string]string{
"hippocratic2": "Hippocratic-2.1", "hippocratic2": "Hippocratic-2.1",
"hippocratic2.1": "Hippocratic-2.1", "hippocratic2.1": "Hippocratic-2.1",
"hippocratic2.1.0": "Hippocratic-2.1", "hippocratic2.1.0": "Hippocratic-2.1",
"hippocratic3.0.0core": "Hippocratic-3.0-core",
"hippocratic3.0core": "Hippocratic-3.0-core",
"hippocratic3core": "Hippocratic-3.0-core",
"hp1986": "HP-1986", "hp1986": "HP-1986",
"hp1986.0": "HP-1986", "hp1986.0": "HP-1986",
"hp1986.0.0": "HP-1986", "hp1986.0.0": "HP-1986",
@@ -813,6 +834,7 @@ var licenseIDs = map[string]string{
"imlib2": "Imlib2", "imlib2": "Imlib2",
"imlib2.0": "Imlib2", "imlib2.0": "Imlib2",
"imlib2.0.0": "Imlib2", "imlib2.0.0": "Imlib2",
"informatica": "Informatica",
"infozip": "Info-ZIP", "infozip": "Info-ZIP",
"innernet2": "Inner-Net-2.0", "innernet2": "Inner-Net-2.0",
"innernet2.0": "Inner-Net-2.0", "innernet2.0": "Inner-Net-2.0",
@@ -1011,6 +1033,9 @@ var licenseIDs = map[string]string{
"mulanpsl2.0.0": "MulanPSL-2.0", "mulanpsl2.0.0": "MulanPSL-2.0",
"multics": "Multics", "multics": "Multics",
"mup": "Mup", "mup": "Mup",
"mvt1": "MVT-1.1",
"mvt1.1": "MVT-1.1",
"mvt1.1.0": "MVT-1.1",
"naist2003": "NAIST-2003", "naist2003": "NAIST-2003",
"naist2003.0": "NAIST-2003", "naist2003.0": "NAIST-2003",
"naist2003.0.0": "NAIST-2003", "naist2003.0.0": "NAIST-2003",
@@ -1489,6 +1514,8 @@ var urlToLicense = map[string]string{
"cdla.io/permissive-1-0": "CDLA-Permissive-1.0", "cdla.io/permissive-1-0": "CDLA-Permissive-1.0",
"cdla.io/sharing-1-0": "CDLA-Sharing-1.0", "cdla.io/sharing-1-0": "CDLA-Sharing-1.0",
"celestrak.org/publications/AIAA/2006-6753/faq.php": "SGP4", "celestrak.org/publications/AIAA/2006-6753/faq.php": "SGP4",
"cgit.freebsd.org/src/plain/sys/dev/aic7xxx/aic7xxx_osm.c": "BSD-Source-alt-GPL",
"cgit.freebsd.org/src/tree/usr.bin/sockstat/sockstat.c": "BSD-2-Clause-pos-unchanged",
"cgit.freedesktop.org/libbsd/tree/man/setproctitle.3bsd": "libutil-David-Nugent", "cgit.freedesktop.org/libbsd/tree/man/setproctitle.3bsd": "libutil-David-Nugent",
"coil.apotheon.org/plaintext/01.0.txt": "COIL-1.0", "coil.apotheon.org/plaintext/01.0.txt": "COIL-1.0",
"core.tcl-lang.org/tk/file?name=compat/unistd.h": "HPND-UC", "core.tcl-lang.org/tk/file?name=compat/unistd.h": "HPND-UC",
@@ -1512,6 +1539,7 @@ var urlToLicense = map[string]string{
"creativecommons.org/licenses/by-nc/2.0/legalcode": "CC-BY-NC-2.0", "creativecommons.org/licenses/by-nc/2.0/legalcode": "CC-BY-NC-2.0",
"creativecommons.org/licenses/by-nc/2.5/legalcode": "CC-BY-NC-2.5", "creativecommons.org/licenses/by-nc/2.5/legalcode": "CC-BY-NC-2.5",
"creativecommons.org/licenses/by-nc/3.0/de/legalcode": "CC-BY-NC-3.0-DE", "creativecommons.org/licenses/by-nc/3.0/de/legalcode": "CC-BY-NC-3.0-DE",
"creativecommons.org/licenses/by-nc/3.0/igo/legalcode": "CC-BY-NC-3.0-IGO",
"creativecommons.org/licenses/by-nc/3.0/legalcode": "CC-BY-NC-3.0", "creativecommons.org/licenses/by-nc/3.0/legalcode": "CC-BY-NC-3.0",
"creativecommons.org/licenses/by-nc/4.0/legalcode": "CC-BY-NC-4.0", "creativecommons.org/licenses/by-nc/4.0/legalcode": "CC-BY-NC-4.0",
"creativecommons.org/licenses/by-nd-nc/1.0/legalcode": "CC-BY-NC-ND-1.0", "creativecommons.org/licenses/by-nd-nc/1.0/legalcode": "CC-BY-NC-ND-1.0",
@@ -1564,6 +1592,7 @@ var urlToLicense = map[string]string{
"directory.fsf.org/wiki/License:Fdk": "FDK-AAC", "directory.fsf.org/wiki/License:Fdk": "FDK-AAC",
"doc.qt.io/archives/3.3/license.html": "QPL-1.0", "doc.qt.io/archives/3.3/license.html": "QPL-1.0",
"doc.qt.nokia.com/3.3/license.html": "QPL-1.0", "doc.qt.nokia.com/3.3/license.html": "QPL-1.0",
"docs.mvt.re/en/latest/license/": "MVT-1.1",
"docs.omniverse.nvidia.com/install-guide/latest/common/licenses.html": "AML-glslang", "docs.omniverse.nvidia.com/install-guide/latest/common/licenses.html": "AML-glslang",
"docs.oracle.com/cd/E81115_01/html/E81116/licenses.html": "man2html", "docs.oracle.com/cd/E81115_01/html/E81116/licenses.html": "man2html",
"docs.python.org/3/license.html": "Python-2.0.1", "docs.python.org/3/license.html": "Python-2.0.1",
@@ -1669,6 +1698,7 @@ var urlToLicense = map[string]string{
"fedoraproject.org/wiki/Licensing:Mplus?rd=Licensing/mplus": "mplus", "fedoraproject.org/wiki/Licensing:Mplus?rd=Licensing/mplus": "mplus",
"fedoraproject.org/wiki/Licensing:ThorPublicLicense": "TPL-1.0", "fedoraproject.org/wiki/Licensing:ThorPublicLicense": "TPL-1.0",
"firstdonoharm.dev/version/2/1/license.html": "Hippocratic-2.1", "firstdonoharm.dev/version/2/1/license.html": "Hippocratic-2.1",
"firstdonoharm.dev/version/3/0/core.txt": "Hippocratic-3.0-core",
"forge.gouv.qc.ca/licence/fr/liliq-v1-1/": "LiLiQ-P-1.1", "forge.gouv.qc.ca/licence/fr/liliq-v1-1/": "LiLiQ-P-1.1",
"forge.gouv.qc.ca/licence/liliq-p": "LiLiQ-R-1.1", "forge.gouv.qc.ca/licence/liliq-p": "LiLiQ-R-1.1",
"forge.gouv.qc.ca/licence/liliq-p/": "LiLiQ-P-1.1", "forge.gouv.qc.ca/licence/liliq-p/": "LiLiQ-P-1.1",
@@ -1686,68 +1716,70 @@ var urlToLicense = map[string]string{
"ftp://ftp.tin.org/pub/news/utils/newsx/newsx-1.6.tar.gz": "Zeeff", "ftp://ftp.tin.org/pub/news/utils/newsx/newsx-1.6.tar.gz": "Zeeff",
"gianluca.dellavedova.org/2011/01/03/clarified-artistic-license/": "ClArtistic", "gianluca.dellavedova.org/2011/01/03/clarified-artistic-license/": "ClArtistic",
"git.denx.de/?p=u-boot.git;a=blob;f=arch/powerpc/cpu/ppc4xx/miiphy.c;h=297155fdafa064b955e53e9832de93bfb0cfb85b;hb=9fab4bf4cc077c21e43941866f3f2c196f28670d": "IBM-pibs", "git.denx.de/?p=u-boot.git;a=blob;f=arch/powerpc/cpu/ppc4xx/miiphy.c;h=297155fdafa064b955e53e9832de93bfb0cfb85b;hb=9fab4bf4cc077c21e43941866f3f2c196f28670d": "IBM-pibs",
"git.enlightenment.org/legacy/imlib2.git/tree/COPYING": "Imlib2", "git.enlightenment.org/legacy/imlib2.git/tree/COPYING": "Imlib2",
"git.kernel.org/pub/scm/docs/man-pages/man-pages.git/tree/man2/getcpu.2#n4": "Linux-man-pages-1-para", "git.kernel.org/pub/scm/docs/man-pages/man-pages.git/tree/man2/getcpu.2#n4": "Linux-man-pages-1-para",
"git.kernel.org/pub/scm/docs/man-pages/man-pages.git/tree/man2/migrate_pages.2#n8": "Linux-man-pages-copyleft-2-para", "git.kernel.org/pub/scm/docs/man-pages/man-pages.git/tree/man2/migrate_pages.2#n8": "Linux-man-pages-copyleft-2-para",
"git.kernel.org/pub/scm/docs/man-pages/man-pages.git/tree/man2/move_pages.2#n5": "Linux-man-pages-copyleft-2-para", "git.kernel.org/pub/scm/docs/man-pages/man-pages.git/tree/man2/move_pages.2#n5": "Linux-man-pages-copyleft-2-para",
"git.kernel.org/pub/scm/docs/man-pages/man-pages.git/tree/man2/set_mempolicy.2#n5": "Linux-man-pages-copyleft-var", "git.kernel.org/pub/scm/docs/man-pages/man-pages.git/tree/man2/set_mempolicy.2#n5": "Linux-man-pages-copyleft-var",
"git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/tree/drivers/infiniband/core/sa.h": "Linux-OpenIB", "git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/tree/drivers/infiniband/core/sa.h": "Linux-OpenIB",
"git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/tree/net/sunrpc/auth_gss/gss_generic_token.c?h=v4.19": "HPND-sell-variant", "git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/tree/net/sunrpc/auth_gss/gss_generic_token.c?h=v4.19": "HPND-sell-variant",
"git.openldap.org/openldap/openldap/-/blob/master/COPYRIGHT#L55-63": "BSD-4.3RENO", "git.openldap.org/openldap/openldap/-/blob/master/COPYRIGHT#L55-63": "BSD-4.3RENO",
"git.openldap.org/openldap/openldap/-/blob/master/COPYRIGHT?ref_type=heads#L39-51": "Furuseth", "git.openldap.org/openldap/openldap/-/blob/master/COPYRIGHT?ref_type=heads#L39-51": "Furuseth",
"git.openldap.org/openldap/openldap/-/blob/master/libraries/libldap/os-local.c?ref_type=heads#L19-23": "PADL", "git.openldap.org/openldap/openldap/-/blob/master/libraries/libldap/os-local.c?ref_type=heads#L19-23": "PADL",
"git.savannah.gnu.org/cgit/freetype/freetype2.git/tree/docs/FTL.TXT": "FTL", "git.savannah.gnu.org/cgit/freetype/freetype2.git/tree/docs/FTL.TXT": "FTL",
"git.savannah.gnu.org/cgit/gcl.git/tree/gcl/lsp/gcl_loop.lsp?h=Version_2_6_13pre": "LOOP", "git.savannah.gnu.org/cgit/gcl.git/tree/gcl/lsp/gcl_loop.lsp?h=Version_2_6_13pre": "LOOP",
"git.savannah.gnu.org/cgit/gnulib.git/tree/modules/COPYING?id=7b08932179d0d6b017f7df01a2ddf6e096b038e3": "FSFULLRSD", "git.savannah.gnu.org/cgit/gnulib.git/tree/modules/COPYING?id=7b08932179d0d6b017f7df01a2ddf6e096b038e3": "FSFULLRSD",
"git.savannah.gnu.org/cgit/indent.git/tree/doc/indent.texi?id=a74c6b4ee49397cf330b333da1042bffa60ed14f#n1788": "BSD-4.3TAHOE", "git.savannah.gnu.org/cgit/indent.git/tree/doc/indent.texi?id=a74c6b4ee49397cf330b333da1042bffa60ed14f#n1788": "BSD-4.3TAHOE",
"git.savannah.gnu.org/cgit/indent.git/tree/doc/indent.texi?id=a74c6b4ee49397cf330b333da1042bffa60ed14f#n74": "Latex2e-translated-notice", "git.savannah.gnu.org/cgit/indent.git/tree/doc/indent.texi?id=a74c6b4ee49397cf330b333da1042bffa60ed14f#n74": "Latex2e-translated-notice",
"git.savannah.gnu.org/cgit/kazlib.git/tree/except.c?id=0062df360c2d17d57f6af19b0e444c51feb99036": "Kazlib", "git.savannah.gnu.org/cgit/kazlib.git/tree/except.c?id=0062df360c2d17d57f6af19b0e444c51feb99036": "Kazlib",
"git.savannah.gnu.org/cgit/wget.git/tree/util/trunc.c?h=v1.21.3&id=40747a11e44ced5a8ac628a41f879ced3e2ebce9#n6": "FSFAP-no-warranty-disclaimer", "git.savannah.gnu.org/cgit/wget.git/tree/util/trunc.c?h=v1.21.3&id=40747a11e44ced5a8ac628a41f879ced3e2ebce9#n6": "FSFAP-no-warranty-disclaimer",
"git.sr.ht/~nabijaczleweli/ossp-var": "OSSP", "git.sr.ht/~nabijaczleweli/ossp-var": "OSSP",
"github.com/389ds/389-ds-base/blob/main/ldap/include/sysexits-compat.h#L15": "BSD-4.3TAHOE", "github.com/389ds/389-ds-base/blob/main/ldap/include/sysexits-compat.h#L15": "BSD-4.3TAHOE",
"github.com/AcademySoftwareFoundation/foundation/blob/main/digital_assets/aswf_digital_assets_license_v1.0.txt": "ASWF-Digital-Assets-1.0", "github.com/AcademySoftwareFoundation/foundation/blob/main/digital_assets/aswf_digital_assets_license_v1.0.txt": "ASWF-Digital-Assets-1.0",
"github.com/AcademySoftwareFoundation/foundation/blob/main/digital_assets/aswf_digital_assets_license_v1.1.txt": "ASWF-Digital-Assets-1.1", "github.com/AcademySoftwareFoundation/foundation/blob/main/digital_assets/aswf_digital_assets_license_v1.1.txt": "ASWF-Digital-Assets-1.1",
"github.com/AdaCore/gnatcoll-core/blob/master/docs/index.rst": "AdaCore-doc", "github.com/AdaCore/gnatcoll-core/blob/master/docs/index.rst": "AdaCore-doc",
"github.com/AdaCore/gnatcoll-db/blob/master/docs/index.rst": "AdaCore-doc", "github.com/AdaCore/gnatcoll-db/blob/master/docs/index.rst": "AdaCore-doc",
"github.com/AdaCore/xmlada/blob/master/docs/index.rst": "AdaCore-doc", "github.com/AdaCore/xmlada/blob/master/docs/index.rst": "AdaCore-doc",
"github.com/BuildCraft/BuildCraft/blob/623d323b1868712f29f4a8b0979a02e8d1835131/LICENSE": "MMPL-1.0.1", "github.com/BuildCraft/BuildCraft/blob/623d323b1868712f29f4a8b0979a02e8d1835131/LICENSE": "MMPL-1.0.1",
"github.com/CommunitySpecification/1.0/blob/master/1._Community_Specification_License-v1.md": "Community-Spec-1.0", "github.com/CommunitySpecification/1.0/blob/master/1._Community_Specification_License-v1.md": "Community-Spec-1.0",
"github.com/CorsixTH/deps/blob/fd339a9f526d1d9c9f01ccf39e438a015da50035/licences/libgsm.txt": "TU-Berlin-2.0", "github.com/CorsixTH/deps/blob/fd339a9f526d1d9c9f01ccf39e438a015da50035/licences/libgsm.txt": "TU-Berlin-2.0",
"github.com/Cyan4973/xxHash/blob/dev/doc/xxhash_spec.md": "LPD-document", "github.com/Cyan4973/xxHash/blob/dev/doc/xxhash_spec.md": "LPD-document",
"github.com/DISIC/politique-de-contribution-open-source/blob/master/LICENSE.pdf": "etalab-2.0", "github.com/DISIC/politique-de-contribution-open-source/blob/master/LICENSE.pdf": "etalab-2.0",
"github.com/Distrotech/hdparm/blob/4517550db29a91420fb2b020349523b1b4512df2/LICENSE.TXT": "hdparm", "github.com/Distrotech/hdparm/blob/4517550db29a91420fb2b020349523b1b4512df2/LICENSE.TXT": "hdparm",
"github.com/Distrotech/transfig/blob/master/transfig/transfig.c": "Xfig", "github.com/Distrotech/transfig/blob/master/transfig/transfig.c": "Xfig",
"github.com/DrItanium/maya/blob/master/LICENSE.CLIPS": "Clips", "github.com/DrItanium/maya/blob/master/LICENSE.CLIPS": "Clips",
"github.com/Dual-Life/mime-base64/blob/master/Base64.xs#L12": "metamail", "github.com/Dual-Life/mime-base64/blob/master/Base64.xs#L12": "metamail",
"github.com/EthicalSource/hippocratic-license/blob/58c0e646d64ff6fbee275bfe2b9492f914e3ab2a/LICENSE.txt": "Hippocratic-2.1", "github.com/EthicalSource/hippocratic-license/blob/58c0e646d64ff6fbee275bfe2b9492f914e3ab2a/LICENSE.txt": "Hippocratic-2.1",
"github.com/FreeRADIUS/freeradius-client/blob/master/COPYRIGHT#L32": "HPND-Fenneberg-Livingston", "github.com/Fraunhofer-IIS/mpeghdec/blob/main/LICENSE.txt": "FDK-MPEG-H",
"github.com/GNOME/gcr/blob/master/docs/COPYING": "GCR-docs", "github.com/FreeRADIUS/freeradius-client/blob/master/COPYRIGHT#L32": "HPND-Fenneberg-Livingston",
"github.com/GNOME/libxml2/blob/master/dict.c": "ISC-Veillard", "github.com/GNOME/gcr/blob/master/docs/COPYING": "GCR-docs",
"github.com/HDFGroup/hdf5/?tab=License-1-ov-file#readme": "HDF5", "github.com/GNOME/libxml2/blob/master/dict.c": "ISC-Veillard",
"github.com/KhronosGroup/SPIRV-Cross/blob/main/LICENSES/LicenseRef-KhronosFreeUse.txt": "MIT-Khronos-old", "github.com/HDFGroup/hdf5/?tab=License-1-ov-file#readme": "HDF5",
"github.com/KhronosGroup/glslang/blob/main/LICENSE.txt#L949": "AML-glslang", "github.com/KhronosGroup/SPIRV-Cross/blob/main/LICENSES/LicenseRef-KhronosFreeUse.txt": "MIT-Khronos-old",
"github.com/MariaDB/server/blob/11.6/libmysqld/lib_sql.cc": "Boehm-GC-without-fee", "github.com/KhronosGroup/glslang/blob/main/LICENSE.txt#L949": "AML-glslang",
"github.com/Neo23x0/sigma/blob/master/LICENSE.Detection.Rules.md": "DRL-1.0", "github.com/MariaDB/server/blob/11.6/libmysqld/lib_sql.cc": "Boehm-GC-without-fee",
"github.com/OGRECave/ogre/blob/master/OgreMain/include/OgreSingleton.h#L28C3-L35C46": "Game-Programming-Gems", "github.com/Neo23x0/sigma/blob/master/LICENSE.Detection.Rules.md": "DRL-1.0",
"github.com/Perl/perl5/blob/blead/util.c#L6136": "Martin-Birgmeier", "github.com/OGRECave/ogre/blob/master/OgreMain/include/OgreSingleton.h#L28C3-L35C46": "Game-Programming-Gems",
"github.com/PixarAnimationStudios/OpenSubdiv/blob/v3_5_0/opensubdiv/version.cpp#L2-L22": "Pixar", "github.com/Perl/perl5/blob/blead/util.c#L6136": "Martin-Birgmeier",
"github.com/PixarAnimationStudios/OpenSubdiv/raw/v3_5_0/LICENSE.txt": "Pixar", "github.com/PicoEFI/PicoEFI/blob/2263ac1a2bc98e5534d64a4896d3cfbbae00745a/LICENSES/LicenseRef-scancode-bsd-no-disclaimer-unmodified.txt": "BSD-Source-Code-no-disclaimer",
"github.com/RTimothyEdwards/magic/blob/master/LICENSE": "HPND-UC-export-US", "github.com/PixarAnimationStudios/OpenSubdiv/blob/v3_5_0/opensubdiv/version.cpp#L2-L22": "Pixar",
"github.com/SELinuxProject/selinux/blob/master/libselinux/LICENSE": "libselinux-1.0", "github.com/PixarAnimationStudios/OpenSubdiv/raw/v3_5_0/LICENSE.txt": "Pixar",
"github.com/SWI-Prolog/packages-clib/blob/master/sha1/brg_endian.h": "Brian-Gladman-3-Clause", "github.com/RTimothyEdwards/magic/blob/master/LICENSE": "HPND-UC-export-US",
"github.com/SWI-Prolog/packages-xpce/blob/161a40cd82004f731ba48024f9d30af388a7edf5/src/img/gifwrite.c#L21-L26": "FBM", "github.com/SELinuxProject/selinux/blob/master/libselinux/LICENSE": "libselinux-1.0",
"github.com/SWI-Prolog/swipl-devel/blob/master/src/os/dtoa.c": "dtoa", "github.com/SWI-Prolog/packages-clib/blob/master/sha1/brg_endian.h": "Brian-Gladman-3-Clause",
"github.com/SigmaHQ/Detection-Rule-License/blob/6ec7fbde6101d101b5b5d1fcb8f9b69fbc76c04a/LICENSE.Detection.Rules.md": "DRL-1.1", "github.com/SWI-Prolog/packages-xpce/blob/161a40cd82004f731ba48024f9d30af388a7edf5/src/img/gifwrite.c#L21-L26": "FBM",
"github.com/Slicer/Slicer/blob/main/License.txt": "3D-Slicer-1.0", "github.com/SWI-Prolog/swipl-devel/blob/master/src/os/dtoa.c": "dtoa",
"github.com/Unidata/UDUNITS-2/blob/master/COPYRIGHT": "UCAR", "github.com/SigmaHQ/Detection-Rule-License/blob/6ec7fbde6101d101b5b5d1fcb8f9b69fbc76c04a/LICENSE.Detection.Rules.md": "DRL-1.1",
"github.com/acpica/acpica/blob/master/source/common/acfileio.c#L119": "BSD-3-Clause-acpica", "github.com/Slicer/Slicer/blob/main/License.txt": "3D-Slicer-1.0",
"github.com/adaptivecomputing/torque/blob/master/PBS_License.txt": "OpenPBS-2.3", "github.com/Unidata/UDUNITS-2/blob/master/COPYRIGHT": "UCAR",
"github.com/adversary-org/wtfnmf/raw/3f2cd8235a64350a57a51b9739715edaea63ec1a/COPYING.WTFNMFPL-utf8": "WTFNMFPL", "github.com/acpica/acpica/blob/master/source/common/acfileio.c#L119": "BSD-3-Clause-acpica",
"github.com/adversary-org/wtfnmf/raw/refs/tags/1.0/COPYING.WTFNMFPL": "WTFNMFPL", "github.com/adaptivecomputing/torque/blob/master/PBS_License.txt": "OpenPBS-2.3",
"github.com/ap/Text-Tabs/blob/master/lib.modern/Text/Tabs.pm#L148": "TTWL", "github.com/adversary-org/wtfnmf/raw/3f2cd8235a64350a57a51b9739715edaea63ec1a/COPYING.WTFNMFPL-utf8": "WTFNMFPL",
"github.com/apache/apr/blob/trunk/LICENSE#L298C6-L298C29": "Caldera-no-preamble", "github.com/adversary-org/wtfnmf/raw/refs/tags/1.0/COPYING.WTFNMFPL": "WTFNMFPL",
"github.com/audacious-media-player/audacious/blob/master/COPYING": "BSD-2-Clause-pkgconf-disclaimer", "github.com/ap/Text-Tabs/blob/master/lib.modern/Text/Tabs.pm#L148": "TTWL",
"github.com/audacious-media-player/audacious/blob/master/src/audacious/main.cc": "BSD-2-Clause-pkgconf-disclaimer", "github.com/apache/apr/blob/trunk/LICENSE#L298C6-L298C29": "Caldera-no-preamble",
"github.com/audacious-media-player/audacious/blob/master/COPYING": "BSD-2-Clause-pkgconf-disclaimer",
"github.com/audacious-media-player/audacious/blob/master/src/audacious/main.cc": "BSD-2-Clause-pkgconf-disclaimer",
"github.com/aws/mit-0": "MIT-0", "github.com/aws/mit-0": "MIT-0",
"github.com/awsdocs/aws-cloud9-user-guide/blob/master/LICENSE-SAMPLECODE": "MIT-0", "github.com/awsdocs/aws-cloud9-user-guide/blob/master/LICENSE-SAMPLECODE": "MIT-0",
"github.com/bagder/curl/blob/master/COPYING": "curl", "github.com/bagder/curl/blob/master/COPYING": "curl",
@@ -1772,6 +1804,7 @@ var urlToLicense = map[string]string{
"github.com/file/file/blob/master/COPYING": "BSD-2-Clause-Darwin", "github.com/file/file/blob/master/COPYING": "BSD-2-Clause-Darwin",
"github.com/freebsd/freebsd-src/blob/main/lib/msun/src/e_acosh.c": "SunPro", "github.com/freebsd/freebsd-src/blob/main/lib/msun/src/e_acosh.c": "SunPro",
"github.com/freebsd/freebsd-src/blob/main/lib/msun/src/e_lgammal.c": "SunPro", "github.com/freebsd/freebsd-src/blob/main/lib/msun/src/e_lgammal.c": "SunPro",
"github.com/freebsd/pkg/blob/main/src/add.c": "BSD-2-Clause-pos-unchanged",
"github.com/gabegundy/atlserver/blob/master/License.txt": "MS-LPL", "github.com/gabegundy/atlserver/blob/master/License.txt": "MS-LPL",
"github.com/greymass/swift-eosio/blob/master/LICENSE": "BSD-3-Clause-No-Military-License", "github.com/greymass/swift-eosio/blob/master/LICENSE": "BSD-3-Clause-No-Military-License",
"github.com/hamano/man2html/blob/master/man2html.c": "man2html", "github.com/hamano/man2html/blob/master/man2html.c": "man2html",
@@ -1784,6 +1817,7 @@ var urlToLicense = map[string]string{
"github.com/jsommers/pytricia/blob/master/patricia.c#L33-L67": "BSD-4-Clause", "github.com/jsommers/pytricia/blob/master/patricia.c#L33-L67": "BSD-4-Clause",
"github.com/kfish/xsel/blob/master/COPYING": "HPND-sell-variant", "github.com/kfish/xsel/blob/master/COPYING": "HPND-sell-variant",
"github.com/kohler/t1utils/blob/master/LICENSE": "MIT-Click", "github.com/kohler/t1utils/blob/master/LICENSE": "MIT-Click",
"github.com/krb5/krb5/blob/kfw-4.0.1-beta1/src/lib/crypto/builtin/aes/aesopt.h": "Brian-Gladman-3-Clause-no-conversion",
"github.com/krb5/krb5/blob/krb5-1.21.2-final/NOTICE#L111-L133": "HPND-export2-US", "github.com/krb5/krb5/blob/krb5-1.21.2-final/NOTICE#L111-L133": "HPND-export2-US",
"github.com/krb5/krb5/blob/krb5-1.21.2-final/NOTICE#L1157-L1182": "HPND-export-US-modify", "github.com/krb5/krb5/blob/krb5-1.21.2-final/NOTICE#L1157-L1182": "HPND-export-US-modify",
"github.com/krb5/krb5/blob/krb5-1.21.2-final/NOTICE#L140-L156": "Brian-Gladman-2-Clause", "github.com/krb5/krb5/blob/krb5-1.21.2-final/NOTICE#L140-L156": "Brian-Gladman-2-Clause",
@@ -1793,6 +1827,7 @@ var urlToLicense = map[string]string{
"github.com/krb5/krb5/blob/krb5-1.21.2-final/NOTICE#L831-L852": "HPND-export-US-acknowledgement", "github.com/krb5/krb5/blob/krb5-1.21.2-final/NOTICE#L831-L852": "HPND-export-US-acknowledgement",
"github.com/krzyzanowskim/CryptoSwift/blob/main/LICENSE": "CryptoSwift", "github.com/krzyzanowskim/CryptoSwift/blob/main/LICENSE": "CryptoSwift",
"github.com/lattera/freebsd/blob/master/sys/cam/cam.c#L4": "BSD-Source-beginning-file", "github.com/lattera/freebsd/blob/master/sys/cam/cam.c#L4": "BSD-Source-beginning-file",
"github.com/libarchive/libarchive/blob/master/libarchive_fe/line_reader.c": "BSD-2-Clause-pos-unchanged",
"github.com/libcheck/check/blob/master/checkmk/checkmk.in": "checkmk", "github.com/libcheck/check/blob/master/checkmk/checkmk.in": "checkmk",
"github.com/libjpeg-turbo/libjpeg-turbo/blob/main/README.ijg#L117-L161": "IJG", "github.com/libjpeg-turbo/libjpeg-turbo/blob/main/README.ijg#L117-L161": "IJG",
"github.com/lsof-org/lsof/blob/master/COPYING": "lsof", "github.com/lsof-org/lsof/blob/master/COPYING": "lsof",
@@ -1805,6 +1840,7 @@ var urlToLicense = map[string]string{
"github.com/mm2/Little-CMS/blob/master/src/cmssm.c#L207": "softSurfer", "github.com/mm2/Little-CMS/blob/master/src/cmssm.c#L207": "softSurfer",
"github.com/mruby/mruby/blob/83d12f8d52522cdb7c8cc46fad34821359f453e6/mrbgems/mruby-dir/src/Win/dirent.c#L127-L140": "HPND-Kevlin-Henney", "github.com/mruby/mruby/blob/83d12f8d52522cdb7c8cc46fad34821359f453e6/mrbgems/mruby-dir/src/Win/dirent.c#L127-L140": "HPND-Kevlin-Henney",
"github.com/mtoyoda/sl/blob/master/LICENSE": "SL", "github.com/mtoyoda/sl/blob/master/LICENSE": "SL",
"github.com/mvt-project/mvt/blob/main/LICENSE": "MVT-1.1",
"github.com/n8n-io/n8n/blob/master/LICENSE.md": "SUL-1.0", "github.com/n8n-io/n8n/blob/master/LICENSE.md": "SUL-1.0",
"github.com/ncbi/datasets/blob/0ea4cd16b61e5b799d9cc55aecfa016d6c9bd2bf/LICENSE.md": "NCBI-PD", "github.com/ncbi/datasets/blob/0ea4cd16b61e5b799d9cc55aecfa016d6c9bd2bf/LICENSE.md": "NCBI-PD",
"github.com/ncbi/datasets/blob/master/LICENSE.md": "NCBI-PD", "github.com/ncbi/datasets/blob/master/LICENSE.md": "NCBI-PD",
@@ -1815,6 +1851,7 @@ var urlToLicense = map[string]string{
"github.com/novnc/noVNC/blob/master/core/crypto/des.js#L24": "Widget-Workshop", "github.com/novnc/noVNC/blob/master/core/crypto/des.js#L24": "Widget-Workshop",
"github.com/oetiker/rrdtool-1.x/blob/master/src/plbasename.c#L8-L11": "gtkbook", "github.com/oetiker/rrdtool-1.x/blob/master/src/plbasename.c#L8-L11": "gtkbook",
"github.com/open-quantum-safe/liboqs/blob/40b01fdbb270f8614fde30e65d30e9da18c02393/src/common/rand/rand_nist.c#L1-L15": "NIST-Software", "github.com/open-quantum-safe/liboqs/blob/40b01fdbb270f8614fde30e65d30e9da18c02393/src/common/rand/rand_nist.c#L1-L15": "NIST-Software",
"github.com/open-webui/open-webui/blob/main/LICENSE": "BSD-3-Clause-OpenWebUI",
"github.com/openjdk/jdk/blob/6162e2c5213c5dd7c1127fd9616b543efa898962/LICENSE": "GPL-2.0-or-later", "github.com/openjdk/jdk/blob/6162e2c5213c5dd7c1127fd9616b543efa898962/LICENSE": "GPL-2.0-or-later",
"github.com/openssh/openssh-portable/blob/1b11ea7c58cd5c59838b5fa574cd456d6047b2d4/LICENCE#L10": "SSH-OpenSSH", "github.com/openssh/openssh-portable/blob/1b11ea7c58cd5c59838b5fa574cd456d6047b2d4/LICENCE#L10": "SSH-OpenSSH",
"github.com/openssh/openssh-portable/blob/1b11ea7c58cd5c59838b5fa574cd456d6047b2d4/pathnames.h": "SSH-short", "github.com/openssh/openssh-portable/blob/1b11ea7c58cd5c59838b5fa574cd456d6047b2d4/pathnames.h": "SSH-short",
@@ -1846,6 +1883,7 @@ var urlToLicense = map[string]string{
"github.com/sigmavirus24/x11-ssh-askpass/blob/master/dynlist.c": "HPND-sell-variant-MIT-disclaimer-rev", "github.com/sigmavirus24/x11-ssh-askpass/blob/master/dynlist.c": "HPND-sell-variant-MIT-disclaimer-rev",
"github.com/signal11/hidapi/blob/master/LICENSE-orig.txt": "HIDAPI", "github.com/signal11/hidapi/blob/master/LICENSE-orig.txt": "HIDAPI",
"github.com/slogan621/gtkbook": "gtkbook", "github.com/slogan621/gtkbook": "gtkbook",
"github.com/sudo-project/sudo/blob/c1307ea9ff340ce0538779f8e456501461fc44b7/plugins/sudoers/redblack.c#L24-L43": "BSD-ask-to-endorse",
"github.com/swh/ladspa/blob/7bf6f3799fdba70fda297c2d8fd9f526803d9680/gsm/COPYRIGHT": "TU-Berlin-1.0", "github.com/swh/ladspa/blob/7bf6f3799fdba70fda297c2d8fd9f526803d9680/gsm/COPYRIGHT": "TU-Berlin-1.0",
"github.com/tats/w3m/blob/master/COPYING": "w3m", "github.com/tats/w3m/blob/master/COPYING": "w3m",
"github.com/tcheneau/Routing/blob/f09f46fcfe636107f22f2c98348188a65a135d98/README.md": "NIST-PD", "github.com/tcheneau/Routing/blob/f09f46fcfe636107f22f2c98348188a65a135d98/README.md": "NIST-PD",
@@ -1856,6 +1894,7 @@ var urlToLicense = map[string]string{
"github.com/uim/libgcroots/blob/master/COPYING": "Boehm-GC", "github.com/uim/libgcroots/blob/master/COPYING": "Boehm-GC",
"github.com/usnistgov/fipy/blob/86aaa5c2ba2c6f1be19593c5986071cf6568cc34/LICENSE.rst": "NIST-PD-fallback", "github.com/usnistgov/fipy/blob/86aaa5c2ba2c6f1be19593c5986071cf6568cc34/LICENSE.rst": "NIST-PD-fallback",
"github.com/usnistgov/jsip/blob/59700e6926cbe96c5cdae897d9a7d2656b42abe3/LICENSE": "NIST-PD-fallback", "github.com/usnistgov/jsip/blob/59700e6926cbe96c5cdae897d9a7d2656b42abe3/LICENSE": "NIST-PD-fallback",
"github.com/vattam/BSDGames/blob/master/atc/extern.c#L36": "atc-game",
"github.com/vixie/cron/tree/545b3f5246824a9cda5905eeb7cf019c95e66995": "Vixie-Cron", "github.com/vixie/cron/tree/545b3f5246824a9cda5905eeb7cf019c95e66995": "Vixie-Cron",
"github.com/vstroebel/jpeg-encoder/blob/main/src/fdct.rs#L1-L72": "IJG", "github.com/vstroebel/jpeg-encoder/blob/main/src/fdct.rs#L1-L72": "IJG",
"github.com/wernerd/ZRTPCPP/blob/6b3cd8e6783642292bad0c21e3e5e5ce45ff3e03/cryptcommon/twofish.c#L113C3-L127": "Ferguson-Twofish", "github.com/wernerd/ZRTPCPP/blob/6b3cd8e6783642292bad0c21e3e5e5ce45ff3e03/cryptcommon/twofish.c#L113C3-L127": "Ferguson-Twofish",
@@ -1869,6 +1908,7 @@ var urlToLicense = map[string]string{
"gitlab.com/embeddable-common-lisp/ecl/-/blob/develop/src/lsp/loop.lsp": "LOOP", "gitlab.com/embeddable-common-lisp/ecl/-/blob/develop/src/lsp/loop.lsp": "LOOP",
"gitlab.com/libeigen/eigen/-/blob/master/COPYING.MINPACK": "Minpack", "gitlab.com/libeigen/eigen/-/blob/master/COPYING.MINPACK": "Minpack",
"gitlab.common-lisp.net/cmucl/cmucl/-/blob/master/src/code/loop.lisp": "LOOP", "gitlab.common-lisp.net/cmucl/cmucl/-/blob/master/src/code/loop.lisp": "LOOP",
"gitlab.freedesktop.org/dbus/dbus/-/blob/789d97ad53044f94351a958e785c2923643b346a/COPYING#L11-207": "AFL-2.1",
"gitlab.freedesktop.org/freetype/freetype/-/raw/master/docs/FTL.TXT": "FTL", "gitlab.freedesktop.org/freetype/freetype/-/raw/master/docs/FTL.TXT": "FTL",
"gitlab.freedesktop.org/libopenraw/libopenraw/blob/master/lib/ljpegdecompressor.cpp#L32": "Cornell-Lossless-JPEG", "gitlab.freedesktop.org/libopenraw/libopenraw/blob/master/lib/ljpegdecompressor.cpp#L32": "Cornell-Lossless-JPEG",
"gitlab.freedesktop.org/mesa/glw/-/blob/master/README?ref_type=heads": "SGI-OpenGL", "gitlab.freedesktop.org/mesa/glw/-/blob/master/README?ref_type=heads": "SGI-OpenGL",
@@ -1892,6 +1932,7 @@ var urlToLicense = map[string]string{
"gitlab.freedesktop.org/xorg/lib/libxinerama/-/blob/cc22c2f60c3862482562955116d5455263b443dc/COPYING#L44-66": "X11-no-permit-persons", "gitlab.freedesktop.org/xorg/lib/libxinerama/-/blob/cc22c2f60c3862482562955116d5455263b443dc/COPYING#L44-66": "X11-no-permit-persons",
"gitlab.freedesktop.org/xorg/lib/libxtst/-/blob/master/COPYING?ref_type=heads#L108-117": "HPND-doc-sell", "gitlab.freedesktop.org/xorg/lib/libxtst/-/blob/master/COPYING?ref_type=heads#L108-117": "HPND-doc-sell",
"gitlab.freedesktop.org/xorg/lib/libxtst/-/blob/master/COPYING?ref_type=heads#L70-77": "HPND-doc", "gitlab.freedesktop.org/xorg/lib/libxtst/-/blob/master/COPYING?ref_type=heads#L70-77": "HPND-doc",
"gitlab.freedesktop.org/xorg/xserver/-/blob/dd5c2595a42d3ff0c4f18d9b53d1f6c3fd934fd4/COPYING#L365-389": "MIT",
"gitlab.freedesktop.org/xorg/xserver/-/blob/master/COPYING?ref_type=heads#L1781": "HPND-sell-MIT-disclaimer-xserver", "gitlab.freedesktop.org/xorg/xserver/-/blob/master/COPYING?ref_type=heads#L1781": "HPND-sell-MIT-disclaimer-xserver",
"gitlab.freedesktop.org/xorg/xserver/-/blob/master/COPYING?ref_type=heads#L239": "DEC-3-Clause", "gitlab.freedesktop.org/xorg/xserver/-/blob/master/COPYING?ref_type=heads#L239": "DEC-3-Clause",
"gitlab.freedesktop.org/xorg/xserver/-/blob/master/COPYING?ref_type=heads#L752": "Adobe-Display-PostScript", "gitlab.freedesktop.org/xorg/xserver/-/blob/master/COPYING?ref_type=heads#L752": "Adobe-Display-PostScript",
@@ -1906,6 +1947,7 @@ var urlToLicense = map[string]string{
"heasarc.gsfc.nasa.gov/docs/software/ftools/fv/doc/license.html": "CFITSIO", "heasarc.gsfc.nasa.gov/docs/software/ftools/fv/doc/license.html": "CFITSIO",
"helixcommunity.org/content/rpsl": "RPSL-1.0", "helixcommunity.org/content/rpsl": "RPSL-1.0",
"hstechdocs.helpsystems.com/manuals/globalscape/archive/cuteftp6/open_ssl_license_agreement.htm": "OpenSSL-standalone", "hstechdocs.helpsystems.com/manuals/globalscape/archive/cuteftp6/open_ssl_license_agreement.htm": "OpenSSL-standalone",
"interoperable-europe.ec.europa.eu/sites/default/files/inline-files/EUPL%20v1_2%20EN(1).txt": "EUPL-1.2",
"java.net/projects/javaeetutorial/pages/BerkeleyLicense": "BSD-3-Clause-No-Nuclear-License-2014", "java.net/projects/javaeetutorial/pages/BerkeleyLicense": "BSD-3-Clause-No-Nuclear-License-2014",
"javaee.github.io/glassfish/LICENSE": "CDDL-1.1", "javaee.github.io/glassfish/LICENSE": "CDDL-1.1",
"jogamp.org/git/?p=gluegen.git;a=blob_plain;f=LICENSE.txt": "BSD-3-Clause-No-Nuclear-Warranty", "jogamp.org/git/?p=gluegen.git;a=blob_plain;f=LICENSE.txt": "BSD-3-Clause-No-Nuclear-Warranty",
@@ -1924,6 +1966,7 @@ var urlToLicense = map[string]string{
"library.netapp.com/ecm/ecm_download_file/ECMP1196395": "OpenSSL-standalone", "library.netapp.com/ecm/ecm_download_file/ECMP1196395": "OpenSSL-standalone",
"license.coscl.org.cn/MulanPSL/": "MulanPSL-1.0", "license.coscl.org.cn/MulanPSL/": "MulanPSL-1.0",
"license.coscl.org.cn/MulanPSL2": "MulanPSL-2.0", "license.coscl.org.cn/MulanPSL2": "MulanPSL-2.0",
"license.mvt.re/1.1/": "MVT-1.1",
"lists.gnu.org/archive/html/autoconf/2012-04/msg00061.html": "FSFULLRWD", "lists.gnu.org/archive/html/autoconf/2012-04/msg00061.html": "FSFULLRWD",
"lists.opensource.org/pipermail/license-discuss_lists.opensource.org/2002-November/006304.html": "HPND", "lists.opensource.org/pipermail/license-discuss_lists.opensource.org/2002-November/006304.html": "HPND",
"mariadb.com/bsl11/": "BUSL-1.1", "mariadb.com/bsl11/": "BUSL-1.1",
@@ -1965,99 +2008,105 @@ var urlToLicense = map[string]string{
"opensource.linux-mirror.org/licenses/afl-1.1.txt": "AFL-1.1", "opensource.linux-mirror.org/licenses/afl-1.1.txt": "AFL-1.1",
"opensource.linux-mirror.org/licenses/afl-1.2.txt": "AFL-1.2", "opensource.linux-mirror.org/licenses/afl-1.2.txt": "AFL-1.2",
"opensource.linux-mirror.org/licenses/afl-2.1.txt": "AFL-2.1", "opensource.linux-mirror.org/licenses/afl-2.1.txt": "AFL-2.1",
"opensource.org/license/apache-2-0": "Apache-2.0", "opensource.org/license/0BSD": "0BSD",
"opensource.org/license/mit/": "MIT", "opensource.org/license/AAL": "AAL",
"opensource.org/license/olfl-1-3/": "OLFL-1.3", "opensource.org/license/AFL-3.0": "AFL-3.0",
"opensource.org/license/osc-license-1-0": "OSC-1.0", "opensource.org/license/AGPL-3.0": "AGPL-3.0-or-later",
"opensource.org/licenses/0BSD": "0BSD", "opensource.org/license/AGPL-3.0-only": "AGPL-3.0-only",
"opensource.org/licenses/AGPL-3.0": "AGPL-3.0-or-later", "opensource.org/license/APL-1.0": "APL-1.0",
"opensource.org/licenses/APL-1.0": "APL-1.0", "opensource.org/license/Artistic-1.0": "Artistic-1.0-cl8",
"opensource.org/licenses/Apache-1.1": "Apache-1.1", "opensource.org/license/Artistic-2.0": "Artistic-2.0",
"opensource.org/licenses/Apache-2.0": "Apache-2.0", "opensource.org/license/BSD-2-Clause": "BSD-2-Clause",
"opensource.org/licenses/Artistic-1.0": "Artistic-1.0-cl8", "opensource.org/license/BSD-2-Clause-Patent": "BSD-2-Clause-Patent",
"opensource.org/licenses/BSD-2-Clause": "BSD-2-Clause", "opensource.org/license/BSD-3-Clause": "BSD-3-Clause",
"opensource.org/licenses/BSD-3-Clause": "BSD-3-Clause", "opensource.org/license/BSL-1.0": "BSL-1.0",
"opensource.org/licenses/BSDplusPatent": "BSD-2-Clause-Patent", "opensource.org/license/CAL-1.0": "CAL-1.0-Combined-Work-Exception",
"opensource.org/licenses/BSL-1.0": "BSL-1.0", "opensource.org/license/CATOSL-1.1": "CATOSL-1.1",
"opensource.org/licenses/CAL-1.0": "CAL-1.0-Combined-Work-Exception", "opensource.org/license/CDDL-1.0": "CDDL-1.0",
"opensource.org/licenses/CATOSL-1.1": "CATOSL-1.1", "opensource.org/license/CNRI-Python": "CNRI-Python",
"opensource.org/licenses/CNRI-Python": "CNRI-Python", "opensource.org/license/CNRI-Python-GPL-Compatible": "CNRI-Python-GPL-Compatible",
"opensource.org/licenses/CPAL-1.0": "CPAL-1.0", "opensource.org/license/CPAL-1.0": "CPAL-1.0",
"opensource.org/licenses/CPL-1.0": "CPL-1.0", "opensource.org/license/CPL-1.0": "CPL-1.0",
"opensource.org/licenses/CUA-OPL-1.0": "CUA-OPL-1.0", "opensource.org/license/CUA-OPL-1.0": "CUA-OPL-1.0",
"opensource.org/licenses/ECL-1.0": "ECL-1.0", "opensource.org/license/ECL-1.0": "ECL-1.0",
"opensource.org/licenses/ECL-2.0": "ECL-2.0", "opensource.org/license/ECL-2.0": "ECL-2.0",
"opensource.org/licenses/EFL-1.0": "EFL-1.0", "opensource.org/license/EFL-1.0": "EFL-1.0",
"opensource.org/licenses/EFL-2.0": "EFL-2.0", "opensource.org/license/EFL-2.0": "EFL-2.0",
"opensource.org/licenses/EPL-1.0": "EPL-1.0", "opensource.org/license/EPL-1.0": "EPL-1.0",
"opensource.org/licenses/EUDatagrid": "EUDatagrid", "opensource.org/license/EUDatagrid": "EUDatagrid",
"opensource.org/licenses/EUPL-1.1": "EUPL-1.1", "opensource.org/license/EUPL-1.1": "EUPL-1.1",
"opensource.org/licenses/EUPL-1.2": "EUPL-1.2", "opensource.org/license/EUPL-1.2": "EUPL-1.2",
"opensource.org/licenses/Entessa": "Entessa", "opensource.org/license/Entessa": "Entessa",
"opensource.org/licenses/Fair": "Fair", "opensource.org/license/Fair": "Fair",
"opensource.org/licenses/Frameworx-1.0": "Frameworx-1.0", "opensource.org/license/Frameworx-1.0": "Frameworx-1.0",
"opensource.org/licenses/GPL-2.0": "GPL-2.0-or-later", "opensource.org/license/GPL-2.0": "GPL-2.0-or-later",
"opensource.org/licenses/GPL-3.0": "GPL-3.0-or-later", "opensource.org/license/GPL-2.0-only": "GPL-2.0-only",
"opensource.org/licenses/HPND": "HPND", "opensource.org/license/GPL-3.0": "GPL-3.0-or-later",
"opensource.org/licenses/IPA": "IPA", "opensource.org/license/GPL-3.0-only": "GPL-3.0-only",
"opensource.org/licenses/IPL-1.0": "IPL-1.0", "opensource.org/license/HPND": "HPND",
"opensource.org/licenses/ISC": "ISC", "opensource.org/license/IPA": "IPA",
"opensource.org/licenses/Intel": "Intel", "opensource.org/license/IPL-1.0": "IPL-1.0",
"opensource.org/licenses/LGPL-2.1": "LGPL-2.1-or-later", "opensource.org/license/ISC": "ISC",
"opensource.org/licenses/LGPL-3.0": "LGPL-3.0-or-later", "opensource.org/license/Intel": "Intel",
"opensource.org/licenses/LPL-1.0": "LPL-1.0", "opensource.org/license/LGPL-2.1": "LGPL-2.1-or-later",
"opensource.org/licenses/LPL-1.02": "LPL-1.02", "opensource.org/license/LGPL-2.1-only": "LGPL-2.1-only",
"opensource.org/licenses/LPPL-1.3c": "LPPL-1.3c", "opensource.org/license/LGPL-3.0": "LGPL-3.0-or-later",
"opensource.org/licenses/LiLiQ-P-1.1": "LiLiQ-P-1.1", "opensource.org/license/LPL-1.0": "LPL-1.0",
"opensource.org/licenses/LiLiQ-R-1.1": "LiLiQ-R-1.1", "opensource.org/license/LPL-1.02": "LPL-1.02",
"opensource.org/licenses/LiLiQ-Rplus-1.1": "LiLiQ-Rplus-1.1", "opensource.org/license/LPPL-1.3c": "LPPL-1.3c",
"opensource.org/licenses/MIT": "MIT", "opensource.org/license/LiLiQ-P-1.1": "LiLiQ-P-1.1",
"opensource.org/licenses/MPL-1.0": "MPL-1.0", "opensource.org/license/LiLiQ-R-1.1": "LiLiQ-R-1.1",
"opensource.org/licenses/MPL-1.1": "MPL-1.1", "opensource.org/license/LiLiQ-Rplus-1.1": "LiLiQ-Rplus-1.1",
"opensource.org/licenses/MPL-2.0": "MPL-2.0-no-copyleft-exception", "opensource.org/license/MIT": "MIT",
"opensource.org/licenses/MS-PL": "MS-PL", "opensource.org/license/MPL-1.0": "MPL-1.0",
"opensource.org/licenses/MS-RL": "MS-RL", "opensource.org/license/MPL-1.1": "MPL-1.1",
"opensource.org/licenses/MirOS": "MirOS", "opensource.org/license/MPL-2.0": "MPL-2.0-no-copyleft-exception",
"opensource.org/licenses/Motosoto": "Motosoto", "opensource.org/license/MS-PL": "MS-PL",
"opensource.org/licenses/Multics": "Multics", "opensource.org/license/MS-RL": "MS-RL",
"opensource.org/licenses/NASA-1.3": "NASA-1.3", "opensource.org/license/MirOS": "MirOS",
"opensource.org/licenses/NCSA": "NCSA", "opensource.org/license/Motosoto": "Motosoto",
"opensource.org/licenses/NGPL": "NGPL", "opensource.org/license/Multics": "Multics",
"opensource.org/licenses/NOSL3.0": "NPOSL-3.0", "opensource.org/license/NASA-1.3": "NASA-1.3",
"opensource.org/licenses/NTP": "NTP", "opensource.org/license/NCSA": "NCSA",
"opensource.org/licenses/Naumen": "Naumen", "opensource.org/license/NGPL": "NGPL",
"opensource.org/licenses/OCLC-2.0": "OCLC-2.0", "opensource.org/license/NPOSL-3.0": "NPOSL-3.0",
"opensource.org/licenses/OFL-1.1": "OFL-1.1-no-RFN", "opensource.org/license/NTP": "NTP",
"opensource.org/licenses/OGTSL": "OGTSL", "opensource.org/license/Naumen": "Naumen",
"opensource.org/licenses/OPL-2.1": "OSET-PL-2.1", "opensource.org/license/Nokia": "Nokia",
"opensource.org/licenses/OSL-1.0": "OSL-1.0", "opensource.org/license/OCLC-2.0": "OCLC-2.0",
"opensource.org/licenses/OSL-2.1": "OSL-2.1", "opensource.org/license/OFL-1.1": "OFL-1.1-no-RFN",
"opensource.org/licenses/OSL-3.0": "OSL-3.0", "opensource.org/license/OGTSL": "OGTSL",
"opensource.org/licenses/PHP-3.0": "PHP-3.0", "opensource.org/license/OLFL-1.3": "OLFL-1.3",
"opensource.org/licenses/PostgreSQL": "PostgreSQL", "opensource.org/license/OSC-1.0": "OSC-1.0",
"opensource.org/licenses/Python-2.0": "Python-2.0", "opensource.org/license/OSET-PL-2.1": "OSET-PL-2.1",
"opensource.org/licenses/QPL-1.0": "QPL-1.0", "opensource.org/license/OSL-1.0": "OSL-1.0",
"opensource.org/licenses/RPL-1.1": "RPL-1.1", "opensource.org/license/OSL-2.1": "OSL-2.1",
"opensource.org/licenses/RPL-1.5": "RPL-1.5", "opensource.org/license/OSL-3.0": "OSL-3.0",
"opensource.org/licenses/RPSL-1.0": "RPSL-1.0", "opensource.org/license/PHP-3.0": "PHP-3.0",
"opensource.org/licenses/RSCPL": "RSCPL", "opensource.org/license/PSF-2-0": "PSF-2.0",
"opensource.org/licenses/SISSL": "SISSL", "opensource.org/license/PostgreSQL": "PostgreSQL",
"opensource.org/licenses/SPL-1.0": "SPL-1.0", "opensource.org/license/Python-2.0": "Python-2.0",
"opensource.org/licenses/SimPL-2.0": "SimPL-2.0", "opensource.org/license/Python-2.0.1": "Python-2.0.1",
"opensource.org/licenses/Sleepycat": "Sleepycat", "opensource.org/license/QPL-1.0": "QPL-1.0",
"opensource.org/licenses/UCL-1.0": "UCL-1.0", "opensource.org/license/RPL-1.1": "RPL-1.1",
"opensource.org/licenses/UPL": "UPL-1.0", "opensource.org/license/RPL-1.5": "RPL-1.5",
"opensource.org/licenses/VSL-1.0": "VSL-1.0", "opensource.org/license/RPSL-1.0": "RPSL-1.0",
"opensource.org/licenses/W3C": "W3C", "opensource.org/license/RSCPL": "RSCPL",
"opensource.org/licenses/Watcom-1.0": "Watcom-1.0", "opensource.org/license/SISSL": "SISSL",
"opensource.org/licenses/Xnet": "Xnet", "opensource.org/license/SPL-1.0": "SPL-1.0",
"opensource.org/licenses/ZPL-2.0": "ZPL-2.0", "opensource.org/license/SiMPL-2.0": "SimPL-2.0",
"opensource.org/licenses/Zlib": "Zlib", "opensource.org/license/Sleepycat": "Sleepycat",
"opensource.org/licenses/afl-3.0": "AFL-3.0", "opensource.org/license/UCL-1.0": "UCL-1.0",
"opensource.org/licenses/artistic-license-2.0": "Artistic-2.0", "opensource.org/license/UPL-1.0": "UPL-1.0",
"opensource.org/licenses/attribution": "AAL", "opensource.org/license/VSL-1.0": "VSL-1.0",
"opensource.org/licenses/cddl1": "CDDL-1.0", "opensource.org/license/W3C": "W3C",
"opensource.org/licenses/nokia": "Nokia", "opensource.org/license/Watcom-1.0": "Watcom-1.0",
"opensource.org/license/Xnet": "Xnet",
"opensource.org/license/ZPL-2.0": "ZPL-2.0",
"opensource.org/license/Zlib": "Zlib",
"opensource.org/license/apache-1.1": "Apache-1.1",
"opensource.org/license/apache-2.0": "Apache-2.0",
"opensource.org/license/curl": "curl",
"oss.sgi.com/projects/FreeB/": "SGI-B-1.1", "oss.sgi.com/projects/FreeB/": "SGI-B-1.1",
"oss.sgi.com/projects/FreeB/SGIFreeSWLicB.1.0.html": "SGI-B-1.0", "oss.sgi.com/projects/FreeB/SGIFreeSWLicB.1.0.html": "SGI-B-1.0",
"oss.sgi.com/projects/FreeB/SGIFreeSWLicB.2.0.pdf": "SGI-B-2.0", "oss.sgi.com/projects/FreeB/SGIFreeSWLicB.2.0.pdf": "SGI-B-2.0",
@@ -2082,37 +2131,40 @@ var urlToLicense = map[string]string{
"raw.githubusercontent.com/NTIA/scos-sensor/refs/heads/master/LICENSE.md": "NTIA-PD", "raw.githubusercontent.com/NTIA/scos-sensor/refs/heads/master/LICENSE.md": "NTIA-PD",
"raw.githubusercontent.com/OpenMDW/OpenMDW/refs/heads/main/1.0/LICENSE.openmdw": "OpenMDW-1.0", "raw.githubusercontent.com/OpenMDW/OpenMDW/refs/heads/main/1.0/LICENSE.openmdw": "OpenMDW-1.0",
"raw.githubusercontent.com/blackhole89/notekit/master/data/fonts/Charter%20license.txt": "Bitstream-Charter", "raw.githubusercontent.com/blackhole89/notekit/master/data/fonts/Charter%20license.txt": "Bitstream-Charter",
"rc.quest.com/topics/openssh/license.php#tcpwrappers": "TCP-wrappers", "raw.githubusercontent.com/landtuna/open-mtools/6c04c2882eca7c4313ec267766ea53ac9bfe129b/mdump.c": "Informatica",
"research.cs.wisc.edu/condor/license.html#condor": "Condor-1.1", "raw.githubusercontent.com/landtuna/open-mtools/6c04c2882eca7c4313ec267766ea53ac9bfe129b/msend.c": "Informatica",
"research.scea.com/scea_shared_source_license.html": "SCEA", "rc.quest.com/topics/openssh/license.php#tcpwrappers": "TCP-wrappers",
"romanrm.net/mit-zero": "MIT-0", "research.cs.wisc.edu/condor/license.html#condor": "Condor-1.1",
"sam.zoy.org/wtfpl/COPYING": "WTFPL", "research.scea.com/scea_shared_source_license.html": "SCEA",
"scripts.sil.org/cms/scripts/page.php?item_id=OFL10_web": "OFL-1.0-no-RFN", "romanrm.net/mit-zero": "MIT-0",
"scripts.sil.org/cms/scripts/page.php?item_id=OFL_web": "OFL-1.1-no-RFN", "sam.zoy.org/wtfpl/COPYING": "WTFPL",
"slicer.org/LICENSE": "3D-Slicer-1.0", "scancode-licensedb.aboutcode.org/bsd-unchanged": "BSD-2-Clause-pos-unchanged",
"solderpad.org/licenses/SHL-0.5/": "SHL-0.5", "scripts.sil.org/cms/scripts/page.php?item_id=OFL10_web": "OFL-1.0-no-RFN",
"solderpad.org/licenses/SHL-0.51/": "SHL-0.51", "scripts.sil.org/cms/scripts/page.php?item_id=OFL_web": "OFL-1.1-no-RFN",
"source.icu-project.org/repos/icu/icu/trunk/license.html": "ICU", "slicer.org/LICENSE": "3D-Slicer-1.0",
"sourceforge.net/p/buddy/gitcode/ci/master/tree/README": "Buddy", "solderpad.org/licenses/SHL-0.5/": "SHL-0.5",
"sourceforge.net/p/ctrio/git/ci/master/tree/README": "ISC-Veillard", "solderpad.org/licenses/SHL-0.51/": "SHL-0.51",
"sourceforge.net/p/mcj/xfig/ci/master/tree/src/Makefile.am": "Xfig", "source.icu-project.org/repos/icu/icu/trunk/license.html": "ICU",
"sourceforge.net/p/netpbm/code/HEAD/tree/super_stable/converter/other/pnmtorle.c": "URT-RLE", "sourceforge.net/p/buddy/gitcode/ci/master/tree/README": "Buddy",
"sourceforge.net/p/netpbm/code/HEAD/tree/super_stable/converter/other/rletopnm.c": "URT-RLE", "sourceforge.net/p/ctrio/git/ci/master/tree/README": "ISC-Veillard",
"sourceforge.net/p/netpbm/code/HEAD/tree/super_stable/converter/ppm/ppmtompeg/jrevdct.c#l1189": "MPEG-SSG", "sourceforge.net/p/mcj/xfig/ci/master/tree/src/Makefile.am": "Xfig",
"sourceforge.net/p/netpbm/code/HEAD/tree/super_stable/editor/pnmstitch.c#l2": "pnmstitch", "sourceforge.net/p/netpbm/code/HEAD/tree/super_stable/converter/other/pnmtorle.c": "URT-RLE",
"sourceforge.net/p/netpbm/code/HEAD/tree/super_stable/netpbm.c#l8": "HPND-Pbmplus", "sourceforge.net/p/netpbm/code/HEAD/tree/super_stable/converter/other/rletopnm.c": "URT-RLE",
"sourceforge.net/p/sbcl/sbcl/ci/master/tree/src/code/loop.lisp": "LOOP", "sourceforge.net/p/netpbm/code/HEAD/tree/super_stable/converter/ppm/ppmtompeg/jrevdct.c#l1189": "MPEG-SSG",
"sourceforge.net/p/trustedqsl/tqsl/ci/master/tree/LICENSE.txt": "TrustedQSL", "sourceforge.net/p/netpbm/code/HEAD/tree/super_stable/editor/pnmstitch.c#l2": "pnmstitch",
"sourceforge.net/p/xmedcon/code/ci/master/tree/libs/dicom/README": "OFFIS", "sourceforge.net/p/netpbm/code/HEAD/tree/super_stable/netpbm.c#l8": "HPND-Pbmplus",
"sourceforge.net/p/xmedcon/code/ci/master/tree/libs/ljpg/": "IJG-short", "sourceforge.net/p/sbcl/sbcl/ci/master/tree/src/code/loop.lisp": "LOOP",
"sourceforge.net/projects/sevenzip/files/LZMA%20SDK/": "LZMA-SDK-9.22", "sourceforge.net/p/trustedqsl/tqsl/ci/master/tree/LICENSE.txt": "TrustedQSL",
"sources.debian.org/copyright/license/debianutils/4.11.2/": "SMAIL-GPL", "sourceforge.net/p/xmedcon/code/ci/master/tree/libs/dicom/README": "OFFIS",
"sources.debian.org/src/openmpi/4.1.0-10/ompi/debuggers/msgq_interface.h/?hl=19#L19": "mpi-permissive", "sourceforge.net/p/xmedcon/code/ci/master/tree/libs/ljpg/": "IJG-short",
"sources.gentoo.org/cgi-bin/viewvc.cgi/gentoo-x86/licenses/BitTorrent?r1=1.1&r2=1.1.1.1&diff_format=s": "BitTorrent-1.0", "sourceforge.net/projects/sevenzip/files/LZMA%20SDK/": "LZMA-SDK-9.22",
"sourceware.org/bzip2/1.0.5/bzip2-manual-1.0.5.html": "bzip2-1.0.6", "sources.debian.org/copyright/license/debianutils/4.11.2/": "SMAIL-GPL",
"sourceware.org/cgit/binutils-gdb/tree/include/coff/sym.h#n11": "MIPS", "sources.debian.org/src/openmpi/4.1.0-10/ompi/debuggers/msgq_interface.h/?hl=19#L19": "mpi-permissive",
"sourceware.org/cgit/binutils-gdb/tree/include/coff/symconst.h#n11": "ThirdEye", "sources.gentoo.org/cgi-bin/viewvc.cgi/gentoo-x86/licenses/BitTorrent?r1=1.1&r2=1.1.1.1&diff_format=s": "BitTorrent-1.0",
"sourceware.org/cgit/valgrind/tree/mpi/libmpiwrap.c": "bzip2-1.0.6", "sourceware.org/bzip2/1.0.5/bzip2-manual-1.0.5.html": "bzip2-1.0.6",
"sourceware.org/cgit/binutils-gdb/tree/include/coff/sym.h#n11": "MIPS",
"sourceware.org/cgit/binutils-gdb/tree/include/coff/symconst.h#n11": "ThirdEye",
"sourceware.org/cgit/valgrind/tree/mpi/libmpiwrap.c": "bzip2-1.0.6",
"sourceware.org/git/?p=binutils-gdb.git;a=blob;f=libiberty/strcasecmp.c;h=131d81c2ce7881fa48c363dc5bf5fb302c61ce0b;hb=HEAD": "BSD-4.3RENO", "sourceware.org/git/?p=binutils-gdb.git;a=blob;f=libiberty/strcasecmp.c;h=131d81c2ce7881fa48c363dc5bf5fb302c61ce0b;hb=HEAD": "BSD-4.3RENO",
"sourceware.org/git/?p=binutils-gdb.git;a=blob;f=readline/readline/support/wcwidth.c;h=0f5ec995796f4813abbcf4972aec0378ab74722a;hb=HEAD#l55": "HPND-Markus-Kuhn", "sourceware.org/git/?p=binutils-gdb.git;a=blob;f=readline/readline/support/wcwidth.c;h=0f5ec995796f4813abbcf4972aec0378ab74722a;hb=HEAD#l55": "HPND-Markus-Kuhn",
"sourceware.org/git/?p=bzip2.git;a=blob;f=LICENSE;hb=bzip2-1.0.6": "bzip2-1.0.6", "sourceware.org/git/?p=bzip2.git;a=blob;f=LICENSE;hb=bzip2-1.0.6": "bzip2-1.0.6",
@@ -2129,6 +2181,7 @@ var urlToLicense = map[string]string{
"tahoe-lafs.org/trac/tahoe-lafs/browser/trunk/COPYING.TGPPL.rst": "TGPPL-1.0", "tahoe-lafs.org/trac/tahoe-lafs/browser/trunk/COPYING.TGPPL.rst": "TGPPL-1.0",
"ti.arc.nasa.gov/opensource/nosa/": "NASA-1.3", "ti.arc.nasa.gov/opensource/nosa/": "NASA-1.3",
"trac.enlightenment.org/e/browser/trunk/imlib2/COPYING": "Imlib2", "trac.enlightenment.org/e/browser/trunk/imlib2/COPYING": "Imlib2",
"tunes.org/legalese/bugroff.html": "Bugroff",
"ubuntu.com/legal/font-licence": "Ubuntu-font-1.0", "ubuntu.com/legal/font-licence": "Ubuntu-font-1.0",
"unlicense.org/": "Unlicense", "unlicense.org/": "Unlicense",
"vimdoc.sourceforge.net/htmldoc/uganda.html": "Vim", "vimdoc.sourceforge.net/htmldoc/uganda.html": "Vim",
@@ -2139,6 +2192,7 @@ var urlToLicense = map[string]string{
"web.archive.org/web/20041020171434/http://www.rosenlaw.com/osl2.0.html": "OSL-2.0", "web.archive.org/web/20041020171434/http://www.rosenlaw.com/osl2.0.html": "OSL-2.0",
"web.archive.org/web/20050212003940/http://www.rosenlaw.com/osl21.htm": "OSL-2.1", "web.archive.org/web/20050212003940/http://www.rosenlaw.com/osl21.htm": "OSL-2.1",
"web.archive.org/web/20060319014854/http://info.borland.com/devsupport/interbase/opensource/IPL.html": "Interbase-1.0", "web.archive.org/web/20060319014854/http://info.borland.com/devsupport/interbase/opensource/IPL.html": "Interbase-1.0",
"web.archive.org/web/20061013013632/http://opensource.linux-mirror.org/licenses/afl-2.1.txt": "AFL-2.1",
"web.archive.org/web/20080207013128/http://www.gnome.org/fonts/": "Bitstream-Vera", "web.archive.org/web/20080207013128/http://www.gnome.org/fonts/": "Bitstream-Vera",
"web.archive.org/web/20100302225219/http://www.zimbra.com/license/zimbra-public-license-1-3.html": "Zimbra-1.3", "web.archive.org/web/20100302225219/http://www.zimbra.com/license/zimbra-public-license-1-3.html": "Zimbra-1.3",
"web.archive.org/web/20111123062036/http://research.cs.wisc.edu/condor/license.html#condor": "Condor-1.1", "web.archive.org/web/20111123062036/http://research.cs.wisc.edu/condor/license.html#condor": "Condor-1.1",
@@ -2243,11 +2297,11 @@ var urlToLicense = map[string]string{
"www.jython.org/license.html": "CNRI-Jython", "www.jython.org/license.html": "CNRI-Jython",
"www.kermitproject.org/ck90.html#source": "HPND-export-US", "www.kermitproject.org/ck90.html#source": "HPND-export-US",
"www.kernel.org/doc/man-pages/licenses.html": "Linux-man-pages-copyleft", "www.kernel.org/doc/man-pages/licenses.html": "Linux-man-pages-copyleft",
"www.latex-project.org/lppl/LPPL-1.3c.txt": "LPPL-1.3c",
"www.latex-project.org/lppl/lppl-1-0.txt": "LPPL-1.0", "www.latex-project.org/lppl/lppl-1-0.txt": "LPPL-1.0",
"www.latex-project.org/lppl/lppl-1-1.txt": "LPPL-1.1", "www.latex-project.org/lppl/lppl-1-1.txt": "LPPL-1.1",
"www.latex-project.org/lppl/lppl-1-2.txt": "LPPL-1.2", "www.latex-project.org/lppl/lppl-1-2.txt": "LPPL-1.2",
"www.latex-project.org/lppl/lppl-1-3a.txt": "LPPL-1.3a", "www.latex-project.org/lppl/lppl-1-3a.txt": "LPPL-1.3a",
"www.latex-project.org/lppl/lppl-1-3c.txt": "LPPL-1.3c",
"www.lemis.com/grog/UNIX/ancient-source-all.pdf": "Caldera", "www.lemis.com/grog/UNIX/ancient-source-all.pdf": "Caldera",
"www.libpng.org/pub/png/src/libpng-LICENSE.txt": "libpng-2.0", "www.libpng.org/pub/png/src/libpng-LICENSE.txt": "libpng-2.0",
"www.mail-archive.com/debian-bugs-rc@lists.debian.org/msg11494.html": "Symlinks", "www.mail-archive.com/debian-bugs-rc@lists.debian.org/msg11494.html": "Symlinks",
@@ -2269,16 +2323,17 @@ var urlToLicense = map[string]string{
"www.netlib.org/lapack/LICENSE.txt": "BSD-3-Clause-Open-MPI", "www.netlib.org/lapack/LICENSE.txt": "BSD-3-Clause-Open-MPI",
"www.netlib.org/minpack/disclaimer": "Minpack", "www.netlib.org/minpack/disclaimer": "Minpack",
"www.networkcultures.org/_uploads/%233notebook_telekommunist.pdf": "PPL", "www.networkcultures.org/_uploads/%233notebook_telekommunist.pdf": "PPL",
"www.oclc.org/research/activities/software/license/v2final.htm": "OCLC-2.0", "www.nist.gov/open/copyright-fair-use-and-licensing-statements-srd-data-software-and-technical-series-publications": "NIST-Software",
"www.ogc.org/ogc/software/1.0": "OGC-1.0", "www.oclc.org/research/activities/software/license/v2final.htm": "OCLC-2.0",
"www.ohwr.org/project/cernohl/wikis/Documents/CERN-OHL-version-2": "CERN-OHL-W-2.0", "www.ogc.org/ogc/software/1.0": "OGC-1.0",
"www.ohwr.org/project/licenses/wikis/cern-ohl-v1.1": "CERN-OHL-1.1", "www.ohwr.org/project/cernohl/wikis/Documents/CERN-OHL-version-2": "CERN-OHL-W-2.0",
"www.ohwr.org/project/licenses/wikis/cern-ohl-v1.2": "CERN-OHL-1.2", "www.ohwr.org/project/licenses/wikis/cern-ohl-v1.1": "CERN-OHL-1.1",
"www.open-mpi.org/community/license.php": "BSD-3-Clause-Open-MPI", "www.ohwr.org/project/licenses/wikis/cern-ohl-v1.2": "CERN-OHL-1.2",
"www.opencascade.com/content/occt-public-license": "OCCT-PL", "www.open-mpi.org/community/license.php": "BSD-3-Clause-Open-MPI",
"www.opendatacommons.org/licenses/odbl/1.0/": "ODbL-1.0", "www.opencascade.com/content/occt-public-license": "OCCT-PL",
"www.opengroup.org/testing/downloads/The_Open_Group_TSL.txt": "OGTSL", "www.opendatacommons.org/licenses/odbl/1.0/": "ODbL-1.0",
"www.openhub.net/licenses/mslpl": "MS-LPL", "www.opengroup.org/testing/downloads/The_Open_Group_TSL.txt": "OGTSL",
"www.openhub.net/licenses/mslpl": "MS-LPL",
"www.openldap.org/devel/gitweb.cgi?p=openldap.git;a=blob;f=LICENSE;hb=1cae062821881f41b73012ba816434897abf4205": "OLDAP-2.6", "www.openldap.org/devel/gitweb.cgi?p=openldap.git;a=blob;f=LICENSE;hb=1cae062821881f41b73012ba816434897abf4205": "OLDAP-2.6",
"www.openldap.org/devel/gitweb.cgi?p=openldap.git;a=blob;f=LICENSE;hb=37b4b3f6cc4bf34e1d3dec61e69914b9819d8894": "NBPL-1.0", "www.openldap.org/devel/gitweb.cgi?p=openldap.git;a=blob;f=LICENSE;hb=37b4b3f6cc4bf34e1d3dec61e69914b9819d8894": "NBPL-1.0",
"www.openldap.org/devel/gitweb.cgi?p=openldap.git;a=blob;f=LICENSE;hb=42b0383c50c299977b5893ee695cf4e486fb0dc7": "OLDAP-1.2", "www.openldap.org/devel/gitweb.cgi?p=openldap.git;a=blob;f=LICENSE;hb=42b0383c50c299977b5893ee695cf4e486fb0dc7": "OLDAP-1.2",
@@ -2301,7 +2356,7 @@ var urlToLicense = map[string]string{
"www.opensource.apple.com/license/apsl/": "APSL-2.0", "www.opensource.apple.com/license/apsl/": "APSL-2.0",
"www.opensource.apple.com/source/IOSerialFamily/IOSerialFamily-7/APPLE_LICENSE": "APSL-1.1", "www.opensource.apple.com/source/IOSerialFamily/IOSerialFamily-7/APPLE_LICENSE": "APSL-1.1",
"www.opensource.apple.com/source/tcl/tcl-5/tcl/generic/regfronts.c": "Spencer-99", "www.opensource.apple.com/source/tcl/tcl-5/tcl/generic/regfronts.c": "Spencer-99",
"www.opensource.org/licenses/EPL-2.0": "EPL-2.0", "www.opensource.org/license/EPL-2.0": "EPL-2.0",
"www.openssl.org/source/license.html": "OpenSSL", "www.openssl.org/source/license.html": "OpenSSL",
"www.osetfoundation.org/public-license": "OSET-PL-2.1", "www.osetfoundation.org/public-license": "OSET-PL-2.1",
"www.parliament.uk/site-information/copyright-parliament/open-parliament-licence/": "OPL-UK-3.0", "www.parliament.uk/site-information/copyright-parliament/open-parliament-licence/": "OPL-UK-3.0",
internal/spdxlicense/license_url_test.go
+90
View File
@@ -123,6 +123,96 @@ func TestLicenseByURL_AlternateScheme(t *testing.T) {
} }
} }
func TestLicenseByURL_OpenSourceOrgVariants(t *testing.T) {
// SPDX has changed the form of opensource.org URLs across list versions (e.g. 3.29.0 moved
// opensource.org/licenses/MIT to opensource.org/license/MIT); every form should keep resolving.
tests := []struct {
name string
url string
wantID string
}{
{name: "old plural path", url: "http://opensource.org/licenses/MIT", wantID: "MIT"},
{name: "new singular path", url: "https://opensource.org/license/MIT", wantID: "MIT"},
{name: "lowercase with trailing slash", url: "https://opensource.org/license/mit/", wantID: "MIT"},
{name: "plural path with trailing slash", url: "https://opensource.org/licenses/MIT/", wantID: "MIT"},
{name: "www prefix", url: "https://www.opensource.org/licenses/MIT", wantID: "MIT"},
{name: "uppercase host", url: "https://WWW.OpenSource.org/licenses/MIT", wantID: "MIT"},
{name: "old plural path BSD-3-Clause", url: "https://opensource.org/licenses/BSD-3-Clause", wantID: "BSD-3-Clause"},
{name: "new singular path BSD-3-Clause", url: "https://opensource.org/license/BSD-3-Clause", wantID: "BSD-3-Clause"},
{name: "lowercase BSD-3-Clause", url: "https://opensource.org/licenses/bsd-3-clause", wantID: "BSD-3-Clause"},
{name: "old plural path Apache-2.0", url: "http://opensource.org/licenses/Apache-2.0", wantID: "Apache-2.0"},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
info, found := LicenseByURL(tt.url)
if !found {
t.Fatalf("LicenseByURL(%q) found = false, want true", tt.url)
}
if info.ID != tt.wantID {
t.Errorf("LicenseByURL(%q) ID = %v, want %v", tt.url, info.ID, tt.wantID)
}
})
}
}
func TestLicenseByURL_NonOpenSourceOrgStaysExact(t *testing.T) {
// the relaxed matching is scoped to opensource.org; other hosts may have case-sensitive paths
if _, found := LicenseByURL("https://www.apache.org/licenses/license-2.0"); found {
t.Errorf("LicenseByURL() found = true for case-altered non-opensource.org URL, want false")
}
if _, found := LicenseByURL("https://opensource.org/licenses/not-a-real-license"); found {
t.Errorf("LicenseByURL() found = true for unknown opensource.org URL, want false")
}
}
func TestNormalizeOpenSourceOrgURL(t *testing.T) {
tests := []struct {
url string
want string
wantOK bool
}{
{url: "opensource.org/licenses/MIT", want: "opensource.org/license/mit", wantOK: true},
{url: "opensource.org/license/mit/", want: "opensource.org/license/mit", wantOK: true},
{url: "www.opensource.org/licenses/EPL-2.0", want: "opensource.org/license/epl-2.0", wantOK: true},
{url: "lists.opensource.org/pipermail/license-discuss", wantOK: false},
{url: "www.apache.org/licenses/LICENSE-2.0", wantOK: false},
}
for _, tt := range tests {
t.Run(tt.url, func(t *testing.T) {
got, ok := normalizeOpenSourceOrgURL(tt.url)
if ok != tt.wantOK {
t.Fatalf("normalizeOpenSourceOrgURL(%q) ok = %v, want %v", tt.url, ok, tt.wantOK)
}
if got != tt.want {
t.Errorf("normalizeOpenSourceOrgURL(%q) = %q, want %q", tt.url, got, tt.want)
}
})
}
}
func TestBuildOpenSourceOrgURLToLicense(t *testing.T) {
index := buildOpenSourceOrgURLToLicense(map[string]string{
"opensource.org/licenses/MIT": "MIT",
"opensource.org/license/mit/": "MIT",
"opensource.org/licenses/Foo": "Foo-1.0",
"opensource.org/license/foo": "Foo-2.0",
"opensource.org/license/FOO/": "Foo-1.0",
"www.apache.org/licenses/Bar": "Bar",
})
if got := index["opensource.org/license/mit"]; got != "MIT" {
t.Errorf("index[mit] = %q, want MIT", got)
}
if got, exists := index["opensource.org/license/foo"]; exists {
t.Errorf("index[foo] = %q, want no entry for URL forms mapping to different licenses", got)
}
if len(index) != 1 {
t.Errorf("len(index) = %d, want 1 (non-opensource.org URLs excluded): %v", len(index), index)
}
}
func TestStripScheme(t *testing.T) { func TestStripScheme(t *testing.T) {
tests := []struct { tests := []struct {
name string name string
internal/task/archive_tasks.go
+13 -1
View File
@@ -302,7 +302,7 @@ func (c *archiveCataloger) runSubPipeline(ctx context.Context, resolver file.Res
scratch := &sbom.SBOM{Artifacts: sbom.Artifacts{Packages: pkg.NewCollection()}} scratch := &sbom.SBOM{Artifacts: sbom.Artifacts{Packages: pkg.NewCollection()}}
scratchBuilder := sbomsync.NewBuilder(scratch) scratchBuilder := sbomsync.NewBuilder(scratch)
for _, t := range c.subPipeline { for _, t := range c.subPipeline {
err := runTaskSafely(ctx, t, resolver, scratchBuilder) err := runSubTaskSafely(ctx, t, resolver, scratchBuilder)
if err == nil { if err == nil {
continue continue
} }
@@ -317,6 +317,18 @@ func (c *archiveCataloger) runSubPipeline(ctx context.Context, resolver file.Res
return scratch return scratch
} }
// runSubTaskSafely is runTaskSafely without the task name in a panic's error, since appendUnknowns
// already prefixes it.
func runSubTaskSafely(ctx context.Context, t Task, resolver file.Resolver, builder sbomsync.Builder) (err error) {
defer func() {
if r := recover(); r != nil {
log.WithFields("task", t.Name()).Debugf("archive sub-cataloger task panic stack:\n%s", debug.Stack())
err = fmt.Errorf("panic: %v", r)
}
}()
return t.Execute(ctx, resolver, builder)
}
// archiveCandidate is one file the walk will try to open. // archiveCandidate is one file the walk will try to open.
type archiveCandidate struct { type archiveCandidate struct {
location file.Location location file.Location
internal/task/archive_tasks_test.go
+1 -1
View File
@@ -577,7 +577,7 @@ func Test_archiveCataloger_subPipelineFailureIsRecordedAsAnUnknown(t *testing.T)
reasons, ok := s.Artifacts.Unknowns[file.Coordinates{RealPath: "/outer.zip"}] reasons, ok := s.Artifacts.Unknowns[file.Coordinates{RealPath: "/outer.zip"}]
require.True(t, ok) require.True(t, ok)
require.Len(t, reasons, 1) require.Len(t, reasons, 1)
assert.Contains(t, reasons[0], "panicking-cataloger: nope") assert.Equal(t, "panicking-cataloger: panic: nope", reasons[0])
}) })
} }
internal/task/executor.go
+2 -1
View File
@@ -51,7 +51,8 @@ func runTaskSafely(ctx context.Context, t Task, resolver file.Resolver, s sbomsy
// handle individual cataloger panics // handle individual cataloger panics
defer func() { defer func() {
if e := recover(); e != nil { if e := recover(); e != nil {
err = fmt.Errorf("%v at:\n%s", e, string(debug.Stack())) log.WithFields("task", t.Name()).Debugf("task panic stack:\n%s", debug.Stack())
err = fmt.Errorf("panic in task %q: %v", t.Name(), e)
} }
}() }()
internal/task/executor_test.go
+20 -2
View File
@@ -8,8 +8,12 @@ import (
"github.com/wagoodman/go-progress" "github.com/wagoodman/go-progress"
"github.com/anchore/syft/internal/sbomsync" "github.com/anchore/syft/internal/sbomsync"
"github.com/anchore/syft/syft/artifact"
"github.com/anchore/syft/syft/event/monitor" "github.com/anchore/syft/syft/event/monitor"
"github.com/anchore/syft/syft/file" "github.com/anchore/syft/syft/file"
"github.com/anchore/syft/syft/pkg"
"github.com/anchore/syft/syft/pkg/cataloger/generic"
"github.com/anchore/syft/syft/sbom"
) )
func Test_TaskExecutor_PanicHandling(t *testing.T) { func Test_TaskExecutor_PanicHandling(t *testing.T) {
@@ -21,6 +25,20 @@ func Test_TaskExecutor_PanicHandling(t *testing.T) {
Manual: progress.NewManual(-1), Manual: progress.NewManual(-1),
}) })
require.Error(t, err) require.EqualError(t, err, `panic in task "panicking-cataloger": something bad happened`)
require.Contains(t, err.Error(), "executor_test.go") }
func Test_RunTask_GenericParserPanicBecomesUnknown(t *testing.T) {
c := generic.NewCataloger("panicking-cataloger").
WithParserByPath(func(context.Context, file.Resolver, *generic.Environment, file.LocationReadCloser) ([]pkg.Package, []artifact.Relationship, error) {
panic("boom")
}, "executor_test.go")
s := &sbom.SBOM{}
err := RunTask(context.Background(), NewPackageTask(DefaultCatalogingFactoryConfig(), c), file.NewMockResolverForPaths("executor_test.go"), sbomsync.NewBuilder(s), &monitor.TaskProgress{
Manual: progress.NewManual(-1),
})
require.NoError(t, err)
require.Len(t, s.Artifacts.Unknowns, 1)
} }
internal/unknown/coordinate_error.go
+4
View File
@@ -183,6 +183,10 @@ func visitErrors(err error, fn func(error) error) error {
// unwrap singly wrapped errors // unwrap singly wrapped errors
if e, ok := err.(interface{ Unwrap() error }); ok { if e, ok := err.(interface{ Unwrap() error }); ok {
wrapped := e.Unwrap() wrapped := e.Unwrap()
if wrapped == nil {
// nothing is wrapped (e.g. a sync.PanicError with a non-error value), so this is a leaf
return fn(err)
}
got := visitErrors(wrapped, fn) got := visitErrors(wrapped, fn)
if got == nil { if got == nil {
return nil return nil
internal/unknown/coordinate_error_test.go
+9
View File
@@ -8,6 +8,7 @@ import (
"github.com/stretchr/testify/require" "github.com/stretchr/testify/require"
"github.com/anchore/go-sync"
"github.com/anchore/syft/syft/file" "github.com/anchore/syft/syft/file"
) )
@@ -234,3 +235,11 @@ func Test_Append(t *testing.T) {
e3 = Appendf(e2, file.NewLocation("l3"), "%s", "e3") e3 = Appendf(e2, file.NewLocation("l3"), "%s", "e3")
require.Equal(t, "l1: e1\nl3: e3", e3.Error()) require.Equal(t, "l1: e1\nl3: e3", e3.Error())
} }
func Test_ExtractCoordinateErrors_keepsNilUnwrap(t *testing.T) {
// sync.PanicError unwraps to nil when the panic value is not an error
err := sync.PanicError{Value: "x"}
unknowns, remaining := ExtractCoordinateErrors(err)
require.Empty(t, unknowns)
require.Equal(t, err, remaining)
}
syft/file/cataloger/executable/cataloger.go
+11 -1
View File
@@ -7,6 +7,7 @@ import (
"debug/macho" "debug/macho"
"encoding/binary" "encoding/binary"
"fmt" "fmt"
"runtime/debug"
"sort" "sort"
"github.com/bmatcuk/doublestar/v4" "github.com/bmatcuk/doublestar/v4"
@@ -89,7 +90,7 @@ func (i *Cataloger) CatalogCtx(ctx context.Context, resolver file.Resolver) (map
return results, errs return results, errs
} }
func processExecutableLocation(loc file.Location, resolver file.Resolver) (*file.Executable, error) { func processExecutableLocation(loc file.Location, resolver file.Resolver) (exec *file.Executable, err error) {
reader, err := resolver.FileContentsByLocation(loc) reader, err := resolver.FileContentsByLocation(loc)
if err != nil { if err != nil {
log.WithFields("error", err, "path", loc.RealPath).Debug("unable to get file contents") log.WithFields("error", err, "path", loc.RealPath).Debug("unable to get file contents")
@@ -97,6 +98,15 @@ func processExecutableLocation(loc file.Location, resolver file.Resolver) (*file
} }
defer internal.CloseAndLogError(reader, loc.RealPath) defer internal.CloseAndLogError(reader, loc.RealPath)
defer func() {
if r := recover(); r != nil {
// a malformed binary must not take down the whole SBOM; the caller records this as an unknown
log.WithFields("path", loc.RealPath, "panic", r).Debug("recovered from panic while reading executable")
log.Tracef("executable panic stack:\n%s", debug.Stack())
exec, err = nil, fmt.Errorf("recovered from panic while reading executable: %v", r)
}
}()
uReader, err := unionreader.GetUnionReader(reader) uReader, err := unionreader.GetUnionReader(reader)
if err != nil { if err != nil {
log.WithFields("error", err, "path", loc.RealPath).Debug("unable to get union reader") log.WithFields("error", err, "path", loc.RealPath).Debug("unable to get union reader")
syft/file/cataloger/executable/cataloger_test.go
+40
View File
@@ -0,0 +1,40 @@
package executable
import (
"context"
"io"
"testing"
"github.com/stretchr/testify/require"
"github.com/anchore/syft/internal/unknown"
"github.com/anchore/syft/syft/file"
)
type panicReader struct{}
func (panicReader) Read([]byte) (int, error) { panic("boom") }
type panickingContentsResolver struct {
*file.MockResolver
}
func (r panickingContentsResolver) FilesByMIMEType(...string) ([]file.Location, error) {
return r.FilesByPath("cataloger_test.go")
}
func (r panickingContentsResolver) FileContentsByLocation(file.Location) (io.ReadCloser, error) {
return io.NopCloser(panicReader{}), nil
}
func TestCatalogCtx_panicBecomesUnknown(t *testing.T) {
resolver := panickingContentsResolver{file.NewMockResolverForPaths("cataloger_test.go")}
results, err := NewCataloger(DefaultConfig()).CatalogCtx(context.Background(), resolver)
require.Empty(t, results)
unknowns, remaining := unknown.ExtractCoordinateErrors(err)
require.NoError(t, remaining)
require.Len(t, unknowns, 1)
require.ErrorContains(t, unknowns[0].Reason, "recovered from panic while reading executable: boom")
}
syft/file/cataloger/executable/elf.go
+3 -23
View File
@@ -156,32 +156,12 @@ func hasElfDynFlag1(f *elf.File, flag elf.DynFlag1) bool {
} }
func hasElfDynTag(f *elf.File, tag elf.DynTag) bool { func hasElfDynTag(f *elf.File, tag elf.DynTag) bool {
// source https://github.com/golang/go/blob/9b4b3e5acca2dabe107fa2c3ed963097d78a4562/src/cmd/cgo/internal/testshared/shared_test.go#L280 vals, err := f.DynValue(tag)
ds := f.SectionByType(elf.SHT_DYNAMIC)
if ds == nil {
return false
}
d, err := ds.Data()
if err != nil { if err != nil {
log.WithFields("error", err).Trace("unable to read dynamic section from elf file")
return false return false
} }
return len(vals) > 0
for len(d) > 0 {
var t elf.DynTag
switch f.Class {
case elf.ELFCLASS32:
t = elf.DynTag(f.ByteOrder.Uint32(d[0:4]))
d = d[8:]
case elf.ELFCLASS64:
t = elf.DynTag(f.ByteOrder.Uint64(d[0:8]))
d = d[16:]
}
if t == tag {
return true
}
}
return false
} }
func isELFPIE(f *elf.File) bool { func isELFPIE(f *elf.File) bool {
syft/file/cataloger/executable/elf_test.go
+88
View File
@@ -1,7 +1,9 @@
package executable package executable
import ( import (
"bytes"
"debug/elf" "debug/elf"
"encoding/binary"
"os" "os"
"path/filepath" "path/filepath"
"testing" "testing"
@@ -226,3 +228,89 @@ func Test_elfHasExports(t *testing.T) {
}) })
} }
} }
func Test_hasElfDynTag(t *testing.T) {
dynEntry := func(class elf.Class, tag elf.DynTag) []byte {
if class == elf.ELFCLASS32 {
return binary.LittleEndian.AppendUint32(binary.LittleEndian.AppendUint32(nil, uint32(tag)), 0)
}
return binary.LittleEndian.AppendUint64(binary.LittleEndian.AppendUint64(nil, uint64(tag)), 0)
}
tests := []struct {
name string
class elf.Class
dynamic []byte
want bool
}{
{
name: "32-bit tag present",
class: elf.ELFCLASS32,
dynamic: dynEntry(elf.ELFCLASS32, elf.DT_BIND_NOW),
want: true,
},
{
name: "64-bit tag present",
class: elf.ELFCLASS64,
dynamic: dynEntry(elf.ELFCLASS64, elf.DT_BIND_NOW),
want: true,
},
{
name: "64-bit tag absent",
class: elf.ELFCLASS64,
dynamic: dynEntry(elf.ELFCLASS64, elf.DT_NULL),
want: false,
},
{
name: "32-bit truncated dynamic section",
class: elf.ELFCLASS32,
dynamic: dynEntry(elf.ELFCLASS32, elf.DT_BIND_NOW)[:4],
want: false,
},
{
name: "64-bit truncated dynamic section",
class: elf.ELFCLASS64,
dynamic: dynEntry(elf.ELFCLASS64, elf.DT_BIND_NOW)[:12],
want: false,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
f, err := elf.NewFile(bytes.NewReader(elfWithDynamicSection(t, tt.class, tt.dynamic)))
require.NoError(t, err)
assert.Equal(t, tt.want, hasElfDynTag(f, elf.DT_BIND_NOW))
})
}
}
// elfWithDynamicSection builds a little-endian ELF holding only a null section and a SHT_DYNAMIC section.
func elfWithDynamicSection(t *testing.T, class elf.Class, dynamic []byte) []byte {
t.Helper()
ident := [elf.EI_NIDENT]byte{0x7f, 'E', 'L', 'F', byte(class), byte(elf.ELFDATA2LSB), byte(elf.EV_CURRENT)}
var buf bytes.Buffer
write := func(v any) {
require.NoError(t, binary.Write(&buf, binary.LittleEndian, v))
}
if class == elf.ELFCLASS32 {
ehsize, shentsize := binary.Size(elf.Header32{}), binary.Size(elf.Section32{})
dataOff := ehsize + 2*shentsize
write(elf.Header32{
Ident: ident, Type: uint16(elf.ET_DYN), Machine: uint16(elf.EM_386), Version: uint32(elf.EV_CURRENT),
Shoff: uint32(ehsize), Ehsize: uint16(ehsize), Shentsize: uint16(shentsize), Shnum: 2,
})
write(elf.Section32{})
write(elf.Section32{Type: uint32(elf.SHT_DYNAMIC), Off: uint32(dataOff), Size: uint32(len(dynamic)), Addralign: 1})
} else {
ehsize, shentsize := binary.Size(elf.Header64{}), binary.Size(elf.Section64{})
dataOff := ehsize + 2*shentsize
write(elf.Header64{
Ident: ident, Type: uint16(elf.ET_DYN), Machine: uint16(elf.EM_X86_64), Version: uint32(elf.EV_CURRENT),
Shoff: uint64(ehsize), Ehsize: uint16(ehsize), Shentsize: uint16(shentsize), Shnum: 2,
})
write(elf.Section64{})
write(elf.Section64{Type: uint32(elf.SHT_DYNAMIC), Off: uint64(dataOff), Size: uint64(len(dynamic)), Addralign: 1})
}
buf.Write(dynamic)
return buf.Bytes()
}
syft/file/cataloger/filedigest/cataloger.go
+12 -2
View File
@@ -5,6 +5,7 @@ import (
"crypto" "crypto"
"errors" "errors"
"fmt" "fmt"
"runtime/debug"
"github.com/dustin/go-humanize" "github.com/dustin/go-humanize"
@@ -87,7 +88,7 @@ func (i *Cataloger) Catalog(ctx context.Context, resolver file.Resolver, coordin
return results, err return results, err
} }
func (i *Cataloger) catalogLocation(ctx context.Context, resolver file.Resolver, location file.Location) ([]file.Digest, error) { func (i *Cataloger) catalogLocation(ctx context.Context, resolver file.Resolver, location file.Location) (digests []file.Digest, err error) {
meta, err := resolver.FileMetadataByLocation(location) meta, err := resolver.FileMetadataByLocation(location)
if err != nil { if err != nil {
return nil, err return nil, err
@@ -104,7 +105,16 @@ func (i *Cataloger) catalogLocation(ctx context.Context, resolver file.Resolver,
} }
defer internal.CloseAndLogError(contentReader, location.AccessPath) defer internal.CloseAndLogError(contentReader, location.AccessPath)
digests, err := intFile.NewDigestsFromFile(ctx, contentReader, i.hashes) defer func() {
if r := recover(); r != nil {
// the caller records this as an unknown for the location instead of failing the whole SBOM
log.WithFields("path", location.RealPath, "panic", r).Debug("recovered from panic while digesting file")
log.Tracef("file digest panic stack:\n%s", debug.Stack())
digests, err = nil, fmt.Errorf("recovered from panic while digesting file: %v", r)
}
}()
digests, err = intFile.NewDigestsFromFile(ctx, contentReader, i.hashes)
if err != nil { if err != nil {
return nil, internal.ErrPath{Context: "digests-cataloger", Path: location.RealPath, Err: err} return nil, internal.ErrPath{Context: "digests-cataloger", Path: location.RealPath, Err: err}
} }
syft/file/cataloger/filedigest/cataloger_test.go
+25
View File
@@ -15,6 +15,7 @@ import (
stereoscopeFile "github.com/anchore/stereoscope/pkg/file" stereoscopeFile "github.com/anchore/stereoscope/pkg/file"
"github.com/anchore/stereoscope/pkg/imagetest" "github.com/anchore/stereoscope/pkg/imagetest"
intFile "github.com/anchore/syft/internal/file" intFile "github.com/anchore/syft/internal/file"
"github.com/anchore/syft/internal/unknown"
"github.com/anchore/syft/syft/file" "github.com/anchore/syft/syft/file"
"github.com/anchore/syft/syft/source" "github.com/anchore/syft/syft/source"
"github.com/anchore/syft/syft/source/directorysource" "github.com/anchore/syft/syft/source/directorysource"
@@ -202,3 +203,27 @@ func TestFileDigestCataloger_GivenCoordinates(t *testing.T) {
} }
} }
type panicReader struct{}
func (panicReader) Read([]byte) (int, error) { panic("boom") }
type panickingContentsResolver struct {
*file.MockResolver
}
func (r panickingContentsResolver) FileContentsByLocation(file.Location) (io.ReadCloser, error) {
return io.NopCloser(panicReader{}), nil
}
func TestDigestsCataloger_panicBecomesUnknown(t *testing.T) {
resolver := panickingContentsResolver{file.NewMockResolverForPaths("cataloger_test.go")}
results, err := NewCataloger([]crypto.Hash{crypto.SHA256}).Catalog(context.Background(), resolver, file.NewLocation("cataloger_test.go").Coordinates)
require.Empty(t, results)
unknowns, remaining := unknown.ExtractCoordinateErrors(err)
require.NoError(t, remaining)
require.Len(t, unknowns, 1)
require.ErrorContains(t, unknowns[0].Reason, "recovered from panic while digesting file: boom")
}
syft/format/common/spdxhelpers/to_format_model.go
+30 -10
View File
@@ -2,6 +2,7 @@
package spdxhelpers package spdxhelpers
import ( import (
"cmp"
"crypto/sha1" "crypto/sha1"
"fmt" "fmt"
"path" "path"
@@ -180,7 +181,8 @@ func toRootRelationships(rootPackage *spdx.Package, packages []*spdx.Package) (o
func toRootPackage(s source.Description) *spdx.Package { func toRootPackage(s source.Description) *spdx.Package {
var prefix string var prefix string
name := s.Name // the SPDX package name is mandatory, and a source decoded from another SBOM may not carry one
name := cmp.Or(s.Name, imageRepoName(s.Metadata), s.ID, helpers.DocumentName(s))
version := s.Version version := s.Version
var purl *packageurl.PackageURL var purl *packageurl.PackageURL
@@ -211,7 +213,7 @@ func toRootPackage(s source.Description) *spdx.Package {
checksums = append(checksums, *c) checksums = append(checksums, *c)
purl = &packageurl.PackageURL{ purl = &packageurl.PackageURL{
Type: "oci", Type: "oci",
Name: s.Name, Name: name,
Version: m.ManifestDigest, Version: m.ManifestDigest,
Qualifiers: qualifiers, Qualifiers: qualifiers,
} }
@@ -241,7 +243,7 @@ func toRootPackage(s source.Description) *spdx.Package {
checksums = append(checksums, *c) checksums = append(checksums, *c)
purl = &packageurl.PackageURL{ purl = &packageurl.PackageURL{
Type: "oci", Type: "oci",
Name: s.Name, Name: name,
Version: m.ManifestDigest, Version: m.ManifestDigest,
Qualifiers: qualifiers, Qualifiers: qualifiers,
} }
@@ -276,10 +278,6 @@ func toRootPackage(s source.Description) *spdx.Package {
default: default:
prefix = prefixUnknown prefix = prefixUnknown
purpose = spdxPrimaryPurposeOther purpose = spdxPrimaryPurposeOther
if name == "" {
name = s.ID
}
} }
p := &spdx.Package{ p := &spdx.Package{
@@ -309,6 +307,25 @@ func toRootPackage(s source.Description) *spdx.Package {
return p return p
} }
// imageRepoName returns the repository name from an image or model user input, the same way the image source names
// itself, so a source decoded from another SBOM (which may only carry the user input) gets a name without the tag.
func imageRepoName(metadata any) string {
var userInput string
switch m := metadata.(type) {
case source.ImageMetadata:
userInput = m.UserInput
case source.OCIModelMetadata:
userInput = m.UserInput
default:
return ""
}
ref, _ := reference.Parse(userInput)
if named, ok := ref.(reference.Named); ok {
return named.Name()
}
return ""
}
func toSPDXID(identifiable artifact.Identifiable) spdx.ElementID { func toSPDXID(identifiable artifact.Identifiable) spdx.ElementID {
id := string(identifiable.ID()) id := string(identifiable.ID())
if after, ok := strings.CutPrefix(id, "SPDXRef-"); ok { if after, ok := strings.CutPrefix(id, "SPDXRef-"); ok {
@@ -582,10 +599,13 @@ func toPackageChecksums(p pkg.Package) ([]spdx.Checksum, bool) {
}) })
case pkg.OpamPackage: case pkg.OpamPackage:
for _, checksum := range meta.Checksums { for _, checksum := range meta.Checksums {
parts := strings.Split(checksum, "=") algorithm, value, ok := strings.Cut(checksum, "=")
if !ok {
continue
}
checksums = append(checksums, spdx.Checksum{ checksums = append(checksums, spdx.Checksum{
Algorithm: spdx.ChecksumAlgorithm(strings.ToUpper(parts[0])), Algorithm: spdx.ChecksumAlgorithm(strings.ToUpper(algorithm)),
Value: parts[1], Value: value,
}) })
} }
} }
syft/format/common/spdxhelpers/to_format_model_test.go
+60
View File
@@ -516,6 +516,20 @@ func Test_toPackageChecksums(t *testing.T) {
}, },
filesAnalyzed: false, filesAnalyzed: false,
}, },
{
name: "Opam Package with a checksum missing its algorithm separator",
pkg: pkg.Package{
Name: "test",
Version: "1.0.0",
Metadata: pkg.OpamPackage{
Checksums: []string{"f5f1c0b4", "md5=abc"},
},
},
expected: []spdx.Checksum{
{Algorithm: "MD5", Value: "abc"},
},
filesAnalyzed: false,
},
{ {
name: "Package with no metadata type", name: "Package with no metadata type",
pkg: pkg.Package{ pkg: pkg.Package{
@@ -1174,3 +1188,49 @@ func Test_toFiles_nestedFilesAreNamedByTheirChainAndRoundTrip(t *testing.T) {
require.NotNil(t, b, "names: %v", byName) require.NotNil(t, b, "names: %v", byName)
assert.Equal(t, inB, toSyftCoordinates(b), "the entry path's escaping is undone on decode") assert.Equal(t, inB, toSyftCoordinates(b), "the entry path's escaping is undone on decode")
} }
func Test_toRootPackage_name(t *testing.T) {
tests := []struct {
name string
src source.Description
wantName string
wantPurl string
}{
{
name: "source name wins",
src: source.Description{Name: "named", ID: "id", Metadata: source.ImageMetadata{UserInput: "alpine:3.18"}},
wantName: "named",
},
{
// a CycloneDX container component decodes to an image source with only the user input set
name: "image user input without a source name",
src: source.Description{Metadata: source.ImageMetadata{
UserInput: "alpine:3.18",
ManifestDigest: "sha256:0e4b2e0d6b2f3c2b1d3bb3a1c8f1c4d9e6a9b1e8c6f4d2b3a1f0e9d8c7b6a5f4",
}},
wantName: "alpine",
wantPurl: "pkg:oci/alpine@sha256%3A0e4b2e0d6b2f3c2b1d3bb3a1c8f1c4d9e6a9b1e8c6f4d2b3a1f0e9d8c7b6a5f4?arch=&tag=3.18",
},
{
name: "id without a source name",
src: source.Description{ID: "some-id"},
wantName: "some-id",
},
{
name: "nothing to name it by",
src: source.Description{Metadata: source.DirectoryMetadata{}},
wantName: "unknown",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
p := toRootPackage(tt.src)
assert.Equal(t, tt.wantName, p.PackageName)
var purl string
if len(p.PackageExternalReferences) > 0 {
purl = p.PackageExternalReferences[0].Locator
}
assert.Equal(t, tt.wantPurl, purl)
})
}
}
syft/format/common/spdxhelpers/to_syft_model.go
+62 -2
View File
@@ -35,6 +35,8 @@ func ToSyftModel(doc *spdx.Document) (*sbom.SBOM, error) {
return nil, errors.New("cannot convert SPDX document to Syft model because document is nil") return nil, errors.New("cannot convert SPDX document to Syft model because document is nil")
} }
removeNilElements(doc)
spdxIDMap := make(map[string]any) spdxIDMap := make(map[string]any)
s := &sbom.SBOM{ s := &sbom.SBOM{
@@ -56,6 +58,28 @@ func ToSyftModel(doc *spdx.Document) (*sbom.SBOM, error) {
return s, nil return s, nil
} }
// removeNilElements drops null entries from the document's lists. A JSON null (or an element the SPDX version
// converter cannot map) decodes to a nil pointer, and the rest of the conversion dereferences these elements.
func removeNilElements(doc *spdx.Document) {
doc.Packages = withoutNil(doc.Packages)
for _, p := range doc.Packages {
p.Files = withoutNil(p.Files)
p.PackageExternalReferences = withoutNil(p.PackageExternalReferences)
}
doc.Files = withoutNil(doc.Files)
doc.Relationships = withoutNil(doc.Relationships)
}
func withoutNil[T any](items []*T) []*T {
var out []*T
for _, item := range items {
if item != nil {
out = append(out, item)
}
}
return out
}
func isDirectory(name string) bool { func isDirectory(name string) bool {
if name == "." || name == ".." || strings.HasSuffix(name, "/") || !strings.Contains(path.Base(name), ".") { if name == "." || name == ".." || strings.HasSuffix(name, "/") || !strings.Contains(path.Base(name), ".") {
return true return true
@@ -124,12 +148,24 @@ func extractSource(spdxIDMap map[string]any, doc *spdx.Document) source.Descript
} }
p := rootPackages[0] p := rootPackages[0]
rootType := syftRootType(p.PackageSPDXIdentifier)
switch p.PrimaryPackagePurpose { purpose := p.PrimaryPackagePurpose
if purpose == "" {
// SPDX 2.1 and 2.2 have no primary package purpose, so fall back to the source type syft puts in the root's ID
purpose = syftRootPurpose(rootType)
}
switch purpose {
case spdxPrimaryPurposeContainer: case spdxPrimaryPurposeContainer:
src = containerSource(p) src = containerSource(p)
case spdxPrimaryPurposeFile: case spdxPrimaryPurposeFile:
src = fileSource(p) src = fileSource(p)
case spdxPrimaryPurposeOther:
// syft writes a root package even for a source it knows nothing about, which is not a package in the SBOM
if rootType != prefixUnknown {
return src
}
default: default:
return src return src
} }
@@ -142,6 +178,30 @@ func extractSource(spdxIDMap map[string]any, doc *spdx.Document) source.Descript
return src return src
} }
// syftRootType returns the source type syft encodes in a root package ID ("DocumentRoot-<type>-<name>"), or "" for
// any other ID.
func syftRootType(id spdx.ElementID) string {
rest, ok := strings.CutPrefix(string(id), "DocumentRoot-")
if !ok {
return ""
}
rootType, _, _ := strings.Cut(rest, "-")
return rootType
}
// syftRootPurpose returns the primary package purpose syft writes for a root package of the given source type.
func syftRootPurpose(rootType string) string {
switch rootType {
case prefixImage, prefixOCIModel, prefixSnap:
return spdxPrimaryPurposeContainer
case prefixDirectory, prefixFile:
return spdxPrimaryPurposeFile
case prefixUnknown:
return spdxPrimaryPurposeOther
}
return ""
}
func containerSource(p *spdx.Package) source.Description { func containerSource(p *spdx.Package) source.Description {
id := string(p.PackageSPDXIdentifier) id := string(p.PackageSPDXIdentifier)
@@ -202,7 +262,7 @@ func fileSource(p *spdx.Package) source.Description {
} }
supplier := "" supplier := ""
if p.PackageSupplier.Supplier != helpers.NOASSERTION { if p.PackageSupplier != nil && p.PackageSupplier.Supplier != helpers.NOASSERTION {
supplier = p.PackageSupplier.Supplier supplier = p.PackageSupplier.Supplier
} }
syft/format/cpes/decoder.go
+5 -1
View File
@@ -7,6 +7,7 @@ import (
"io" "io"
"strings" "strings"
syftinternal "github.com/anchore/syft/internal"
"github.com/anchore/syft/internal/log" "github.com/anchore/syft/internal/log"
"github.com/anchore/syft/syft/cpe" "github.com/anchore/syft/syft/cpe"
"github.com/anchore/syft/syft/format/internal" "github.com/anchore/syft/syft/format/internal"
@@ -61,7 +62,7 @@ func toSyftModel(r io.Reader) (*sbom.SBOM, error) {
var errs []error var errs []error
pkgs := pkg.NewCollection() pkgs := pkg.NewCollection()
scanner := bufio.NewScanner(r) scanner := syftinternal.NewLineScanner(r)
for scanner.Scan() { for scanner.Scan() {
line := strings.TrimSpace(scanner.Text()) line := strings.TrimSpace(scanner.Text())
@@ -86,6 +87,9 @@ func toSyftModel(r io.Reader) (*sbom.SBOM, error) {
p.SetID() p.SetID()
pkgs.Add(p) pkgs.Add(p)
} }
if err := scanner.Err(); err != nil {
errs = append(errs, err)
}
return &sbom.SBOM{ return &sbom.SBOM{
Artifacts: sbom.Artifacts{ Artifacts: sbom.Artifacts{
syft/format/internal/spdxutil/helpers/document_name.go
+10 -5
View File
@@ -9,16 +9,21 @@ func DocumentName(src source.Description) string {
return src.Name return src.Name
} }
var name string
switch metadata := src.Metadata.(type) { switch metadata := src.Metadata.(type) {
case source.ImageMetadata: case source.ImageMetadata:
return metadata.UserInput name = metadata.UserInput
case source.OCIModelMetadata: case source.OCIModelMetadata:
return metadata.UserInput name = metadata.UserInput
case source.DirectoryMetadata: case source.DirectoryMetadata:
return metadata.Path name = metadata.Path
case source.FileMetadata: case source.FileMetadata:
return metadata.Path name = metadata.Path
default: }
// the SPDX document name is mandatory, and a source read from another SBOM may not have one
if name == "" {
return "unknown" return "unknown"
} }
return name
} }
syft/format/internal/spdxutil/helpers/document_name_test.go
+13 -3
View File
@@ -1,8 +1,6 @@
package helpers package helpers
import ( import (
"fmt"
"strings"
"testing" "testing"
"github.com/stretchr/testify/assert" "github.com/stretchr/testify/assert"
@@ -73,11 +71,23 @@ func Test_DocumentName(t *testing.T) {
}, },
expected: "some/name", expected: "some/name",
}, },
{
name: "no name or metadata",
srcMetadata: source.Description{},
expected: "unknown",
},
{
name: "metadata without a name",
srcMetadata: source.Description{
Metadata: source.ImageMetadata{},
},
expected: "unknown",
},
} }
for _, test := range tests { for _, test := range tests {
t.Run(test.name, func(t *testing.T) { t.Run(test.name, func(t *testing.T) {
actual := DocumentName(test.srcMetadata) actual := DocumentName(test.srcMetadata)
assert.True(t, strings.HasPrefix(actual, test.expected), fmt.Sprintf("actual name %q", actual)) assert.Equal(t, test.expected, actual)
// track each scheme tested (passed or not) // track each scheme tested (passed or not)
tracker.Tested(t, test.srcMetadata.Metadata) tracker.Tested(t, test.srcMetadata.Metadata)
syft/format/purls/decoder.go
+5 -1
View File
@@ -8,6 +8,7 @@ import (
"strings" "strings"
"github.com/anchore/packageurl-go" "github.com/anchore/packageurl-go"
syftinternal "github.com/anchore/syft/internal"
"github.com/anchore/syft/internal/log" "github.com/anchore/syft/internal/log"
"github.com/anchore/syft/syft/format/internal" "github.com/anchore/syft/syft/format/internal"
"github.com/anchore/syft/syft/pkg" "github.com/anchore/syft/syft/pkg"
@@ -61,7 +62,7 @@ func toSyftModel(r io.Reader) (*sbom.SBOM, error) {
var errs []error var errs []error
pkgs := pkg.NewCollection() pkgs := pkg.NewCollection()
scanner := bufio.NewScanner(r) scanner := syftinternal.NewLineScanner(r)
for scanner.Scan() { for scanner.Scan() {
line := strings.TrimSpace(scanner.Text()) line := strings.TrimSpace(scanner.Text())
@@ -84,6 +85,9 @@ func toSyftModel(r io.Reader) (*sbom.SBOM, error) {
p.SetID() p.SetID()
pkgs.Add(p) pkgs.Add(p)
} }
if err := scanner.Err(); err != nil {
errs = append(errs, err)
}
return &sbom.SBOM{ return &sbom.SBOM{
Artifacts: sbom.Artifacts{ Artifacts: sbom.Artifacts{
syft/format/spdxjson/decoder_test.go
+35
View File
@@ -5,6 +5,7 @@ import (
"io" "io"
"os" "os"
"path/filepath" "path/filepath"
"strings"
"testing" "testing"
"github.com/stretchr/testify/assert" "github.com/stretchr/testify/assert"
@@ -183,3 +184,37 @@ func TestDecoder_Identify(t *testing.T) {
}) })
} }
} }
func TestDecoder_DecodeNullElements(t *testing.T) {
// a JSON null in these lists (or an SPDX 2.1 package the version converter cannot map) decodes to a nil pointer.
// a null entry directly in "packages" is not covered: for SPDX 2.2 and 2.3 it panics inside spdx/tools-golang
for _, doc := range []string{
`{"spdxVersion":"SPDX-2.1","packages":[{}]}`,
`{"spdxVersion":"SPDX-2.3","files":[null]}`,
`{"spdxVersion":"SPDX-2.3","packages":[{"SPDXID":"SPDXRef-a","name":"a","files":[null]}]}`,
`{"spdxVersion":"SPDX-2.3","packages":[{"SPDXID":"SPDXRef-a","name":"a","externalRefs":[null]}]}`,
} {
t.Run(doc, func(t *testing.T) {
assert.NotPanics(t, func() {
_, _, _, _ = NewFormatDecoder().Decode(strings.NewReader(doc))
})
})
}
}
func TestDecoder_DecodeFileRootWithoutSupplier(t *testing.T) {
// the package supplier is optional in SPDX, including on the package the document describes
f, err := os.Open("testdata/spdx/file-root-without-supplier.spdx.json")
require.NoError(t, err)
defer f.Close()
s, _, _, err := NewFormatDecoder().Decode(f)
require.NoError(t, err)
assert.Equal(t, "app.tar.gz", s.Source.Name)
var names []string
for _, p := range s.Artifacts.Packages.Sorted() {
names = append(names, p.Name)
}
assert.Equal(t, []string{"requests"}, names)
}
syft/format/spdxjson/encoder_test.go
+2 -23
View File
@@ -236,23 +236,6 @@ func TestSupportedVersions(t *testing.T) {
subject := testutil.DirectoryInput(t, t.TempDir()) subject := testutil.DirectoryInput(t, t.TempDir())
dec := NewFormatDecoder() dec := NewFormatDecoder()
relationshipOffsetPerVersion := map[string]int{
// the package representing the source gets a relationship from the source package to all other packages found
// these relationships cannot be removed until the primaryPackagePurpose info is available in 2.3
"2.1": 2,
"2.2": 2,
// the source-to-package relationships can be removed since the primaryPackagePurpose info is available in 2.3
"2.3": 0,
"3.0": 0,
}
pkgCountOffsetPerVersion := map[string]int{
"2.1": 1, // the source is mapped as a package, but cannot distinguish it since the primaryPackagePurpose info is not available until 2.3
"2.2": 1, // the source is mapped as a package, but cannot distinguish it since the primaryPackagePurpose info is not available until 2.3
"2.3": 0, // the source package can be removed since the primaryPackagePurpose info is available
"3.0": 0,
}
for _, enc := range encs { for _, enc := range encs {
t.Run(enc.Version(), func(t *testing.T) { t.Run(enc.Version(), func(t *testing.T) {
require.Contains(t, versions, enc.Version()) require.Contains(t, versions, enc.Version())
@@ -274,13 +257,9 @@ func TestSupportedVersions(t *testing.T) {
require.NotEmpty(t, s.Artifacts.Packages.PackageCount()) require.NotEmpty(t, s.Artifacts.Packages.PackageCount())
offset := relationshipOffsetPerVersion[enc.Version()] assert.Equal(t, len(subject.Relationships), len(s.Relationships), "mismatched relationship count")
assert.Equal(t, len(subject.Relationships)+offset, len(s.Relationships), "mismatched relationship count") if !assert.Equal(t, subject.Artifacts.Packages.PackageCount(), s.Artifacts.Packages.PackageCount(), "mismatched package count") {
offset = pkgCountOffsetPerVersion[enc.Version()]
if !assert.Equal(t, subject.Artifacts.Packages.PackageCount()+offset, s.Artifacts.Packages.PackageCount(), "mismatched package count") {
t.Logf("expected: %d", subject.Artifacts.Packages.PackageCount()) t.Logf("expected: %d", subject.Artifacts.Packages.PackageCount())
for _, p := range subject.Artifacts.Packages.Sorted() { for _, p := range subject.Artifacts.Packages.Sorted() {
t.Logf(" - %s", p.String()) t.Logf(" - %s", p.String())
syft/format/spdxjson/testdata/snapshot/TestSPDX22JSONRequredProperties.golden
Vendored
+5 -5
View File
@@ -47,13 +47,13 @@
"copyrightText": "NOASSERTION" "copyrightText": "NOASSERTION"
}, },
{ {
"SPDXID": "SPDXRef-DocumentRoot-Unknown-", "SPDXID": "SPDXRef-DocumentRoot-Unknown-unknown",
"copyrightText": "NOASSERTION", "copyrightText": "NOASSERTION",
"downloadLocation": "NOASSERTION", "downloadLocation": "NOASSERTION",
"filesAnalyzed": false, "filesAnalyzed": false,
"licenseConcluded": "NOASSERTION", "licenseConcluded": "NOASSERTION",
"licenseDeclared": "NOASSERTION", "licenseDeclared": "NOASSERTION",
"name": "", "name": "unknown",
"supplier": "NOASSERTION" "supplier": "NOASSERTION"
} }
], ],
@@ -82,18 +82,18 @@
"relationshipType": "CONTAINS" "relationshipType": "CONTAINS"
}, },
{ {
"spdxElementId": "SPDXRef-DocumentRoot-Unknown-", "spdxElementId": "SPDXRef-DocumentRoot-Unknown-unknown",
"relatedSpdxElement": "SPDXRef-Package-files-analyzed-false-7d37ba9d2f7c574b", "relatedSpdxElement": "SPDXRef-Package-files-analyzed-false-7d37ba9d2f7c574b",
"relationshipType": "CONTAINS" "relationshipType": "CONTAINS"
}, },
{ {
"spdxElementId": "SPDXRef-DocumentRoot-Unknown-", "spdxElementId": "SPDXRef-DocumentRoot-Unknown-unknown",
"relatedSpdxElement": "SPDXRef-Package-files-analyzed-true-035066c2086b8bb4", "relatedSpdxElement": "SPDXRef-Package-files-analyzed-true-035066c2086b8bb4",
"relationshipType": "CONTAINS" "relationshipType": "CONTAINS"
}, },
{ {
"spdxElementId": "SPDXRef-DOCUMENT", "spdxElementId": "SPDXRef-DOCUMENT",
"relatedSpdxElement": "SPDXRef-DocumentRoot-Unknown-", "relatedSpdxElement": "SPDXRef-DocumentRoot-Unknown-unknown",
"relationshipType": "DESCRIBES" "relationshipType": "DESCRIBES"
} }
] ]
syft/format/spdxjson/testdata/spdx/file-root-without-supplier.spdx.json
Vendored
+29
View File
@@ -0,0 +1,29 @@
{
"spdxVersion": "SPDX-2.3",
"dataLicense": "CC0-1.0",
"SPDXID": "SPDXRef-DOCUMENT",
"name": "app.tar.gz",
"documentNamespace": "https://example.com/spdx/app-1.0",
"creationInfo": { "created": "2026-09-24T00:00:00Z", "creators": ["Tool: example-1.0"] },
"packages": [
{
"SPDXID": "SPDXRef-app",
"name": "app.tar.gz",
"downloadLocation": "NOASSERTION",
"filesAnalyzed": false,
"primaryPackagePurpose": "FILE"
},
{
"SPDXID": "SPDXRef-requests",
"name": "requests",
"versionInfo": "2.31.0",
"downloadLocation": "NOASSERTION",
"filesAnalyzed": false,
"externalRefs": [{ "referenceCategory": "PACKAGE-MANAGER", "referenceType": "purl", "referenceLocator": "pkg:pypi/requests@2.31.0" }]
}
],
"relationships": [
{ "spdxElementId": "SPDXRef-DOCUMENT", "relationshipType": "DESCRIBES", "relatedSpdxElement": "SPDXRef-app" },
{ "spdxElementId": "SPDXRef-app", "relationshipType": "CONTAINS", "relatedSpdxElement": "SPDXRef-requests" }
]
}
syft/format/spdxtagvalue/encoder_test.go
+39 -21
View File
@@ -3,6 +3,7 @@ package spdxtagvalue
import ( import (
"bytes" "bytes"
"flag" "flag"
"fmt"
"testing" "testing"
"github.com/stretchr/testify/assert" "github.com/stretchr/testify/assert"
@@ -143,21 +144,6 @@ func TestSupportedVersions(t *testing.T) {
subject := testutil.DirectoryInput(t, t.TempDir()) subject := testutil.DirectoryInput(t, t.TempDir())
dec := NewFormatDecoder() dec := NewFormatDecoder()
relationshipOffsetPerVersion := map[string]int{
// the package representing the source gets a relationship from the source package to all other packages found
// these relationships cannot be removed until the primaryPackagePurpose info is available in 2.3
"2.1": 2,
"2.2": 2,
// the source-to-package relationships can be removed since the primaryPackagePurpose info is available in 2.3
"2.3": 0,
}
pkgCountOffsetPerVersion := map[string]int{
"2.1": 1, // the source is mapped as a package, but cannot distinguish it since the primaryPackagePurpose info is not available until 2.3
"2.2": 1, // the source is mapped as a package, but cannot distinguish it since the primaryPackagePurpose info is not available until 2.3
"2.3": 0, // the source package can be removed since the primaryPackagePurpose info is available
}
for _, enc := range encs { for _, enc := range encs {
t.Run(enc.Version(), func(t *testing.T) { t.Run(enc.Version(), func(t *testing.T) {
require.Contains(t, versions, enc.Version()) require.Contains(t, versions, enc.Version())
@@ -178,13 +164,9 @@ func TestSupportedVersions(t *testing.T) {
require.NotEmpty(t, s.Artifacts.Packages.PackageCount()) require.NotEmpty(t, s.Artifacts.Packages.PackageCount())
offset := relationshipOffsetPerVersion[enc.Version()] assert.Equal(t, len(subject.Relationships), len(s.Relationships), "mismatched relationship count")
assert.Equal(t, len(subject.Relationships)+offset, len(s.Relationships), "mismatched relationship count") if !assert.Equal(t, subject.Artifacts.Packages.PackageCount(), s.Artifacts.Packages.PackageCount(), "mismatched package count") {
offset = pkgCountOffsetPerVersion[enc.Version()]
if !assert.Equal(t, subject.Artifacts.Packages.PackageCount()+offset, s.Artifacts.Packages.PackageCount(), "mismatched package count") {
t.Logf("expected: %d", subject.Artifacts.Packages.PackageCount()) t.Logf("expected: %d", subject.Artifacts.Packages.PackageCount())
for _, p := range subject.Artifacts.Packages.Sorted() { for _, p := range subject.Artifacts.Packages.Sorted() {
t.Logf(" - %s", p.String()) t.Logf(" - %s", p.String())
@@ -209,3 +191,39 @@ func defaultFormatEncoders() []sbom.FormatEncoder {
} }
return encs return encs
} }
func TestEncodeDecodeSourceWithoutName(t *testing.T) {
// a source read from another SBOM (a purl list, a CycloneDX document without metadata.component) may have no name,
// but the SPDX package name is mandatory and the tag-value decoder needs it to start the root package section
p := pkg.Package{Name: "left-pad", Version: "1.3.0", Type: pkg.NpmPkg}
p.SetID()
for _, src := range []source.Description{
{},
{Metadata: source.FileMetadata{}},
{Metadata: source.DirectoryMetadata{}},
{Metadata: source.ImageMetadata{}},
{Metadata: source.OCIModelMetadata{}},
{Metadata: source.SnapMetadata{}},
} {
for _, enc := range defaultFormatEncoders() {
t.Run(fmt.Sprintf("%T/%s", src.Metadata, enc.Version()), func(t *testing.T) {
var buf bytes.Buffer
require.NoError(t, enc.Encode(&buf, sbom.SBOM{
Source: src,
Artifacts: sbom.Artifacts{Packages: pkg.NewCollection(p)},
}))
s, _, _, err := NewFormatDecoder().Decode(bytes.NewReader(buf.Bytes()))
require.NoError(t, err, "encoded document:\n%s", buf.String())
var names []string
for _, decoded := range s.Artifacts.Packages.Sorted() {
names = append(names, decoded.Name)
}
// the root package describes the source, so it must not come back as a package of the SBOM
assert.Equal(t, []string{"left-pad"}, names)
})
}
}
}
syft/format/untyped_metadata_test.go
+94
View File
@@ -0,0 +1,94 @@
package format
import (
"bytes"
"fmt"
"math"
"testing"
"github.com/anchore/syft/syft/format/syftjson"
"github.com/anchore/syft/syft/pkg"
"github.com/anchore/syft/syft/sbom"
"github.com/anchore/syft/syft/source"
)
// TestEncodersHandleUntypedMetadataValues pushes awkward values through the untyped metadata fields
// allowlisted in internal/packagemetadata (see untypedFieldAllowlist) and asserts that SetID and every
// encoder neither panic nor fail. A new allowlist entry should get a case here too.
func TestEncodersHandleUntypedMetadataValues(t *testing.T) {
cases := []struct {
name string
value any
// encoding/json rejects non-finite floats, so syft-json returns an error for them. Producers
// have to replace these before they reach the model (the GGUF cataloger does); the case stays
// here so the expectation flips visibly if syft-json ever learns to encode them.
syftJSONRejects bool
}{
{name: "NaN", value: math.NaN(), syftJSONRejects: true},
{name: "+Inf float32", value: float32(math.Inf(1)), syftJSONRejects: true},
{name: "-Inf", value: math.Inf(-1), syftJSONRejects: true},
{name: "NaN in array", value: []any{1.0, math.NaN()}, syftJSONRejects: true},
{name: "invalid utf-8", value: "\xff\xfe"},
{name: "NUL and control characters", value: "a\x00b\x1b"},
{name: "max uint64", value: uint64(math.MaxUint64)},
{name: "deep array", value: nested(2000, func(v any) any { return []any{v} })},
{name: "deep map", value: nested(2000, func(v any) any { return map[string]any{"k": v} })},
{name: "large array", value: make([]any, 100_000)},
}
for _, tt := range cases {
t.Run(tt.name, func(t *testing.T) {
p := pkg.Package{
Name: "model",
Version: "1.0",
Type: pkg.ModelPkg,
Metadata: pkg.GGUFFileHeader{
RemainingKeyValues: map[string]any{"general.value": tt.value},
},
}
if !noPanic(t, "SetID", p.SetID) {
return
}
s := sbom.SBOM{
Artifacts: sbom.Artifacts{Packages: pkg.NewCollection(p)},
Source: source.Description{Metadata: source.DirectoryMetadata{Path: "/models"}},
}
for _, enc := range Encoders() {
id := fmt.Sprintf("%s@%s", enc.ID(), enc.Version())
var buf bytes.Buffer
noPanic(t, id, func() {
err := enc.Encode(&buf, s)
switch {
case tt.syftJSONRejects && enc.ID() == syftjson.ID:
if err == nil {
t.Errorf("%s: expected an encode error for a non-finite float", id)
}
case err != nil:
t.Errorf("%s: encode failed: %v", id, err)
}
})
}
})
}
}
func nested(depth int, wrap func(any) any) any {
var v any = "leaf"
for range depth {
v = wrap(v)
}
return v
}
func noPanic(t *testing.T, what string, fn func()) (ok bool) {
t.Helper()
defer func() {
if r := recover(); r != nil {
t.Errorf("%s: panic: %v", what, r)
ok = false
}
}()
fn()
return true
}
syft/internal/fileresolver/chroot_context.go
+4 -1
View File
@@ -216,5 +216,8 @@ func (r ChrootContext) ToChrootPath(nativePath string) string {
responsePath = strings.TrimPrefix(responsePath, prefix) responsePath = strings.TrimPrefix(responsePath, prefix)
} }
return responsePath // chroot paths are posix regardless of host (these end up as location paths in the SBOM). This uses
// filepath (not path) since responsePath is native here (FromPosix above, then trimmed of a native prefix),
// and only filepath.ToSlash knows the host separator to convert.
return filepath.ToSlash(responsePath)
} }
syft/internal/fileresolver/chroot_context_test.go
+7
View File
@@ -3,6 +3,7 @@ package fileresolver
import ( import (
"os" "os"
"path/filepath" "path/filepath"
"runtime"
"testing" "testing"
"github.com/stretchr/testify/assert" "github.com/stretchr/testify/assert"
@@ -10,6 +11,9 @@ import (
) )
func Test_ChrootContext_RequestResponse(t *testing.T) { func Test_ChrootContext_RequestResponse(t *testing.T) {
if runtime.GOOS == "windows" {
t.Skip("fixture relies on symlinks, which git does not check out as links on windows by default")
}
// / // /
// somewhere/ // somewhere/
// outside.txt // outside.txt
@@ -563,6 +567,9 @@ func TestNewChrootContext_BaseValidation(t *testing.T) {
} }
func TestToNativeGlob(t *testing.T) { func TestToNativeGlob(t *testing.T) {
if runtime.GOOS == "windows" {
t.Skip("chroot context and expectations are posix paths")
}
tests := []struct { tests := []struct {
name string name string
chrootContext ChrootContext chrootContext ChrootContext
syft/internal/fileresolver/directory_indexer_test.go
+37 -11
View File
@@ -4,7 +4,6 @@ import (
"fmt" "fmt"
"io/fs" "io/fs"
"os" "os"
"path"
"path/filepath" "path/filepath"
"runtime" "runtime"
"sort" "sort"
@@ -18,6 +17,7 @@ import (
"github.com/wagoodman/go-progress" "github.com/wagoodman/go-progress"
"github.com/anchore/stereoscope/pkg/file" "github.com/anchore/stereoscope/pkg/file"
"github.com/anchore/syft/syft/internal/windows"
) )
type indexerMock struct { type indexerMock struct {
@@ -138,6 +138,9 @@ func TestDirectoryIndexer_handleFileAccessErr(t *testing.T) {
} }
func TestDirectoryIndexer_IncludeRootPathInIndex(t *testing.T) { func TestDirectoryIndexer_IncludeRootPathInIndex(t *testing.T) {
if runtime.GOOS == "windows" {
t.Skip("indexes the posix filesystem root")
}
filterFn := func(_, path string, _ os.FileInfo, _ error) error { filterFn := func(_, path string, _ os.FileInfo, _ error) error {
if path != "/" { if path != "/" {
return fs.SkipDir return fs.SkipDir
@@ -206,11 +209,7 @@ func TestDirectoryIndexer_index(t *testing.T) {
info, err := os.Stat(test.path) info, err := os.Stat(test.path)
assert.NoError(t, err) assert.NoError(t, err)
// note: the index uses absolute paths, so assertions MUST keep this in mind p := indexedPath(t, test.path)
cwd, err := os.Getwd()
require.NoError(t, err)
p := file.Path(path.Join(cwd, test.path))
assert.Equal(t, true, tree.HasPath(p)) assert.Equal(t, true, tree.HasPath(p))
exists, ref, err := tree.File(p) exists, ref, err := tree.File(p)
assert.Equal(t, true, exists) assert.Equal(t, true, exists)
@@ -226,6 +225,9 @@ func TestDirectoryIndexer_index(t *testing.T) {
} }
func TestDirectoryIndexer_index_for_AncestorSymlinks(t *testing.T) { func TestDirectoryIndexer_index_for_AncestorSymlinks(t *testing.T) {
if runtime.GOOS == "windows" {
t.Skip("fixture relies on symlinks, which git does not check out as links on windows by default")
}
// note: this test is testing the effects from NewFromDirectory, indexTree, and addPathToIndex // note: this test is testing the effects from NewFromDirectory, indexTree, and addPathToIndex
_, filename, _, ok := runtime.Caller(0) _, filename, _, ok := runtime.Caller(0)
require.True(t, ok) require.True(t, ok)
@@ -256,11 +258,7 @@ func TestDirectoryIndexer_index_for_AncestorSymlinks(t *testing.T) {
info, err := os.Stat(test.path) info, err := os.Stat(test.path)
assert.NoError(t, err) assert.NoError(t, err)
// note: the index uses absolute paths, so assertions MUST keep this in mind p := indexedPath(t, test.path)
cwd, err := os.Getwd()
require.NoError(t, err)
p := file.Path(path.Join(cwd, test.path))
assert.Equal(t, true, tree.HasPath(p)) assert.Equal(t, true, tree.HasPath(p))
exists, ref, err := tree.File(p) exists, ref, err := tree.File(p)
assert.Equal(t, true, exists) assert.Equal(t, true, exists)
@@ -287,6 +285,9 @@ func TestDirectoryIndexer_index_survive_badSymlink(t *testing.T) {
} }
func TestDirectoryIndexer_index_survive_inaccessibleSymlinkTarget(t *testing.T) { func TestDirectoryIndexer_index_survive_inaccessibleSymlinkTarget(t *testing.T) {
if runtime.GOOS == "windows" {
t.Skip("relies on symlinks and posix permissions")
}
// a symlink that resolves into a directory we don't have permission to traverse should be skipped // a symlink that resolves into a directory we don't have permission to traverse should be skipped
// with a warning like any other inaccessible path, not abort the entire scan (see #3286). // with a warning like any other inaccessible path, not abort the entire scan (see #3286).
if os.Geteuid() == 0 { if os.Geteuid() == 0 {
@@ -323,6 +324,9 @@ func TestDirectoryIndexer_index_survive_inaccessibleSymlinkTarget(t *testing.T)
} }
func TestDirectoryIndexer_SkipsAlreadyVisitedLinkDestinations(t *testing.T) { func TestDirectoryIndexer_SkipsAlreadyVisitedLinkDestinations(t *testing.T) {
if runtime.GOOS == "windows" {
t.Skip("fixture relies on symlinks, which git does not check out as links on windows by default")
}
var observedPaths []string var observedPaths []string
pathObserver := func(_, p string, _ os.FileInfo, _ error) error { pathObserver := func(_, p string, _ os.FileInfo, _ error) error {
fields := strings.Split(p, "testdata/symlinks-prune-indexing") fields := strings.Split(p, "testdata/symlinks-prune-indexing")
@@ -430,6 +434,11 @@ func TestDirectoryIndexer_IndexesAllTypes(t *testing.T) {
} }
func Test_allContainedPaths(t *testing.T) { func Test_allContainedPaths(t *testing.T) {
if runtime.GOOS == "windows" {
// ponytail: production passes native windows paths here, which these posix cases don't represent.
// Ancestor indexing for a root reached through a symlink is likely broken on windows; fix separately.
t.Skip("cases are posix paths")
}
tests := []struct { tests := []struct {
name string name string
@@ -485,6 +494,9 @@ func Test_allContainedPaths(t *testing.T) {
} }
func Test_relativePath(t *testing.T) { func Test_relativePath(t *testing.T) {
if runtime.GOOS == "windows" {
t.Skip("exercises a posix-only test helper")
}
tests := []struct { tests := []struct {
name string name string
basePath string basePath string
@@ -548,3 +560,17 @@ func relativePath(basePath, givenPath string) string {
return relPath return relPath
} }
// indexedPath returns how the given cwd-relative path is keyed in the index (absolute and always posix)
func indexedPath(t *testing.T, rel string) file.Path {
t.Helper()
cwd, err := os.Getwd()
require.NoError(t, err)
// filepath (not path): cwd is a native host path, so it must be joined with the host separator before
// being converted to the posix form the index is keyed on
p := filepath.Join(cwd, rel)
if windows.HostRunningOnWindows() {
p = windows.ToPosix(p)
}
return file.Path(p)
}
syft/internal/fileresolver/directory_roundtrip_test.go
+48
View File
@@ -0,0 +1,48 @@
package fileresolver
import (
"context"
"os"
"path/filepath"
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
stereoscopeFile "github.com/anchore/stereoscope/pkg/file"
)
// a location handed out by the resolver should be resolvable again by its own path. This is what the file
// digest cataloger relies on, and it silently found nothing on windows (see #5325).
func TestDirectory_LocationPathsRoundTrip(t *testing.T) {
// resolve symlinks up front (e.g. /var -> /private/var on macOS) so the cwd prefix matches the indexed paths
root, err := filepath.EvalSymlinks(t.TempDir())
require.NoError(t, err)
require.NoError(t, os.MkdirAll(filepath.Join(root, "sub", "dir"), 0o755))
for _, p := range []string{"top.txt", filepath.Join("sub", "mid.txt"), filepath.Join("sub", "dir", "deep.txt")} {
require.NoError(t, os.WriteFile(filepath.Join(root, p), []byte(p), 0o600))
}
// mirror `syft dir:.` from within the scanned directory
t.Chdir(root)
resolver, err := NewFromDirectory(".", "")
require.NoError(t, err)
var regular int
for loc := range resolver.AllLocations(context.Background()) {
md, err := resolver.FileMetadataByLocation(loc)
require.NoError(t, err)
if md.Type != stereoscopeFile.TypeRegular {
continue
}
regular++
assert.NotContains(t, loc.RealPath, `\`, "location paths should be posix")
found, err := resolver.FilesByPath(loc.RealPath)
require.NoError(t, err)
assert.Len(t, found, 1, "unable to resolve %q by its own path", loc.RealPath)
}
assert.Equal(t, 3, regular)
}
syft/internal/fileresolver/directory_windows_test.go
-126
View File
@@ -1,126 +0,0 @@
package fileresolver
import "testing"
func Test_windowsToPosix(t *testing.T) {
type args struct {
windowsPath string
}
tests := []struct {
name string
args args
wantPosixPath string
}{
{
name: "basic case",
args: args{
windowsPath: `C:\some\windows\place`,
},
wantPosixPath: "/c/some/windows/place",
},
{
name: "escaped case",
args: args{
windowsPath: `C:\\some\\windows\\place`,
},
wantPosixPath: "/c/some/windows/place",
},
{
name: "forward slash",
args: args{
windowsPath: `C:/foo/bar`,
},
wantPosixPath: "/c/foo/bar",
},
{
name: "mix slash",
args: args{
windowsPath: `C:\foo/bar\`,
},
wantPosixPath: "/c/foo/bar",
},
{
name: "case sensitive case",
args: args{
windowsPath: `C:\Foo/bAr\`,
},
wantPosixPath: "/c/Foo/bAr",
},
{
name: "special char case",
args: args{
windowsPath: `C:\ふー\バー`,
},
wantPosixPath: "/c/ふー/バー",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
if gotPosixPath := windowsToPosix(tt.args.windowsPath); gotPosixPath != tt.wantPosixPath {
t.Errorf("windowsToPosix() = %v, want %v", gotPosixPath, tt.wantPosixPath)
}
})
}
}
func Test_posixToWindows(t *testing.T) {
type args struct {
posixPath string
}
tests := []struct {
name string
args args
wantWindowsPath string
}{
{
name: "basic case",
args: args{
posixPath: "/c/some/windows/place",
},
wantWindowsPath: `C:\some\windows\place`,
},
{
name: "escaped case",
args: args{
posixPath: "/c/some/windows/place",
},
wantWindowsPath: `C:\\some\\windows\\place`,
},
{
name: "forward slash",
args: args{
posixPath: "/c/foo/bar",
},
wantWindowsPath: `C:/foo/bar`,
},
{
name: "mix slash",
args: args{
posixPath: "/c/foo/bar",
},
wantWindowsPath: `C:\foo/bar\`,
},
{
name: "case sensitive case",
args: args{
posixPath: "/c/Foo/bAr",
},
wantWindowsPath: `C:\Foo/bAr\`,
},
{
name: "special char case",
args: args{
posixPath: "/c/ふー/バー",
},
wantWindowsPath: `C:\ふー\バー`,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
if gotWindowsPath := posixToWindows(tt.args.posixPath); gotWindowsPath != tt.wantWindowsPath {
t.Errorf("posixToWindows() = %v, want %v", gotWindowsPath, tt.wantWindowsPath)
}
})
}
}
syft/internal/fileresolver/file_indexer.go
+5 -1
View File
@@ -29,11 +29,15 @@ func newFileIndexer(path, base string, visitors ...PathIndexVisitor) *fileIndexe
base: base, base: base,
tree: filetree.New(), tree: filetree.New(),
index: filetree.NewIndex(), index: filetree.NewIndex(),
// Unlike the directory indexer there is no skipPathsByMountTypeAndName here: it exists
// to keep a directory walk out of /proc, /sys and similar, and to do so it reads the
// whole mount table on construction. A file source indexes exactly one file and its
// parent, so there is nothing to walk into, and the mount table read is paid for
// every file source with nothing in return.
pathIndexVisitors: append( pathIndexVisitors: append(
[]PathIndexVisitor{ []PathIndexVisitor{
requireFileInfo, requireFileInfo,
disallowByFileType, disallowByFileType,
skipPathsByMountTypeAndName(path),
}, },
visitors..., visitors...,
), ),
syft/internal/fileresolver/file_indexer_bench_test.go
+51
View File
@@ -0,0 +1,51 @@
package fileresolver
import (
"os"
"path/filepath"
"testing"
"github.com/moby/sys/mountinfo"
)
// BenchmarkNewFromFile is the cost of building a resolver for a single-file source, which is
// paid once per file by callers that catalog files individually. The default case is
// NewFromFile as shipped. The with-mount-skipper case attaches skipPathsByMountTypeAndName
// the way the file indexer used to, so the difference is what reading the mount table for
// every file source cost. That cost scales with the mount table, so its size is reported as
// the mounts metric: expect a bigger gap on a container host than on a laptop.
//
// go test ./syft/internal/fileresolver/ -run '^$' -bench NewFromFile -benchmem -count 6 | tee bench.txt
// benchstat -col /variant bench.txt
func BenchmarkNewFromFile(b *testing.B) {
path := filepath.Join(b.TempDir(), "file.txt")
if err := os.WriteFile(path, []byte("hello\n"), 0o644); err != nil {
b.Fatal(err)
}
mounts, err := mountinfo.GetMounts(nil)
if err != nil {
b.Fatal(err)
}
b.Run("variant=default", func(b *testing.B) {
for b.Loop() {
if _, err := NewFromFile(path); err != nil {
b.Fatal(err)
}
}
b.ReportMetric(float64(len(mounts)), "mounts")
})
b.Run("variant=with-mount-skipper", func(b *testing.B) {
for b.Loop() {
r, err := newFromFileWithoutIndex(path, skipPathsByMountTypeAndName(path))
if err != nil {
b.Fatal(err)
}
if err := r.buildIndex(); err != nil {
b.Fatal(err)
}
}
b.ReportMetric(float64(len(mounts)), "mounts")
})
}
syft/internal/fileresolver/file_indexer_test.go
+7 -10
View File
@@ -3,13 +3,11 @@ package fileresolver
import ( import (
"io/fs" "io/fs"
"os" "os"
"path" "path/filepath"
"testing" "testing"
"github.com/stretchr/testify/assert" "github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require" "github.com/stretchr/testify/require"
"github.com/anchore/stereoscope/pkg/file"
) )
// - Verify that both the parent and the path are indexed // - Verify that both the parent and the path are indexed
@@ -38,11 +36,7 @@ func Test_index(t *testing.T) {
info, err := os.Stat(test.path) info, err := os.Stat(test.path)
assert.NoError(t, err) assert.NoError(t, err)
// note: the index uses absolute paths, so assertions MUST keep this in mind p := indexedPath(t, test.path)
cwd, err := os.Getwd()
require.NoError(t, err)
p := file.Path(path.Join(cwd, test.path))
assert.Equal(t, true, tree.HasPath(p)) assert.Equal(t, true, tree.HasPath(p))
exists, ref, err := tree.File(p) exists, ref, err := tree.File(p)
assert.Equal(t, true, exists) assert.Equal(t, true, exists)
@@ -71,7 +65,9 @@ func Test_ignoresPathIfFiltered(t *testing.T) {
testPath := "testdata/system_paths/target/home/place" testPath := "testdata/system_paths/target/home/place"
cwd, cwdErr := os.Getwd() cwd, cwdErr := os.Getwd()
require.NoError(t, cwdErr) require.NoError(t, cwdErr)
ignorePath := path.Join(cwd, testPath) // filepath (not path): filters see the native absolute path (before posix normalization), so the
// expected value must be joined with the host separator to match on windows
ignorePath := filepath.Join(cwd, testPath)
filterFn := func(_, path string, _ os.FileInfo, _ error) error { filterFn := func(_, path string, _ os.FileInfo, _ error) error {
if path == ignorePath { if path == ignorePath {
return ErrSkipPath return ErrSkipPath
@@ -91,7 +87,8 @@ func Test_ignoresPathIfParentFiltered(t *testing.T) {
cwd, cwdErr := os.Getwd() cwd, cwdErr := os.Getwd()
require.NoError(t, cwdErr) require.NoError(t, cwdErr)
ignorePath := path.Join(cwd, parentPath) // filepath (not path): see Test_ignoresPathIfFiltered
ignorePath := filepath.Join(cwd, parentPath)
filterFn := func(_, path string, _ os.FileInfo, _ error) error { filterFn := func(_, path string, _ os.FileInfo, _ error) error {
if path == ignorePath { if path == ignorePath {
return fs.SkipDir return fs.SkipDir
syft/internal/fileresolver/filetree_resolver.go
+23 -6
View File
@@ -6,6 +6,7 @@ import (
"fmt" "fmt"
"io" "io"
"os" "os"
"path/filepath"
stereoscopeFile "github.com/anchore/stereoscope/pkg/file" stereoscopeFile "github.com/anchore/stereoscope/pkg/file"
"github.com/anchore/stereoscope/pkg/filetree" "github.com/anchore/stereoscope/pkg/filetree"
@@ -34,8 +35,17 @@ func nativeOSFileOpener(ref stereoscopeFile.Reference) (io.ReadCloser, error) {
return stereoscopeFile.NewLazyReadCloser(filePath), nil return stereoscopeFile.NewLazyReadCloser(filePath), nil
} }
// requestPath converts a user (chroot) path into the path the file tree is keyed on. The tree is always posix
// (windows paths are volume-encoded, e.g. /c/some/path), while ToNativePath yields a native host path.
func (r *FiletreeResolver) requestPath(userPath string) (string, error) { func (r *FiletreeResolver) requestPath(userPath string) (string, error) {
return r.Chroot.ToNativePath(userPath) nativePath, err := r.Chroot.ToNativePath(userPath)
if err != nil {
return "", err
}
if windows.HostRunningOnWindows() {
return windows.ToPosix(nativePath), nil
}
return nativePath, nil
} }
// responsePath takes a path from the underlying fs domain and converts it to a path that is relative to the root of the file resolver. // responsePath takes a path from the underlying fs domain and converts it to a path that is relative to the root of the file resolver.
@@ -85,10 +95,6 @@ func (r FiletreeResolver) FilesByPath(userPaths ...string) ([]file.Location, err
continue continue
} }
if windows.HostRunningOnWindows() {
userStrPath = windows.ToPosix(userStrPath)
}
if ref.HasReference() { if ref.HasReference() {
references = append(references, references = append(references,
file.NewVirtualLocationFromDirectory( file.NewVirtualLocationFromDirectory(
@@ -103,8 +109,19 @@ func (r FiletreeResolver) FilesByPath(userPaths ...string) ([]file.Location, err
return references, nil return references, nil
} }
// requestGlob is the glob equivalent of requestPath.
func (r FiletreeResolver) requestGlob(pattern string) (string, error) { func (r FiletreeResolver) requestGlob(pattern string) (string, error) {
return r.Chroot.ToNativeGlob(pattern) glob, err := r.Chroot.ToNativeGlob(pattern)
if err != nil {
return "", err
}
// only a glob that was anchored to the root comes back native. Globs that start with a wildcard
// (e.g. **/foo) are returned untouched and are already posix, and ToPosix would root them at "/".
// This uses filepath (not path) since we are asking whether this is a native absolute path.
if windows.HostRunningOnWindows() && filepath.IsAbs(glob) {
return windows.ToPosix(glob), nil
}
return glob, nil
} }
// FilesByGlob returns all file.References that match the given path glob pattern from any layer in the image. // FilesByGlob returns all file.References that match the given path glob pattern from any layer in the image.
syft/internal/parsing/basic_parsing.go
+24 -7
View File
@@ -22,6 +22,15 @@ func SkipWhitespace(data []byte, i *int) {
} }
} }
// MaxDepth bounds nesting in the recursive hand-written parsers. Real files nest a handful of levels; without a
// bound, deep input overflows the goroutine stack, which is fatal rather than a recoverable panic.
const MaxDepth = 1000
// errorContext is how many bytes of each echoed line PrintError keeps around the error column, so an error in a
// huge single-line input stays small (and does not copy the file into the SBOM as an unknown).
const errorContext = 80
// PrintError describes where in data offset i is, echoing up to two lines of context clipped around the column.
func PrintError(data []byte, i int) string { func PrintError(data []byte, i int) string {
line := 1 line := 1
char := 1 char := 1
@@ -57,15 +66,23 @@ func PrintError(data []byte, i int) string {
sep := ": " sep := ": "
lines := "" start := max(0, char-1-errorContext/2)
if len(prev) > 1 { clip := func(s string) string {
lines += fmt.Sprintf("%s%s%s\n", l1, sep, prev[len(prev)-2]) if start >= len(s) {
} return ""
if len(prev) > 0 { }
lines += fmt.Sprintf("%s%s%s\n", l2, sep, prev[len(prev)-1]) return s[start:min(len(s), start+errorContext)]
} }
pointer := strings.Repeat(" ", len(l2)+len(sep)+char-1) + "^" lines := ""
if len(prev) > 1 {
lines += fmt.Sprintf("%s%s%s\n", l1, sep, clip(prev[len(prev)-2]))
}
if len(prev) > 0 {
lines += fmt.Sprintf("%s%s%s\n", l2, sep, clip(prev[len(prev)-1]))
}
pointer := strings.Repeat(" ", len(l2)+len(sep)+char-1-start) + "^"
return fmt.Sprintf("line: %v, char: %v\n%s%s", line, char, lines, pointer) return fmt.Sprintf("line: %v, char: %v\n%s%s", line, char, lines, pointer)
} }
syft/internal/windows/path.go
+8
View File
@@ -35,7 +35,15 @@ func ToPosix(windowsPath string) (posixPath string) {
func FromPosix(posixPath string) (windowsPath string) { func FromPosix(posixPath string) (windowsPath string) {
// decode the volume (e.g. /c/<path> --> C:\\) - There should always be a volume name. // decode the volume (e.g. /c/<path> --> C:\\) - There should always be a volume name.
// The volume may be a UNC path (e.g. /\\localhost\C$\ --> \\localhost\C$\) // The volume may be a UNC path (e.g. /\\localhost\C$\ --> \\localhost\C$\)
if !strings.HasPrefix(posixPath, "/") {
// not a volume-encoded posix path (e.g. relative or already native), so there is nothing to decode
return posixPath
}
pathFields := strings.Split(posixPath, "/") pathFields := strings.Split(posixPath, "/")
if len(pathFields) < 2 {
// no separator means no volume to decode
return filepath.Clean(posixPath)
}
rootPath := strings.ToUpper(pathFields[1]) rootPath := strings.ToUpper(pathFields[1])
volumeName := AppendRootTerminator(rootPath) volumeName := AppendRootTerminator(rootPath)
syft/internal/windows/path_windows_test.go
+35
View File
@@ -0,0 +1,35 @@
package windows
import (
"testing"
"github.com/stretchr/testify/assert"
)
func TestToPosix(t *testing.T) {
tests := map[string]string{
`C:\some\windows\place`: "/c/some/windows/place",
`C:\\some\\windows\\place`: "/c/some/windows/place",
`C:/foo/bar`: "/c/foo/bar",
`C:\foo/bar\`: "/c/foo/bar",
`C:\Foo/bAr\`: "/c/Foo/bAr",
`C:\ふー\バー`: "/c/ふー/バー",
}
for in, want := range tests {
assert.Equal(t, want, ToPosix(in), in)
}
}
func TestFromPosix(t *testing.T) {
tests := map[string]string{
"/c/some/windows/place": `C:\some\windows\place`,
"/c/Foo/bAr": `C:\Foo\bAr`,
"/c/ふー/バー": `C:\ふー\バー`,
// non volume-encoded input is left alone
"relative/path": "relative/path",
`D:\already\native`: `D:\already\native`,
}
for in, want := range tests {
assert.Equal(t, want, FromPosix(in), in)
}
}
syft/linux/identify_release.go
+2 -2
View File
@@ -156,8 +156,8 @@ var busyboxVersionMatcher = regexp.MustCompile(`BusyBox v[\d.]+`)
func parseBusyBox(contents string) (*Release, error) { func parseBusyBox(contents string) (*Release, error) {
matches := busyboxVersionMatcher.FindAllString(contents, -1) matches := busyboxVersionMatcher.FindAllString(contents, -1)
for _, match := range matches { for _, match := range matches {
parts := strings.Split(match, " ") _, version, _ := strings.Cut(match, " ")
version := strings.ReplaceAll(parts[1], "v", "") version = strings.ReplaceAll(version, "v", "")
return simpleRelease(match, "busybox", version, ""), nil return simpleRelease(match, "busybox", version, ""), nil
} }
syft/pkg/cataloger/ai/cataloger.go
+6 -2
View File
@@ -11,8 +11,12 @@ import (
const ( const (
catalogerName = "gguf-cataloger" catalogerName = "gguf-cataloger"
ggufLayerMediaType = "application/vnd.docker.ai*"
safeTensorsCatalogerName = "safetensors-cataloger" safeTensorsCatalogerName = "safetensors-cataloger"
// dockerAIGGUFMediaType mirrors ggufLayerMediaType in
// syft/source/ocimodelsource/registry_client.go. It is matched exactly: the
// other application/vnd.docker.ai.* layers are not GGUF.
dockerAIGGUFMediaType = "application/vnd.docker.ai.gguf.v3"
) )
// NewGGUFCataloger returns a new cataloger instance for GGUF model files. // NewGGUFCataloger returns a new cataloger instance for GGUF model files.
@@ -21,7 +25,7 @@ const (
func NewGGUFCataloger() pkg.Cataloger { func NewGGUFCataloger() pkg.Cataloger {
return generic.NewCataloger(catalogerName). return generic.NewCataloger(catalogerName).
WithParserByGlobs(parseGGUFModel, "**/*.gguf"). WithParserByGlobs(parseGGUFModel, "**/*.gguf").
WithParserByMediaType(parseGGUFModel, ggufLayerMediaType). WithParserByMediaType(parseGGUFModel, dockerAIGGUFMediaType).
WithProcessors(ggufMergeProcessor) WithProcessors(ggufMergeProcessor)
} }
syft/pkg/cataloger/ai/cataloger_test.go
+45
View File
@@ -1,11 +1,17 @@
package ai package ai
import ( import (
"context"
"os" "os"
"path/filepath" "path/filepath"
"testing" "testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/anchore/syft/internal/tmpdir"
"github.com/anchore/syft/syft/artifact" "github.com/anchore/syft/syft/artifact"
"github.com/anchore/syft/syft/internal/fileresolver"
"github.com/anchore/syft/syft/pkg" "github.com/anchore/syft/syft/pkg"
"github.com/anchore/syft/syft/pkg/cataloger/internal/pkgtest" "github.com/anchore/syft/syft/pkg/cataloger/internal/pkgtest"
) )
@@ -138,3 +144,42 @@ func TestGGUFCataloger(t *testing.T) {
}) })
} }
} }
func TestGGUFCataloger_ociLayers(t *testing.T) {
// a GGUF OCI artifact is merged into one named package, while the other
// application/vnd.docker.ai.* layers (config, model.file, license and
// safetensors weights) never reach the GGUF parser.
named := newTestGGUFBuilder().
withStringKV("general.architecture", "llama").
withStringKV("general.name", "llama3-8b").
build()
part := newTestGGUFBuilder().withStringKV("general.architecture", "llama").build()
dir := t.TempDir()
layers := map[string]fileresolver.LayerInfo{}
for digest, l := range map[string]struct {
mediaType string
content []byte
}{
"sha256:named": {dockerAIGGUFMediaType, named},
"sha256:part": {dockerAIGGUFMediaType, part},
"sha256:config": {dockerAIModelConfigMediaTypes[0], []byte("not a gguf file")},
"sha256:file": {dockerAIModelFileMediaType, []byte("not a gguf file")},
"sha256:license": {dockerAILicenseMediaType, []byte("not a gguf file")},
"sha256:weights": {dockerAISafeTensorsMediaType, []byte("not a gguf file")},
} {
p := filepath.Join(dir, digest[len("sha256:"):])
require.NoError(t, os.WriteFile(p, l.content, 0o600))
layers[digest] = fileresolver.LayerInfo{TempPath: p, MediaType: l.mediaType}
}
resolver := fileresolver.NewContainerImageModel(dir, layers, "example/model:latest")
ctx := tmpdir.WithValue(context.Background(), tmpdir.FromPath(t.TempDir()))
pkgs, _, err := NewGGUFCataloger().Catalog(ctx, resolver)
require.NoError(t, err)
require.Len(t, pkgs, 1)
assert.Equal(t, "llama3-8b", pkgs[0].Name)
header, ok := pkgs[0].Metadata.(pkg.GGUFFileHeader)
require.True(t, ok)
assert.Len(t, header.Parts, 1)
}
syft/pkg/cataloger/ai/gguf_processor.go
+7 -7
View File
@@ -8,17 +8,17 @@ import (
// ggufMergeProcessor consolidates multiple GGUF packages into a single package // ggufMergeProcessor consolidates multiple GGUF packages into a single package
// representing the AI model. When scanning OCI images with multiple layers, // representing the AI model. When scanning OCI images with multiple layers,
// each layer may produce a separate package. This processor finds the package // each layer may produce a separate package. This processor finds the package
// with a name and merges metadata from nameless packages into its GGUFFileParts field. // with a name and merges metadata from nameless packages into its Parts field.
// Only packages with a non-empty name are returned in the final result. // Only packages with a non-empty name are returned in the final result.
func ggufMergeProcessor(pkgs []pkg.Package, rels []artifact.Relationship, err error) ([]pkg.Package, []artifact.Relationship, error) { func ggufMergeProcessor(pkgs []pkg.Package, rels []artifact.Relationship, err error) ([]pkg.Package, []artifact.Relationship, error) {
if err != nil { // don't early-return on err: it aggregates per-file failures, and skipping the
return pkgs, rels, err // merge would drop every nameless layer and the named package's Parts along
} // with it (see safeTensorsMergeProcessor).
if len(pkgs) == 0 { if len(pkgs) == 0 {
return pkgs, rels, err return pkgs, rels, err
} }
// Separate packages with names from those without // separate packages with names from those without
var namedPkgs []pkg.Package var namedPkgs []pkg.Package
var namelessHeaders []pkg.GGUFFileHeader var namelessHeaders []pkg.GGUFFileHeader
@@ -27,14 +27,14 @@ func ggufMergeProcessor(pkgs []pkg.Package, rels []artifact.Relationship, err er
namedPkgs = append(namedPkgs, p) namedPkgs = append(namedPkgs, p)
} else { } else {
if header, ok := p.Metadata.(pkg.GGUFFileHeader); ok { if header, ok := p.Metadata.(pkg.GGUFFileHeader); ok {
// We do not want a kv hash for nameless headers // we do not want a kv hash for nameless headers
header.MetadataKeyValuesHash = "" header.MetadataKeyValuesHash = ""
namelessHeaders = append(namelessHeaders, header) namelessHeaders = append(namelessHeaders, header)
} }
} }
} }
// If there are no named packages, return nothing // if there are no named packages, return nothing
if len(namedPkgs) == 0 { if len(namedPkgs) == 0 {
return nil, rels, err return nil, rels, err
} }
syft/pkg/cataloger/ai/huggingface.go
+60 -20
View File
@@ -3,7 +3,7 @@ package ai
import ( import (
"bytes" "bytes"
"gopkg.in/yaml.v3" "go.yaml.in/yaml/v3"
"github.com/anchore/syft/internal/log" "github.com/anchore/syft/internal/log"
"github.com/anchore/syft/syft/pkg" "github.com/anchore/syft/syft/pkg"
@@ -15,6 +15,12 @@ type hfConfig struct {
NameOrPath string `json:"_name_or_path"` NameOrPath string `json:"_name_or_path"`
} }
// looksLikeHF reports whether the config carries any field we use. Other JSON
// (generation_config.json, unrelated app configs) is ignored.
func (c hfConfig) looksLikeHF() bool {
return len(c.Architectures) > 0 || c.NameOrPath != ""
}
func applyHFConfig(md *pkg.SafeTensorsModelInfo, cfg *hfConfig) { func applyHFConfig(md *pkg.SafeTensorsModelInfo, cfg *hfConfig) {
if md.Architecture == "" && len(cfg.Architectures) > 0 { if md.Architecture == "" && len(cfg.Architectures) > 0 {
md.Architecture = cfg.Architectures[0] md.Architecture = cfg.Architectures[0]
@@ -23,8 +29,8 @@ func applyHFConfig(md *pkg.SafeTensorsModelInfo, cfg *hfConfig) {
// readmeFrontmatter holds the subset of YAML frontmatter fields we extract. // readmeFrontmatter holds the subset of YAML frontmatter fields we extract.
type readmeFrontmatter struct { type readmeFrontmatter struct {
License string `yaml:"license"` Licenses []string
BaseModel []string `yaml:"base_model"` BaseModel []string
} }
type licenseFrontmatter struct { type licenseFrontmatter struct {
@@ -32,21 +38,33 @@ type licenseFrontmatter struct {
} }
// extractFrontmatterBlock returns the YAML bytes between the first and second // extractFrontmatterBlock returns the YAML bytes between the first and second
// "---" delimiters of a file // "---" delimiter lines of a file
func extractFrontmatterBlock(buf []byte) []byte { func extractFrontmatterBlock(buf []byte) []byte {
trimmed := bytes.TrimLeft(buf, "\xef\xbb\xbf \t\r\n") trimmed := bytes.TrimLeft(buf, "\xef\xbb\xbf \t\r\n")
if !bytes.HasPrefix(trimmed, []byte("---")) { first, rest, _ := bytes.Cut(trimmed, []byte("\n"))
if !isFrontmatterDelimiter(first) {
return nil return nil
} }
rest := trimmed[3:] for off := 0; off < len(rest); {
if i := bytes.IndexByte(rest, '\n'); i >= 0 { line, _, found := bytes.Cut(rest[off:], []byte("\n"))
rest = rest[i+1:] if isFrontmatterDelimiter(line) {
if off == 0 || off > maxFrontmatterSize {
return nil
}
return rest[:off]
}
if !found {
break
}
off += len(line) + 1
} }
block, _, found := bytes.Cut(rest, []byte("\n---")) return nil
if !found { }
return nil
} // isFrontmatterDelimiter reports whether a line is exactly "---", ignoring
return block // trailing whitespace and \r.
func isFrontmatterDelimiter(line []byte) bool {
return bytes.Equal(bytes.TrimRight(line, " \t\r"), []byte("---"))
} }
// parseFrontmatter decodes a Hugging Face model card YAML frontmatter block // parseFrontmatter decodes a Hugging Face model card YAML frontmatter block
@@ -57,8 +75,9 @@ func parseFrontmatter(buf []byte) *readmeFrontmatter {
return nil return nil
} }
// both fields may be a scalar or a list, so decode them as nodes
var raw struct { var raw struct {
License string `yaml:"license"` License yaml.Node `yaml:"license"`
BaseModel yaml.Node `yaml:"base_model"` BaseModel yaml.Node `yaml:"base_model"`
} }
if err := yaml.Unmarshal(block, &raw); err != nil { if err := yaml.Unmarshal(block, &raw); err != nil {
@@ -66,16 +85,37 @@ func parseFrontmatter(buf []byte) *readmeFrontmatter {
return nil return nil
} }
fm := readmeFrontmatter{License: raw.License} return &readmeFrontmatter{
switch raw.BaseModel.Kind { Licenses: yamlStrings(raw.License),
BaseModel: yamlStrings(raw.BaseModel),
}
}
// yamlStrings returns the non-empty, non-null values of a scalar or a sequence
// of scalars, keeping at most maxFrontmatterValues. Anything else yields nil.
func yamlStrings(n yaml.Node) []string {
switch n.Kind {
case yaml.ScalarNode: case yaml.ScalarNode:
if raw.BaseModel.Value != "" { if isYAMLString(&n) {
fm.BaseModel = []string{raw.BaseModel.Value} return []string{n.Value}
} }
case yaml.SequenceNode: case yaml.SequenceNode:
_ = raw.BaseModel.Decode(&fm.BaseModel) var out []string
for _, c := range n.Content {
if len(out) == maxFrontmatterValues {
break
}
if c.Kind == yaml.ScalarNode && isYAMLString(c) {
out = append(out, c.Value)
}
}
return out
} }
return &fm return nil
}
func isYAMLString(n *yaml.Node) bool {
return n.Value != "" && n.ShortTag() != "!!null"
} }
// parseLicenseFrontmatter returns the producer-declared SPDX identifier // parseLicenseFrontmatter returns the producer-declared SPDX identifier
syft/pkg/cataloger/ai/identity_dir.go
+28 -20
View File
@@ -3,7 +3,6 @@ package ai
import ( import (
"context" "context"
"encoding/json" "encoding/json"
"io"
"path" "path"
"github.com/anchore/syft/internal" "github.com/anchore/syft/internal"
@@ -13,42 +12,42 @@ import (
) )
// resolveSafeTensorsDirIdentity handles the directory-scan case for safe tensors // resolveSafeTensorsDirIdentity handles the directory-scan case for safe tensors
// find config.json beside the model files (walking up parent directories to the // find config.json beside the model files (or one directory up, for the shard
// scanned source root if no sibling exists) and a sibling README.md. It returns // subdirectory layout) and a sibling README.md. It returns
// the group's name candidates, resolved licenses, and supporting evidence. // the group's name candidates, resolved licenses, and supporting evidence.
func resolveSafeTensorsDirIdentity(ctx context.Context, resolver file.Resolver, dir string, md *pkg.SafeTensorsModelInfo) safeTensorsIdentity { func resolveSafeTensorsDirIdentity(ctx context.Context, resolver file.Resolver, dir string, md *pkg.SafeTensorsModelInfo) safeTensorsIdentity {
id := safeTensorsIdentity{fallbackName: safeTensorsDirName(dir)} id := safeTensorsIdentity{fallbackName: safeTensorsDirName(dir)}
if loc, cfg := findDirHFConfig(resolver, dir); cfg != nil { if loc, cfg := findDirHFConfig(resolver, dir); cfg != nil {
applyHFConfig(md, cfg) applyHFConfig(md, cfg)
id.nameOrPath = cfg.NameOrPath id.nameOrPath = sanitizeModelName(cfg.NameOrPath)
id.supporting = append(id.supporting, *loc) id.supporting = append(id.supporting, *loc)
} }
if loc, fm := readDirReadmeFrontmatter(resolver, path.Join(dir, "README.md")); fm != nil { if loc, fm := readDirReadmeFrontmatter(resolver, path.Join(dir, "README.md")); fm != nil {
if fm.License != "" { if len(fm.Licenses) > 0 {
id.licenses = pkg.NewLicensesFromValuesWithContext(ctx, fm.License) id.licenses = pkg.NewLicensesFromValuesWithContext(ctx, fm.Licenses...)
} }
if id.nameOrPath == "" && len(fm.BaseModel) > 0 { if id.nameOrPath == "" {
id.nameOrPath = fm.BaseModel[0] id.nameOrPath = firstModelName(fm.BaseModel)
} }
id.supporting = append(id.supporting, *loc) id.supporting = append(id.supporting, *loc)
} }
return id return id
} }
// findDirHFConfig looks for a config.json beside the model files // findDirHFConfig looks for an HF config.json beside the model files, then in
// the parent directory (covers weights kept in a subdirectory of the model). It
// does not walk further up, so an unrelated config.json higher in the tree can't
// name every model below it.
func findDirHFConfig(resolver file.Resolver, dir string) (*file.Location, *hfConfig) { func findDirHFConfig(resolver file.Resolver, dir string) (*file.Location, *hfConfig) {
for { if loc, cfg := readDirHFConfig(resolver, path.Join(dir, "config.json")); cfg != nil {
if loc, cfg := readDirHFConfig(resolver, path.Join(dir, "config.json")); cfg != nil { return loc, cfg
return loc, cfg
}
parent := path.Dir(dir)
if parent == dir {
return nil, nil // reached the source root
}
dir = parent
} }
if parent := path.Dir(dir); parent != dir {
return readDirHFConfig(resolver, path.Join(parent, "config.json"))
}
return nil, nil
} }
func readDirHFConfig(resolver file.Resolver, p string) (*file.Location, *hfConfig) { func readDirHFConfig(resolver file.Resolver, p string) (*file.Location, *hfConfig) {
@@ -62,11 +61,19 @@ func readDirHFConfig(resolver file.Resolver, p string) (*file.Location, *hfConfi
} }
defer internal.CloseAndLogError(rc, p) defer internal.CloseAndLogError(rc, p)
buf, err := readBounded(rc, maxHFConfigSize)
if err != nil {
log.Debugf("failed to read %s: %v", p, err)
return nil, nil
}
var cfg hfConfig var cfg hfConfig
if err := json.NewDecoder(rc).Decode(&cfg); err != nil { if err := json.Unmarshal(buf, &cfg); err != nil {
log.Debugf("failed to decode %s: %v", p, err) log.Debugf("failed to decode %s: %v", p, err)
return nil, nil return nil, nil
} }
if !cfg.looksLikeHF() {
return nil, nil
}
return &locations[0], &cfg return &locations[0], &cfg
} }
@@ -81,8 +88,9 @@ func readDirReadmeFrontmatter(resolver file.Resolver, p string) (*file.Location,
} }
defer internal.CloseAndLogError(rc, p) defer internal.CloseAndLogError(rc, p)
buf, err := io.ReadAll(io.LimitReader(rc, 1024*1024)) buf, err := readPrefix(rc, maxReadmePrefixSize)
if err != nil { if err != nil {
log.Debugf("failed to read %s: %v", p, err)
return nil, nil return nil, nil
} }
fm := parseFrontmatter(buf) fm := parseFrontmatter(buf)
syft/pkg/cataloger/ai/identity_oci.go
+32 -20
View File
@@ -4,8 +4,8 @@ import (
"bytes" "bytes"
"context" "context"
"encoding/json" "encoding/json"
"io"
"path" "path"
"sort"
gcrname "github.com/google/go-containerregistry/pkg/name" gcrname "github.com/google/go-containerregistry/pkg/name"
@@ -28,9 +28,18 @@ func resolveSafeTensorsOCIIdentity(ctx context.Context, resolver file.Resolver,
log.Debugf("failed to list docker AI model-file layers: %v", err) log.Debugf("failed to list docker AI model-file layers: %v", err)
} }
// Collect config / readme candidates separately so the layer-iteration order // the resolver returns layers in map order, and the first config / README
// returned by the resolver doesn't decide the precedence. // seen wins below, so sort for a stable result across runs. For OCI
var configName, readmeName, readmeLicense string // artifacts every RealPath is "/", so in practice this is digest order.
sort.Slice(modelFileLocs, func(i, j int) bool {
if modelFileLocs[i].RealPath != modelFileLocs[j].RealPath {
return modelFileLocs[i].RealPath < modelFileLocs[j].RealPath
}
return modelFileLocs[i].FileSystemID < modelFileLocs[j].FileSystemID
})
var configName, readmeName string
var readmeLicenses []string
var supporting []file.Location var supporting []file.Location
for _, loc := range modelFileLocs { for _, loc := range modelFileLocs {
cfg, fm := classifyOCIModelFileLayer(resolver, loc) cfg, fm := classifyOCIModelFileLayer(resolver, loc)
@@ -38,15 +47,15 @@ func resolveSafeTensorsOCIIdentity(ctx context.Context, resolver file.Resolver,
case cfg != nil: case cfg != nil:
applyHFConfig(md, cfg) applyHFConfig(md, cfg)
if configName == "" { if configName == "" {
configName = cfg.NameOrPath configName = sanitizeModelName(cfg.NameOrPath)
} }
supporting = append(supporting, loc) supporting = append(supporting, loc)
case fm != nil: case fm != nil:
if readmeLicense == "" { if len(readmeLicenses) == 0 {
readmeLicense = fm.License readmeLicenses = fm.Licenses
} }
if readmeName == "" && len(fm.BaseModel) > 0 { if readmeName == "" {
readmeName = fm.BaseModel[0] readmeName = firstModelName(fm.BaseModel)
} }
supporting = append(supporting, loc) supporting = append(supporting, loc)
} }
@@ -64,7 +73,7 @@ func resolveSafeTensorsOCIIdentity(ctx context.Context, resolver file.Resolver,
supporting: supporting, supporting: supporting,
} }
// License precedence: a dedicated vnd.docker.ai.license layer is a // License precedence: a dedicated vnd.docker.ai.license layer
// outranks the free-text license field in a model card's README frontmatter. // outranks the free-text license field in a model card's README frontmatter.
licLocs, err := ociResolver.FilesByMediaType(dockerAILicenseMediaType) licLocs, err := ociResolver.FilesByMediaType(dockerAILicenseMediaType)
if err != nil { if err != nil {
@@ -74,8 +83,8 @@ func resolveSafeTensorsOCIIdentity(ctx context.Context, resolver file.Resolver,
case len(licLocs) > 0: case len(licLocs) > 0:
id.licenses = identifyLicenseLayers(ctx, resolver, licLocs) id.licenses = identifyLicenseLayers(ctx, resolver, licLocs)
id.supporting = append(id.supporting, licLocs...) id.supporting = append(id.supporting, licLocs...)
case readmeLicense != "": case len(readmeLicenses) > 0:
id.licenses = pkg.NewLicensesFromValuesWithContext(ctx, readmeLicense) id.licenses = pkg.NewLicensesFromValuesWithContext(ctx, readmeLicenses...)
} }
return id return id
@@ -108,10 +117,11 @@ func ociImageRefBasename(resolver file.Resolver) string {
return path.Base(parsed.Context().RepositoryStr()) return path.Base(parsed.Context().RepositoryStr())
} }
// classifyOCIModelFileLayer reads up to 4 MiB of a model.file layer and decodes // classifyOCIModelFileLayer reads a model.file layer (up to maxModelFileLayerSize)
// it as either an HF config.json or a README model card's YAML frontmatter, // and decodes it as either an HF config.json or a README model card's YAML
// based on its leading bytes. It returns whichever it recognized; both are nil // frontmatter, based on its leading bytes. It returns whichever it recognized;
// when the layer is neither (or fails to decode). The caller owns precedence and // both are nil when the layer is neither, fails to decode, or is JSON that
// doesn't look like an HF config (e.g. generation_config.json). The caller owns precedence and
// metadata enrichment. // metadata enrichment.
func classifyOCIModelFileLayer(resolver file.Resolver, loc file.Location) (*hfConfig, *readmeFrontmatter) { func classifyOCIModelFileLayer(resolver file.Resolver, loc file.Location) (*hfConfig, *readmeFrontmatter) {
rc, err := resolver.FileContentsByLocation(loc) rc, err := resolver.FileContentsByLocation(loc)
@@ -120,17 +130,18 @@ func classifyOCIModelFileLayer(resolver file.Resolver, loc file.Location) (*hfCo
} }
defer internal.CloseAndLogError(rc, loc.RealPath) defer internal.CloseAndLogError(rc, loc.RealPath)
buf, err := io.ReadAll(io.LimitReader(rc, 4*1024*1024)) buf, err := readBounded(rc, maxModelFileLayerSize)
if err != nil { if err != nil {
log.Debugf("failed to read docker AI model-file layer: %v", err)
return nil, nil return nil, nil
} }
trimmed := bytes.TrimLeft(buf, "\xef\xbb\xbf \t\r\n") trimmed := bytes.TrimLeft(buf, "\xef\xbb\xbf \t\r\n")
switch { switch {
case bytes.HasPrefix(trimmed, []byte("---")): case bytes.HasPrefix(trimmed, []byte("---")):
return nil, parseFrontmatter(buf) return nil, parseFrontmatter(trimmed)
case bytes.HasPrefix(trimmed, []byte("{")): case bytes.HasPrefix(trimmed, []byte("{")):
var cfg hfConfig var cfg hfConfig
if err := json.Unmarshal(buf, &cfg); err != nil { if err := json.Unmarshal(trimmed, &cfg); err != nil || !cfg.looksLikeHF() {
return nil, nil return nil, nil
} }
return &cfg, nil return &cfg, nil
@@ -165,8 +176,9 @@ func readLicenseSPDXIDFromFrontmatter(resolver file.Resolver, loc file.Location)
} }
defer internal.CloseAndLogError(rc, loc.RealPath) defer internal.CloseAndLogError(rc, loc.RealPath)
buf, err := io.ReadAll(io.LimitReader(rc, 64*1024)) buf, err := readPrefix(rc, maxLicensePrefixSize)
if err != nil { if err != nil {
log.Debugf("failed to read docker AI license layer: %v", err)
return "" return ""
} }
return parseLicenseFrontmatter(buf) return parseLicenseFrontmatter(buf)
syft/pkg/cataloger/ai/limits.go
+46
View File
@@ -0,0 +1,46 @@
package ai
import (
"fmt"
"io"
"github.com/anchore/syft/internal"
)
// size limits for model companion files and headers. JSON reads that hit a
// limit fail (see readBounded) instead of parsing a truncated document.
// Frontmatter-only reads (README, license) take a prefix instead, since the
// frontmatter sits at the top and is itself capped by maxFrontmatterSize.
const (
maxHFConfigSize = 4 * 1024 * 1024
maxReadmePrefixSize = 1 * 1024 * 1024 // dir scans; OCI README layers use maxModelFileLayerSize
maxModelFileLayerSize = 4 * 1024 * 1024
maxLicensePrefixSize = 64 * 1024
maxModelConfigBlobSize = 1 * 1024 * 1024
maxFrontmatterSize = 256 * 1024
maxFrontmatterValues = 32
maxModelNameLength = 256
// maxSafeTensorsHeaderSize caps the header JSON (excluding its 8-byte length
// prefix). It is shared with the OCI source so dir and OCI scans accept the
// same headers.
maxSafeTensorsHeaderSize = internal.MaxSafeTensorsHeaderSize
)
// readPrefix reads at most limit bytes. It is for frontmatter-only reads, where
// the rest of the file is never needed.
func readPrefix(r io.Reader, limit int64) ([]byte, error) {
return io.ReadAll(io.LimitReader(r, limit))
}
// readBounded reads at most limit bytes and reports overflow instead of silently truncating.
func readBounded(r io.Reader, limit int64) ([]byte, error) {
b, err := io.ReadAll(io.LimitReader(r, limit+1))
if err != nil {
return nil, err
}
if int64(len(b)) > limit {
return nil, fmt.Errorf("content exceeds %d bytes", limit)
}
return b, nil
}
syft/pkg/cataloger/ai/merge.go
+1 -1
View File
@@ -75,7 +75,7 @@ func mergeShardsInto(merged *pkg.SafeTensorsModelInfo, shards []pkg.SafeTensorsM
seenKV := map[string]bool{} seenKV := map[string]bool{}
for _, s := range shards { for _, s := range shards {
shardTensorTotal += s.TensorCount shardTensorTotal += s.TensorCount
merged.Parameters += s.Parameters merged.Parameters = saturatingAdd(merged.Parameters, s.Parameters)
firstNonEmpty(&merged.Quantization, s.Quantization) firstNonEmpty(&merged.Quantization, s.Quantization)
for _, kv := range s.UserMetadata { for _, kv := range s.UserMetadata {
if seenKV[kv.Key] { if seenKV[kv.Key] {
syft/pkg/cataloger/ai/merge_test.go
+8
View File
@@ -1,6 +1,7 @@
package ai package ai
import ( import (
"math"
"testing" "testing"
"github.com/stretchr/testify/assert" "github.com/stretchr/testify/assert"
@@ -164,3 +165,10 @@ func TestRollupHash(t *testing.T) {
assert.NotEqual(t, "a", ab) assert.NotEqual(t, "a", ab)
assert.NotEqual(t, "b", ab) assert.NotEqual(t, "b", ab)
} }
func TestMergeShardsInto_parametersSaturate(t *testing.T) {
var merged pkg.SafeTensorsModelInfo
shards := []pkg.SafeTensorsModelInfo{{Parameters: math.MaxUint64 - 1}, {Parameters: 2}}
mergeShardsInto(&merged, shards)
assert.Equal(t, uint64(math.MaxUint64), merged.Parameters)
}
syft/pkg/cataloger/ai/naming.go
+32 -3
View File
@@ -1,15 +1,44 @@
package ai package ai
import "path" import (
"path"
"strings"
"unicode/utf8"
)
// pickSafeTensorsName implements the documented naming precedence chain: // pickSafeTensorsName implements the documented naming precedence chain:
func pickSafeTensorsName(nameOrPath, fallbackName string) string { func pickSafeTensorsName(nameOrPath, fallbackName string) string {
if nameOrPath != "" { if name := sanitizeModelName(nameOrPath); name != "" {
return path.Base(nameOrPath) return name
} }
return fallbackName return fallbackName
} }
// firstModelName returns the first value that sanitizes to a usable name.
func firstModelName(values []string) string {
for _, v := range values {
if name := sanitizeModelName(v); name != "" {
return name
}
}
return ""
}
// sanitizeModelName reduces a producer-supplied name or path (_name_or_path,
// base_model) to its last path element, capped at maxModelNameLength runes. It
// returns "" when nothing usable is left.
func sanitizeModelName(nameOrPath string) string {
base := path.Base(strings.ReplaceAll(nameOrPath, `\`, "/"))
switch base {
case "/", ".", "..", "":
return ""
}
if utf8.RuneCountInString(base) > maxModelNameLength {
base = string([]rune(base)[:maxModelNameLength])
}
return base
}
// safeTensorsDirName returns the directory-scan naming fallback: the base name // safeTensorsDirName returns the directory-scan naming fallback: the base name
// of the group's parent directory (the group key is already that directory). // of the group's parent directory (the group key is already that directory).
func safeTensorsDirName(directory string) string { func safeTensorsDirName(directory string) string {
syft/pkg/cataloger/ai/package.go
+4 -2
View File
@@ -1,17 +1,19 @@
package ai package ai
import ( import (
"context"
"github.com/anchore/syft/syft/file" "github.com/anchore/syft/syft/file"
"github.com/anchore/syft/syft/pkg" "github.com/anchore/syft/syft/pkg"
) )
func newGGUFPackage(metadata *pkg.GGUFFileHeader, modelName, version, license string, locations ...file.Location) pkg.Package { func newGGUFPackage(ctx context.Context, metadata *pkg.GGUFFileHeader, modelName, version, license string, locations ...file.Location) pkg.Package {
p := pkg.Package{ p := pkg.Package{
Name: modelName, Name: modelName,
Version: version, Version: version,
Locations: file.NewLocationSet(locations...), Locations: file.NewLocationSet(locations...),
Type: pkg.ModelPkg, Type: pkg.ModelPkg,
Licenses: pkg.NewLicenseSet(pkg.NewLicensesFromValues(license)...), Licenses: pkg.NewLicenseSet(pkg.NewLicensesFromValuesWithContext(ctx, license)...),
Metadata: *metadata, Metadata: *metadata,
// NOTE: PURL is intentionally not set as the package-url spec // NOTE: PURL is intentionally not set as the package-url spec
// has not yet finalized support for ML model packages // has not yet finalized support for ML model packages
syft/pkg/cataloger/ai/package_test.go
+2
View File
@@ -1,6 +1,7 @@
package ai package ai
import ( import (
"context"
"testing" "testing"
"github.com/stretchr/testify/require" "github.com/stretchr/testify/require"
@@ -103,6 +104,7 @@ func TestNewGGUFPackage(t *testing.T) {
for _, tt := range tests { for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) { t.Run(tt.name, func(t *testing.T) {
actual := newGGUFPackage( actual := newGGUFPackage(
context.Background(),
tt.metadata, tt.metadata,
tt.input.modelName, tt.input.modelName,
tt.input.version, tt.input.version,
syft/pkg/cataloger/ai/parse_gguf.go
+300 -5
View File
@@ -1,9 +1,14 @@
package ai package ai
import ( import (
"bufio"
"encoding/binary" "encoding/binary"
"errors"
"fmt" "fmt"
"io" "io"
"math"
"slices"
"strings"
gguf_parser "github.com/gpustack/gguf-parser-go" gguf_parser "github.com/gpustack/gguf-parser-go"
) )
@@ -12,12 +17,41 @@ import (
const ( const (
ggufMagicNumber = 0x46554747 // "GGUF" in little-endian ggufMagicNumber = 0x46554747 // "GGUF" in little-endian
maxHeaderSize = 50 * 1024 * 1024 // 50MB for large tokenizer vocabularies maxHeaderSize = 50 * 1024 * 1024 // 50MB for large tokenizer vocabularies
// maxGGUFArrayDepth allows arrays of arrays but nothing deeper.
maxGGUFArrayDepth = 2
// maxGGUFDecodedItems bounds the items, across the whole header, of arrays that gguf-parser-go
// always decodes in full. Real files hold a few per-layer arrays.
maxGGUFDecodedItems = 65536
// maxGGUFSplitTensors bounds split.tensors.count, which the library preallocates tensor infos for.
// Shards declare the total across all splits, so this can't be the header tensor count.
maxGGUFSplitTensors = 1 << 17
// maxGGUFKeyLength is the key length limit from the GGUF spec.
maxGGUFKeyLength = 65535
// maxGGUFKVCount bounds the metadata KV count. Real files hold hundreds; vocabularies live in arrays.
maxGGUFKVCount = 65536
// maxGGUFReadStringBytes bounds, across the header, the key and string bytes the library reads in
// full and syft then hashes and encodes (chat templates are the largest, at tens of KB).
maxGGUFReadStringBytes = 8 << 20
// maxGGUFSkippedStrings bounds, across the header, the strings the library skips one at a time
// inside arrays (vocabularies run to a few hundred thousand tokens and merges).
maxGGUFSkippedStrings = 1 << 22
) )
// copyHeader copies the GGUF header from the reader to the writer. // ggufDecodedArraySuffixes are the keys gguf-parser-go decodes in full even with SkipLargeMetadata
// (hardcoded in its ReadArray as of v0.26.3; re-check on upgrade).
var ggufDecodedArraySuffixes = []string{
".feed_forward_length", ".attention.head_count", ".attention.head_count_kv", ".attention.sliding_window_pattern",
}
// ggufScalarSizes is the encoded size of each value type, indexed by type (0 for strings and arrays).
var ggufScalarSizes = [...]int{1, 1, 2, 2, 4, 4, 4, 1, 0, 0, 8, 8, 8}
// copyHeader copies at most maxHeaderSize bytes of the GGUF header from the reader to the writer.
// It validates the magic number first, then copies the rest of the data. // It validates the magic number first, then copies the rest of the data.
// The reader should be wrapped with io.LimitedReader to prevent OOM issues.
func copyHeader(w io.Writer, r io.Reader) error { func copyHeader(w io.Writer, r io.Reader) error {
r = io.LimitReader(r, maxHeaderSize)
// Read initial chunk to validate magic number // Read initial chunk to validate magic number
// GGUF format: magic(4) + version(4) + tensor_count(8) + metadata_kv_count(8) + metadata_kvs + tensors_info // GGUF format: magic(4) + version(4) + tensor_count(8) + metadata_kv_count(8) + metadata_kvs + tensors_info
initialBuf := make([]byte, 24) // Enough for magic, version, tensor count, and kv count initialBuf := make([]byte, 24) // Enough for magic, version, tensor count, and kv count
@@ -37,7 +71,6 @@ func copyHeader(w io.Writer, r io.Reader) error {
} }
// Copy the rest of the header from reader to writer // Copy the rest of the header from reader to writer
// The LimitedReader will return EOF once maxHeaderSize is reached
if _, err := io.Copy(w, r); err != nil { if _, err := io.Copy(w, r); err != nil {
return fmt.Errorf("failed to copy GGUF header: %w", err) return fmt.Errorf("failed to copy GGUF header: %w", err)
} }
@@ -45,7 +78,269 @@ func copyHeader(w io.Writer, r io.Reader) error {
return nil return nil
} }
// Helper to convert gguf_parser metadata to simpler types // validateGGUFHeader walks the metadata KV section and rejects values that gguf-parser-go would panic
// on, recurse on without bound, or overallocate for. It reads without storing anything.
// ponytail: tensor infos are not checked here (zero block size types, dimension overflow); those
// panics are recoverable and are left to the per-file panic recovery in the generic cataloger.
func validateGGUFHeader(r io.Reader) error {
v := &ggufHeaderValidator{
r: bufio.NewReader(r),
decodedBudget: maxGGUFDecodedItems,
readStringBudget: maxGGUFReadStringBytes,
skippedStringBudget: maxGGUFSkippedStrings,
}
var prefix struct {
Magic, Version uint32
}
if err := binary.Read(v.r, binary.LittleEndian, &prefix); err != nil {
return fmt.Errorf("reading header prefix: %w", err)
}
v.v1 = prefix.Version <= 1
// the tensor count is validated by the library
if _, err := v.readLen(); err != nil {
return fmt.Errorf("reading tensor count: %w", err)
}
kvCount, err := v.readLen()
if err != nil {
return fmt.Errorf("reading metadata count: %w", err)
}
if kvCount > maxGGUFKVCount {
return fmt.Errorf("metadata count %d exceeds %d", kvCount, maxGGUFKVCount)
}
for i := uint64(0); i < kvCount; i++ {
key, err := v.readKey()
if err != nil {
return fmt.Errorf("metadata key %d: %w", i, err)
}
var vt uint32
if err := binary.Read(v.r, binary.LittleEndian, &vt); err != nil {
return fmt.Errorf("metadata key %q: %w", key, err)
}
if err := v.checkKey(key, gguf_parser.GGUFMetadataValueType(vt)); err != nil {
return fmt.Errorf("metadata key %q: %w", key, err)
}
}
return nil
}
type ggufHeaderValidator struct {
r *bufio.Reader
v1 bool // v1 uses 32-bit lengths
decodedBudget uint64
readStringBudget uint64
skippedStringBudget uint64
}
func (v *ggufHeaderValidator) checkKey(key string, vt gguf_parser.GGUFMetadataValueType) error {
decoded := slices.ContainsFunc(ggufDecodedArraySuffixes, func(s string) bool { return strings.HasSuffix(key, s) })
n, err := v.readValue(vt, decoded, 1)
if err != nil {
return err
}
// these are read with typed accessors that panic on a type mismatch
switch key {
case "general.architecture", "general.name", "general.license", "general.type", "general.author",
"general.url", "general.description", "controlvector.model_hint":
if vt != gguf_parser.GGUFMetadataValueTypeString {
return fmt.Errorf("expected string, got %v", vt)
}
case "general.quantization_version", "general.file_type":
if !vt.IsNumeric() {
return fmt.Errorf("expected number, got %v", vt)
}
case "general.alignment":
if vt != gguf_parser.GGUFMetadataValueTypeUint32 || n == 0 {
return fmt.Errorf("expected non-zero uint32, got %v %v", vt, n)
}
case "split.tensors.count":
if !vt.IsNumeric() {
return fmt.Errorf("expected number, got %v", vt)
}
if math.IsNaN(n) || n < 0 || n > maxGGUFSplitTensors {
return fmt.Errorf("unreasonable split tensor count %.0f", n)
}
}
return nil
}
// readValue consumes one value, returning scalar numbers as float64 for the key checks.
func (v *ggufHeaderValidator) readValue(vt gguf_parser.GGUFMetadataValueType, decoded bool, depth int) (float64, error) {
if uint32(vt) >= uint32(len(ggufScalarSizes)) {
return 0, fmt.Errorf("invalid value type %d", vt)
}
switch vt {
case gguf_parser.GGUFMetadataValueTypeString:
// only strings inside skipped arrays are skipped by the library, the rest are read in full
return 0, v.skipString(depth > 1 && !decoded)
case gguf_parser.GGUFMetadataValueTypeArray:
return 0, v.readArray(decoded, depth)
}
size := ggufScalarSizes[vt]
b, err := v.r.Peek(size)
if err != nil {
return 0, err
}
var buf [8]byte
copy(buf[:], b)
if _, err := v.r.Discard(size); err != nil {
return 0, err
}
u := binary.LittleEndian.Uint64(buf[:])
switch vt {
case gguf_parser.GGUFMetadataValueTypeInt8:
return float64(int8(u)), nil
case gguf_parser.GGUFMetadataValueTypeInt16:
return float64(int16(u)), nil
case gguf_parser.GGUFMetadataValueTypeInt32:
return float64(int32(u)), nil
case gguf_parser.GGUFMetadataValueTypeInt64:
return float64(int64(u)), nil
case gguf_parser.GGUFMetadataValueTypeFloat32:
return float64(math.Float32frombits(uint32(u))), nil
case gguf_parser.GGUFMetadataValueTypeFloat64:
return math.Float64frombits(u), nil
}
return float64(u), nil
}
func (v *ggufHeaderValidator) readArray(decoded bool, depth int) error {
var itemType uint32
if err := binary.Read(v.r, binary.LittleEndian, &itemType); err != nil {
return err
}
n, err := v.readLen()
if err != nil {
return err
}
it := gguf_parser.GGUFMetadataValueType(itemType)
switch {
case itemType >= uint32(len(ggufScalarSizes)):
return fmt.Errorf("invalid array item type %d", itemType)
case decoded && n > v.decodedBudget:
return fmt.Errorf("decoded arrays exceed %d items", maxGGUFDecodedItems)
case it == gguf_parser.GGUFMetadataValueTypeArray && (!decoded || depth >= maxGGUFArrayDepth):
// the library can only skip over arrays of scalars
return errors.New("unsupported nested array")
case n > maxHeaderSize:
return fmt.Errorf("array length %d exceeds header size", n)
case !decoded && ggufScalarSizes[itemType] > 0:
// skipped arrays of fixed size items can be passed over in one go
_, err := v.r.Discard(int(n) * ggufScalarSizes[itemType])
return err
}
if decoded {
v.decodedBudget -= n
}
for i := uint64(0); i < n; i++ {
if _, err := v.readValue(it, decoded, depth+1); err != nil {
return err
}
}
return nil
}
func (v *ggufHeaderValidator) readLen() (uint64, error) {
if v.v1 {
var n uint32
err := binary.Read(v.r, binary.LittleEndian, &n)
return uint64(n), err
}
var n uint64
err := binary.Read(v.r, binary.LittleEndian, &n)
return n, err
}
// readKey reads a key the way gguf-parser-go will see it, whitespace trimmed.
func (v *ggufHeaderValidator) readKey() (string, error) {
n, err := v.readLen()
if err != nil {
return "", err
}
if n == 0 || n > maxGGUFKeyLength {
return "", fmt.Errorf("invalid key length %d", n)
}
if err := v.spendReadString(n); err != nil {
return "", err
}
buf := make([]byte, n)
if _, err := io.ReadFull(v.r, buf); err != nil {
return "", err
}
return strings.TrimSpace(string(buf)), nil
}
func (v *ggufHeaderValidator) skipString(allowEmpty bool) error {
n, err := v.readLen()
if err != nil {
return err
}
// gguf-parser-go reads an empty string as a full pool buffer, so everything after it would be
// parsed out of step with this walk
if n == 0 && !allowEmpty {
return errors.New("empty string")
}
if n > maxHeaderSize {
return fmt.Errorf("string length %d exceeds header size", n)
}
if allowEmpty {
if v.skippedStringBudget == 0 {
return fmt.Errorf("skipped strings exceed %d", maxGGUFSkippedStrings)
}
v.skippedStringBudget--
} else if err := v.spendReadString(n); err != nil {
return err
}
_, err = v.r.Discard(int(n))
return err
}
func (v *ggufHeaderValidator) spendReadString(n uint64) error {
if n > v.readStringBudget {
return fmt.Errorf("strings exceed %d bytes", maxGGUFReadStringBytes)
}
v.readStringBudget -= n
return nil
}
// sanitizeGGUFValue replaces non-finite floats, which JSON can not encode, leaving everything else as is.
func sanitizeGGUFValue(v any) any {
switch t := v.(type) {
case float32:
if f := float64(t); math.IsNaN(f) || math.IsInf(f, 0) {
return nonFiniteString(f)
}
case float64:
if math.IsNaN(t) || math.IsInf(t, 0) {
return nonFiniteString(t)
}
case gguf_parser.GGUFMetadataKVArrayValue:
// keep the library shape as is; items are only present for the few keys the library
// decodes in full, which the pre-scan keeps small and shallow
if len(t.Array) > 0 {
items := make([]any, len(t.Array))
for i, item := range t.Array {
items[i] = sanitizeGGUFValue(item)
}
t.Array = items
}
return t
}
return v
}
func nonFiniteString(f float64) string {
switch {
case math.IsInf(f, 1):
return "+Inf"
case math.IsInf(f, -1):
return "-Inf"
}
return "NaN"
}
// helper to convert gguf_parser metadata to simpler types
func convertGGUFMetadataKVs(kvs gguf_parser.GGUFMetadataKVs) map[string]any { func convertGGUFMetadataKVs(kvs gguf_parser.GGUFMetadataKVs) map[string]any {
result := make(map[string]any) result := make(map[string]any)
@@ -56,7 +351,7 @@ func convertGGUFMetadataKVs(kvs gguf_parser.GGUFMetadataKVs) map[string]any {
"general.version", "general.parameter_count", "general.quantization": "general.version", "general.parameter_count", "general.quantization":
continue continue
} }
result[kv.Key] = kv.Value result[kv.Key] = sanitizeGGUFValue(kv.Value)
} }
return result return result
syft/pkg/cataloger/ai/parse_gguf_model.go
+26 -14
View File
@@ -13,9 +13,7 @@ import (
gguf_parser "github.com/gpustack/gguf-parser-go" gguf_parser "github.com/gpustack/gguf-parser-go"
"github.com/anchore/syft/internal" "github.com/anchore/syft/internal"
"github.com/anchore/syft/internal/log"
"github.com/anchore/syft/internal/tmpdir" "github.com/anchore/syft/internal/tmpdir"
"github.com/anchore/syft/internal/unknown"
"github.com/anchore/syft/syft/artifact" "github.com/anchore/syft/syft/artifact"
"github.com/anchore/syft/syft/file" "github.com/anchore/syft/syft/file"
"github.com/anchore/syft/syft/pkg" "github.com/anchore/syft/syft/pkg"
@@ -38,14 +36,20 @@ func parseGGUFModel(ctx context.Context, _ file.Resolver, _ *generic.Environment
defer cleanup() defer cleanup()
tempPath := tempFile.Name() tempPath := tempFile.Name()
// Copy and validate the GGUF file header using LimitedReader to prevent OOM // copy and validate the GGUF file header (bounded to maxHeaderSize)
// We use LimitedReader to cap reads at maxHeaderSize (50MB) if err := copyHeader(tempFile, reader); err != nil {
limitedReader := &io.LimitedReader{R: reader, N: maxHeaderSize}
if err := copyHeader(tempFile, limitedReader); err != nil {
tempFile.Close() tempFile.Close()
return nil, nil, fmt.Errorf("failed to copy GGUF header: %w", err) return nil, nil, fmt.Errorf("failed to copy GGUF header: %w", err)
} }
if _, err := tempFile.Seek(0, io.SeekStart); err != nil {
tempFile.Close()
return nil, nil, fmt.Errorf("failed to rewind GGUF header: %w", err)
}
err = validateGGUFHeader(tempFile)
tempFile.Close() tempFile.Close()
if err != nil {
return nil, nil, fmt.Errorf("invalid GGUF header: %w", err)
}
// Parse using gguf-parser-go with options to skip unnecessary data // Parse using gguf-parser-go with options to skip unnecessary data
ggufFile, err := gguf_parser.ParseGGUFFile(tempPath, ggufFile, err := gguf_parser.ParseGGUFFile(tempPath,
@@ -61,6 +65,11 @@ func parseGGUFModel(ctx context.Context, _ file.Resolver, _ *generic.Environment
// Extract version separately (will be set on Package.Version) // Extract version separately (will be set on Package.Version)
modelVersion := extractVersion(ggufFile.Header.MetadataKV) modelVersion := extractVersion(ggufFile.Header.MetadataKV)
metadataHash, err := computeKVMetadataHash(ggufFile.Header.MetadataKV)
if err != nil {
return nil, nil, err
}
// Convert to syft metadata structure // Convert to syft metadata structure
syftMetadata := &pkg.GGUFFileHeader{ syftMetadata := &pkg.GGUFFileHeader{
Architecture: metadata.Architecture, Architecture: metadata.Architecture,
@@ -69,7 +78,7 @@ func parseGGUFModel(ctx context.Context, _ file.Resolver, _ *generic.Environment
GGUFVersion: uint32(ggufFile.Header.Version), GGUFVersion: uint32(ggufFile.Header.Version),
TensorCount: ggufFile.Header.TensorCount, TensorCount: ggufFile.Header.TensorCount,
RemainingKeyValues: convertGGUFMetadataKVs(ggufFile.Header.MetadataKV), RemainingKeyValues: convertGGUFMetadataKVs(ggufFile.Header.MetadataKV),
MetadataKeyValuesHash: computeKVMetadataHash(ggufFile.Header.MetadataKV), MetadataKeyValuesHash: metadataHash,
} }
// If model name is not in metadata, use filename // If model name is not in metadata, use filename
@@ -79,6 +88,7 @@ func parseGGUFModel(ctx context.Context, _ file.Resolver, _ *generic.Environment
// Create package from metadata // Create package from metadata
p := newGGUFPackage( p := newGGUFPackage(
ctx,
syftMetadata, syftMetadata,
metadata.Name, metadata.Name,
modelVersion, modelVersion,
@@ -86,14 +96,17 @@ func parseGGUFModel(ctx context.Context, _ file.Resolver, _ *generic.Environment
reader.WithAnnotation(pkg.EvidenceAnnotationKey, pkg.PrimaryEvidenceAnnotation), reader.WithAnnotation(pkg.EvidenceAnnotationKey, pkg.PrimaryEvidenceAnnotation),
) )
return []pkg.Package{p}, nil, unknown.IfEmptyf([]pkg.Package{p}, "unable to parse GGUF file") return []pkg.Package{p}, nil, nil
} }
// computeKVMetadataHash computes a stable hash of the KV metadata for use as a global identifier // computeKVMetadataHash computes a stable hash of the sanitized KV metadata for use as a global identifier
func computeKVMetadataHash(metadata gguf_parser.GGUFMetadataKVs) string { func computeKVMetadataHash(metadata gguf_parser.GGUFMetadataKVs) (string, error) {
// Sort the KV pairs by key for stable hashing // Sort the KV pairs by key for stable hashing
sortedKVs := make([]gguf_parser.GGUFMetadataKV, len(metadata)) sortedKVs := make([]gguf_parser.GGUFMetadataKV, len(metadata))
copy(sortedKVs, metadata) for i, kv := range metadata {
kv.Value = sanitizeGGUFValue(kv.Value)
sortedKVs[i] = kv
}
sort.Slice(sortedKVs, func(i, j int) bool { sort.Slice(sortedKVs, func(i, j int) bool {
return sortedKVs[i].Key < sortedKVs[j].Key return sortedKVs[i].Key < sortedKVs[j].Key
}) })
@@ -101,13 +114,12 @@ func computeKVMetadataHash(metadata gguf_parser.GGUFMetadataKVs) string {
// Marshal sorted KVs to JSON for stable hashing // Marshal sorted KVs to JSON for stable hashing
jsonBytes, err := json.Marshal(sortedKVs) jsonBytes, err := json.Marshal(sortedKVs)
if err != nil { if err != nil {
log.Debugf("failed to marshal metadata for hashing: %v", err) return "", fmt.Errorf("failed to marshal GGUF metadata for hashing: %w", err)
return ""
} }
// Compute xxhash // Compute xxhash
hash := xxhash.Sum64(jsonBytes) hash := xxhash.Sum64(jsonBytes)
return fmt.Sprintf("%016x", hash) // 16 hex chars (64 bits) return fmt.Sprintf("%016x", hash), nil // 16 hex chars (64 bits)
} }
// extractVersion attempts to extract version from metadata KV pairs // extractVersion attempts to extract version from metadata KV pairs
syft/pkg/cataloger/ai/parse_gguf_test.go
+251
View File
@@ -0,0 +1,251 @@
package ai
import (
"bytes"
"context"
"encoding/binary"
"encoding/json"
"io"
"math"
"strings"
"testing"
gguf_parser "github.com/gpustack/gguf-parser-go"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/anchore/syft/internal/tmpdir"
"github.com/anchore/syft/syft/file"
"github.com/anchore/syft/syft/format/syftjson"
"github.com/anchore/syft/syft/pkg"
"github.com/anchore/syft/syft/sbom"
)
func le(v any) []byte {
buf := new(bytes.Buffer)
binary.Write(buf, binary.LittleEndian, v)
return buf.Bytes()
}
func ggufString(s string) []byte {
return append(le(uint64(len(s))), s...)
}
func parseGGUFBytes(t *testing.T, data []byte) ([]pkg.Package, error) {
t.Helper()
ctx, td := tmpdir.Root(context.Background(), "gguf-test")
t.Cleanup(func() { _ = td.Cleanup() })
reader := file.NewLocationReadCloser(file.NewLocation("/model.gguf"), io.NopCloser(bytes.NewReader(data)))
var pkgs []pkg.Package
var err error
require.NotPanics(t, func() {
pkgs, _, err = parseGGUFModel(ctx, nil, nil, reader)
})
return pkgs, err
}
func TestParseGGUFModel_malformedHeader(t *testing.T) {
// nested arrays of arrays, each one level deeper, ending in an empty uint8 array
nested := func(depth int) []byte {
v := ggufArray(ggufTypeUint8, 0, nil)
for i := 1; i < depth; i++ {
v = ggufArray(ggufTypeArray, 1, v)
}
return v
}
// a header that declares more KVs than the cap, with none following
tooManyKVs := append(append([]byte("GGUF"), le(uint32(3))...), le([]uint64{0, maxGGUFKVCount + 1})...)
tests := []struct {
name string
builder *testGGUFBuilder
data []byte // used instead of builder when set
wantErr bool
}{
{name: "too many KVs", data: tooManyKVs, wantErr: true},
{name: "split tensor count is a string", builder: newTestGGUFBuilder().withStringKV("split.tensors.count", "1"), wantErr: true},
{name: "split tensor count over the cap", builder: newTestGGUFBuilder().withUint32KV("split.tensors.count", maxGGUFSplitTensors+1), wantErr: true},
{name: "read strings too long in total", builder: newTestGGUFBuilder().withStringKV("x", strings.Repeat("a", maxGGUFReadStringBytes)), wantErr: true},
{
name: "skipped strings too many in total",
builder: newTestGGUFBuilder().withRawKV("tokenizer.ggml.tokens", ggufTypeArray, ggufArray(ggufTypeString, maxGGUFSkippedStrings+1, make([]byte, 8*(maxGGUFSkippedStrings+1)))),
wantErr: true,
},
{name: "v1 header", builder: newTestGGUFBuilder().withVersion(1).withStringKV("general.name", "m").withUint32KV("llama.context_length", 8)},
{name: "architecture not a string", builder: newTestGGUFBuilder().withUint32KV("general.architecture", 1), wantErr: true},
{name: "name not a string", builder: newTestGGUFBuilder().withUint32KV("general.name", 1), wantErr: true},
{name: "license not a string", builder: newTestGGUFBuilder().withUint32KV("general.license", 1), wantErr: true},
{name: "type not a string", builder: newTestGGUFBuilder().withUint32KV("general.type", 1), wantErr: true},
{name: "author not a string", builder: newTestGGUFBuilder().withUint32KV("general.author", 1), wantErr: true},
{name: "url not a string", builder: newTestGGUFBuilder().withUint32KV("general.url", 1), wantErr: true},
{name: "description not a string", builder: newTestGGUFBuilder().withUint32KV("general.description", 1), wantErr: true},
{name: "model hint not a string", builder: newTestGGUFBuilder().withUint32KV("controlvector.model_hint", 1), wantErr: true},
{name: "quantization version is a string", builder: newTestGGUFBuilder().withStringKV("general.quantization_version", "2"), wantErr: true},
{name: "alignment is a string", builder: newTestGGUFBuilder().withStringKV("general.alignment", "32"), wantErr: true},
{name: "alignment is uint64", builder: newTestGGUFBuilder().withUint64KV("general.alignment", 32), wantErr: true},
{name: "alignment is zero", builder: newTestGGUFBuilder().withUint32KV("general.alignment", 0), wantErr: true},
{
name: "array of arrays on a skipped key",
builder: newTestGGUFBuilder().withRawKV("tokenizer.ggml.merges", ggufTypeArray, ggufArray(ggufTypeArray, 1, ggufArray(ggufTypeUint8, 0, nil))),
wantErr: true,
},
{
name: "invalid array item type",
builder: newTestGGUFBuilder().withRawKV("tokenizer.ggml.scores", ggufTypeArray, ggufArray(99, 0, nil)),
wantErr: true,
},
{name: "split tensor count out of range", builder: newTestGGUFBuilder().withUint64KV("split.tensors.count", 1<<62), wantErr: true},
{name: "split tensor count huge", builder: newTestGGUFBuilder().withUint64KV("split.tensors.count", 1<<40), wantErr: true},
{name: "split tensor count negative", builder: newTestGGUFBuilder().withRawKV("split.tensors.count", ggufTypeInt64, le(int64(-1))), wantErr: true},
{
name: "nesting too deep on a decoded key",
builder: newTestGGUFBuilder().withRawKV("llama.attention.head_count", ggufTypeArray, nested(3)),
wantErr: true,
},
{
name: "decoded array too long",
builder: newTestGGUFBuilder().withRawKV("llama.feed_forward_length", ggufTypeArray, ggufArray(ggufTypeUint8, maxGGUFDecodedItems+1, make([]byte, maxGGUFDecodedItems+1))),
wantErr: true,
},
{name: "empty string value", builder: newTestGGUFBuilder().withStringKV("general.author", ""), wantErr: true},
{name: "padded key", builder: newTestGGUFBuilder().withUint32KV(" general.alignment\t", 0), wantErr: true},
{name: "long padded key", builder: newTestGGUFBuilder().withUint32KV(strings.Repeat(" ", 300)+"general.alignment", 0), wantErr: true},
{name: "file type is a string", builder: newTestGGUFBuilder().withUint32KV("general.file_type", 1).withStringKV("general.file_type", "x"), wantErr: true},
{
name: "decoded arrays too long in total",
builder: newTestGGUFBuilder().
withRawKV("a.feed_forward_length", ggufTypeArray, ggufArray(ggufTypeUint8, maxGGUFDecodedItems, make([]byte, maxGGUFDecodedItems))).
withRawKV("b.feed_forward_length", ggufTypeArray, ggufArray(ggufTypeUint8, 1, make([]byte, 1))),
wantErr: true,
},
{name: "invalid value type", builder: newTestGGUFBuilder().withRawKV("x", 0xFFFFFFFF, nil), wantErr: true},
{name: "empty key", builder: newTestGGUFBuilder().withUint32KV("", 1), wantErr: true},
{
name: "empty string in a decoded array",
builder: newTestGGUFBuilder().withRawKV("llama.attention.head_count", ggufTypeArray, ggufArray(ggufTypeString, 1, le(uint64(0)))),
wantErr: true,
},
{
name: "empty string in a skipped array",
builder: newTestGGUFBuilder().withRawKV("tokenizer.ggml.tokens", ggufTypeArray, ggufArray(ggufTypeString, 1, le(uint64(0)))),
},
{
name: "array of arrays on a decoded key",
builder: newTestGGUFBuilder().withRawKV("llama.attention.head_count", ggufTypeArray, nested(2)),
},
{name: "alignment is a uint32", builder: newTestGGUFBuilder().withUint32KV("general.alignment", 32)},
{
name: "split keys as written by llama.cpp",
builder: newTestGGUFBuilder().
withRawKV("split.no", ggufTypeUint16, le(uint16(0))).
withRawKV("split.count", ggufTypeUint16, le(uint16(3))).
withRawKV("split.tensors.count", ggufTypeInt32, le(int32(3000))),
},
{
name: "split shard declares the total tensor count",
builder: newTestGGUFBuilder().withRawKV("split.tensors.count", ggufTypeInt32, le(int32(3000))),
},
{
name: "skipped scalar array",
builder: newTestGGUFBuilder().withRawKV("tokenizer.ggml.scores", ggufTypeArray, ggufArray(ggufTypeFloat32, 3, make([]byte, 12))),
},
// ponytail: zero block size tensor types and dimension overflow panic inside tensor info parsing;
// those are left to the generic cataloger's panic handling rather than checked here.
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
data := tt.data
if data == nil {
data = tt.builder.withStringKV("general.architecture", "llama").build()
}
pkgs, err := parseGGUFBytes(t, data)
if tt.wantErr {
// the pre-scan has to be what rejects these, before the library sees them
require.ErrorContains(t, err, "invalid GGUF header")
assert.Empty(t, pkgs)
return
}
require.NoError(t, err)
require.Len(t, pkgs, 1)
})
}
}
func TestParseGGUFModel_nonFiniteFloat(t *testing.T) {
data := newTestGGUFBuilder().
withStringKV("general.architecture", "llama").
withRawKV("llama.rope.freq_base", ggufTypeFloat32, le(float32(math.Inf(1)))).
withRawKV("llama.rope.scale", ggufTypeFloat64, le(math.Inf(-1))).
withRawKV("llama.attention.head_count", ggufTypeArray, ggufArray(ggufTypeFloat32, 2, le([]float32{8, float32(math.NaN())}))).
withRawKV("tokenizer.ggml.scores", ggufTypeArray, ggufArray(ggufTypeFloat32, 1, le(float32(0)))).
build()
pkgs, err := parseGGUFBytes(t, data)
require.NoError(t, err)
require.Len(t, pkgs, 1)
meta := pkgs[0].Metadata.(pkg.GGUFFileHeader)
assert.NotEmpty(t, meta.MetadataKeyValuesHash)
assert.Equal(t, "+Inf", meta.RemainingKeyValues["llama.rope.freq_base"])
assert.Equal(t, "-Inf", meta.RemainingKeyValues["llama.rope.scale"])
headCount := meta.RemainingKeyValues["llama.attention.head_count"].(gguf_parser.GGUFMetadataKVArrayValue)
assert.Equal(t, []any{float32(8), "NaN"}, headCount.Array)
var buf bytes.Buffer
s := sbom.SBOM{Artifacts: sbom.Artifacts{Packages: pkg.NewCollection(pkgs...)}}
require.NoError(t, syftjson.NewFormatEncoder().Encode(&buf, s))
assert.NotEmpty(t, buf.Bytes())
}
// TestParseGGUFModel_outputUnchanged pins the header output and hash for ordinary values to what syft
// produced as of f7f3c6efc, before malformed header handling. The expected values were captured by
// running this same fixture on that commit. float32 0.1 catches any widening to float64.
func TestParseGGUFModel_outputUnchanged(t *testing.T) {
data := newTestGGUFBuilder().
withStringKV("general.architecture", "llama").
withRawKV("tokenizer.ggml.tokens", ggufTypeArray, ggufArray(ggufTypeString, 2, append(ggufString("a"), ggufString("b")...))).
withRawKV("llama.attention.head_count", ggufTypeArray, ggufArray(ggufTypeFloat32, 2, le([]float32{0.1, 0.2}))).
withRawKV("llama.feed_forward_length", ggufTypeArray, ggufArray(ggufTypeArray, 2, append(
ggufArray(ggufTypeUint32, 2, le([]uint32{1, 2})),
ggufArray(ggufTypeUint32, 1, le([]uint32{3}))...))).
withRawKV("llama.rope.freq_base", ggufTypeFloat32, le(float32(0.1))).
build()
pkgs, err := parseGGUFBytes(t, data)
require.NoError(t, err)
require.Len(t, pkgs, 1)
meta := pkgs[0].Metadata.(pkg.GGUFFileHeader)
assert.Equal(t, "4f28d55f0985a985", meta.MetadataKeyValuesHash)
header, err := json.Marshal(meta.RemainingKeyValues)
require.NoError(t, err)
assert.JSONEq(t, `{
"llama.attention.head_count": {"type": 6, "len": 2, "array": [0.1, 0.2], "startOffset": 170, "size": 8},
"llama.feed_forward_length": {"type": 9, "len": 2, "array": [
{"type": 4, "len": 2, "array": [1, 2], "startOffset": 239, "size": 8},
{"type": 4, "len": 1, "array": [3], "startOffset": 259, "size": 4}
], "startOffset": 227, "size": 36},
"llama.rope.freq_base": 0.1,
"tokenizer.ggml.tokens": {"type": 8, "len": 2, "startOffset": 102, "size": 18}
}`, string(header))
}
// TestCopyHeader_appliesItsOwnBound pins that copyHeader caps the copy at maxHeaderSize itself,
// whatever reader it is handed.
//
// Bytes retained is asserted directly rather than through testutils.MeasureAlloc: the destination is a
// bytes.Buffer, whose doubling growth costs a multiple of what it holds and lands on a different power
// of two depending on the platform. buf.Len() is the same property without that noise.
func TestCopyHeader_appliesItsOwnBound(t *testing.T) {
const payload = 2*maxHeaderSize + 1
file := make([]byte, 24, 24+payload)
binary.LittleEndian.PutUint32(file[0:4], ggufMagicNumber)
file = append(file, bytes.Repeat([]byte{'x'}, payload)...)
var buf bytes.Buffer
require.NoError(t, copyHeader(&buf, bytes.NewReader(file)))
assert.Equal(t, maxHeaderSize, buf.Len(), "copyHeader must cap the copy at maxHeaderSize")
}
syft/pkg/cataloger/ai/parse_safetensors.go
+52 -32
View File
@@ -5,6 +5,8 @@ import (
"encoding/json" "encoding/json"
"fmt" "fmt"
"io" "io"
"math"
"math/bits"
"sort" "sort"
"strings" "strings"
@@ -15,9 +17,7 @@ import (
// SafeTensors file format: [8 bytes u64 LE header size] [N bytes JSON header] [tensor data]. // SafeTensors file format: [8 bytes u64 LE header size] [N bytes JSON header] [tensor data].
// Reference: https://github.com/huggingface/safetensors#format // Reference: https://github.com/huggingface/safetensors#format
const ( // The header JSON is capped at maxSafeTensorsHeaderSize (see limits.go).
maxSafeTensorsHeaderSize = 100 * 1024 * 1024 // 100MB ceiling on header JSON to prevent OOM
)
// safeTensorsHeader is the decoded JSON header. Tensor entries live alongside a // safeTensorsHeader is the decoded JSON header. Tensor entries live alongside a
// reserved "__metadata__" key holding a string-to-string producer map. We decode // reserved "__metadata__" key holding a string-to-string producer map. We decode
@@ -84,47 +84,67 @@ func readSafeTensorsHeader(r io.Reader) (*safeTensorsHeader, error) {
return h, nil return h, nil
} }
// parameterCount sums the element counts across all tensors in the header. // modelInfo returns the metadata derivable from the header bytes alone. Naming,
func (h *safeTensorsHeader) parameterCount() uint64 { // licenses and ShardCount are left to the merge processor.
var total uint64 func (h *safeTensorsHeader) modelInfo() pkg.SafeTensorsModelInfo {
for _, t := range h.tensors { params, dtype := h.parameterStats()
count := uint64(1) return pkg.SafeTensorsModelInfo{
for _, dim := range t.Shape { Format: "safetensors",
if dim <= 0 { TensorCount: uint64(len(h.tensors)),
count = 0 Parameters: params,
break Quantization: normalizeDType(dtype),
} UserMetadata: userMetadataKeyValues(h.metadata),
count *= uint64(dim) MetadataHash: h.metadataHash(),
}
total += count
} }
return total
} }
// dominantDType returns the dtype that accounts for the largest fraction of parameters. // parameterStats sums the element counts across all tensors and returns the
// For mixed-precision models the "dominant" dtype is still a useful summary. // dtype that accounts for the largest share of them. For mixed-precision models
func (h *safeTensorsHeader) dominantDType() string { // the "dominant" dtype is still a useful summary. Totals saturate rather than wrap.
func (h *safeTensorsHeader) parameterStats() (total uint64, dominantDType string) {
sizeByDType := make(map[string]uint64) sizeByDType := make(map[string]uint64)
for _, t := range h.tensors { for _, t := range h.tensors {
count := uint64(1) count, ok := tensorElements(t.Shape)
for _, dim := range t.Shape { if !ok {
if dim <= 0 { continue
count = 0
break
}
count *= uint64(dim)
} }
sizeByDType[t.DType] += count total = saturatingAdd(total, count)
sizeByDType[t.DType] = saturatingAdd(sizeByDType[t.DType], count)
} }
var best string
var bestSize uint64 var bestSize uint64
for dtype, size := range sizeByDType { for dtype, size := range sizeByDType {
if size > bestSize || (size == bestSize && dtype < best) { if size > bestSize || (size == bestSize && dtype < dominantDType) {
best = dtype dominantDType = dtype
bestSize = size bestSize = size
} }
} }
return best return total, dominantDType
}
// tensorElements returns the element count for a shape, or ok=false on a
// non-positive dim or when the product overflows uint64.
func tensorElements(shape []int64) (uint64, bool) {
n := uint64(1)
for _, d := range shape {
if d <= 0 {
return 0, false
}
hi, lo := bits.Mul64(n, uint64(d))
if hi != 0 {
return 0, false
}
n = lo
}
return n, true
}
// saturatingAdd returns a+b, clamped to math.MaxUint64 instead of wrapping.
func saturatingAdd(a, b uint64) uint64 {
sum, carry := bits.Add64(a, b, 0)
if carry != 0 {
return math.MaxUint64
}
return sum
} }
// metadataHash returns a stable xxhash64 over the logical tensor content // metadataHash returns a stable xxhash64 over the logical tensor content
syft/pkg/cataloger/ai/parse_safetensors_model.go
+2 -12
View File
@@ -6,7 +6,6 @@ import (
"io" "io"
"github.com/anchore/syft/internal" "github.com/anchore/syft/internal"
"github.com/anchore/syft/internal/unknown"
"github.com/anchore/syft/syft/artifact" "github.com/anchore/syft/syft/artifact"
"github.com/anchore/syft/syft/file" "github.com/anchore/syft/syft/file"
"github.com/anchore/syft/syft/pkg" "github.com/anchore/syft/syft/pkg"
@@ -26,22 +25,13 @@ func parseSafeTensorsFile(_ context.Context, _ file.Resolver, _ *generic.Environ
return nil, nil, fmt.Errorf("failed to read safetensors header: %w", err) return nil, nil, fmt.Errorf("failed to read safetensors header: %w", err)
} }
// ShardCount is intentionally not set here: the merge processor is the single md := header.modelInfo()
// owner of ShardCount and derives it from the number of shards in the group.
md := pkg.SafeTensorsModelInfo{
Format: "safetensors",
TensorCount: uint64(len(header.tensors)),
Parameters: header.parameterCount(),
Quantization: normalizeDType(header.dominantDType()),
UserMetadata: userMetadataKeyValues(header.metadata),
MetadataHash: header.metadataHash(),
}
p := newSafeTensorsPackage( p := newSafeTensorsPackage(
&md, &md,
reader.WithAnnotation(pkg.EvidenceAnnotationKey, pkg.PrimaryEvidenceAnnotation), reader.WithAnnotation(pkg.EvidenceAnnotationKey, pkg.PrimaryEvidenceAnnotation),
) )
return []pkg.Package{p}, nil, unknown.IfEmptyf([]pkg.Package{p}, "unable to parse safetensors file") return []pkg.Package{p}, nil, nil
} }
// integrity check // integrity check
syft/pkg/cataloger/ai/parse_safetensors_oci.go
+3 -11
View File
@@ -8,7 +8,6 @@ import (
"strings" "strings"
"github.com/anchore/syft/internal" "github.com/anchore/syft/internal"
"github.com/anchore/syft/internal/unknown"
"github.com/anchore/syft/syft/artifact" "github.com/anchore/syft/syft/artifact"
"github.com/anchore/syft/syft/file" "github.com/anchore/syft/syft/file"
"github.com/anchore/syft/syft/pkg" "github.com/anchore/syft/syft/pkg"
@@ -50,7 +49,7 @@ type dockerAIModelConfig struct {
func parseSafeTensorsOCIConfig(_ context.Context, _ file.Resolver, _ *generic.Environment, reader file.LocationReadCloser) ([]pkg.Package, []artifact.Relationship, error) { func parseSafeTensorsOCIConfig(_ context.Context, _ file.Resolver, _ *generic.Environment, reader file.LocationReadCloser) ([]pkg.Package, []artifact.Relationship, error) {
defer internal.CloseAndLogError(reader, reader.Path()) defer internal.CloseAndLogError(reader, reader.Path())
body, err := io.ReadAll(io.LimitReader(reader, 1024*1024)) body, err := readBounded(reader, maxModelConfigBlobSize)
if err != nil { if err != nil {
return nil, nil, fmt.Errorf("failed to read docker AI model config: %w", err) return nil, nil, fmt.Errorf("failed to read docker AI model config: %w", err)
} }
@@ -81,7 +80,7 @@ func parseSafeTensorsOCIConfig(_ context.Context, _ file.Resolver, _ *generic.En
&md, &md,
reader.WithAnnotation(pkg.EvidenceAnnotationKey, pkg.PrimaryEvidenceAnnotation), reader.WithAnnotation(pkg.EvidenceAnnotationKey, pkg.PrimaryEvidenceAnnotation),
) )
return []pkg.Package{p}, nil, unknown.IfEmptyf([]pkg.Package{p}, "unable to parse docker AI safetensors config") return []pkg.Package{p}, nil, nil
} }
// parseSafeTensorsOCILayer decodes the JSON header of a SafeTensors weight // parseSafeTensorsOCILayer decodes the JSON header of a SafeTensors weight
@@ -94,14 +93,7 @@ func parseSafeTensorsOCILayer(_ context.Context, _ file.Resolver, _ *generic.Env
return nil, nil, fmt.Errorf("failed to read safetensors layer header: %w", err) return nil, nil, fmt.Errorf("failed to read safetensors layer header: %w", err)
} }
md := pkg.SafeTensorsModelInfo{ md := header.modelInfo()
Format: "safetensors",
TensorCount: uint64(len(header.tensors)),
Parameters: header.parameterCount(),
Quantization: normalizeDType(header.dominantDType()),
UserMetadata: userMetadataKeyValues(header.metadata),
MetadataHash: header.metadataHash(),
}
p := newSafeTensorsPackage( p := newSafeTensorsPackage(
&md, &md,
syft/pkg/cataloger/ai/parse_safetensors_test.go
+268 -41
View File
@@ -6,8 +6,11 @@ import (
"encoding/binary" "encoding/binary"
"encoding/json" "encoding/json"
"io" "io"
"math"
"os" "os"
"path/filepath" "path/filepath"
"slices"
"strings"
"testing" "testing"
"github.com/stretchr/testify/assert" "github.com/stretchr/testify/assert"
@@ -48,7 +51,7 @@ func buildSafeTensorsFile(t *testing.T, metadata map[string]string, tensors map[
// //
// The naming precedence (owned by the merge processor's pickSafeTensorsName) is: // The naming precedence (owned by the merge processor's pickSafeTensorsName) is:
// 1. config.json _name_or_path (path.Base applied), found beside the model or // 1. config.json _name_or_path (path.Base applied), found beside the model or
// by walking up parent directories to the scan root // in its parent directory
// 2. otherwise the model's immediate parent directory base name // 2. otherwise the model's immediate parent directory base name
// → drop (no package) when neither yields a usable name // → drop (no package) when neither yields a usable name
// //
@@ -231,9 +234,9 @@ func TestSafeTensorsCataloger(t *testing.T) {
expectedPackages: nil, expectedPackages: nil,
}, },
{ {
// rung 1 via parent-walk: findDirHFConfig walks up from the model // rung 1 via the parent: findDirHFConfig also checks the model
// directory, so a config.json in an ancestor names a nested model. // directory's parent, so a config.json there names a nested model.
name: "config.json in an ancestor directory names a nested model", name: "config.json in the parent directory names a nested model",
setup: func(t *testing.T) string { setup: func(t *testing.T) string {
dir := t.TempDir() dir := t.TempDir()
model(t, filepath.Join(dir, "dir", "someothertensor")) model(t, filepath.Join(dir, "dir", "someothertensor"))
@@ -248,6 +251,87 @@ func TestSafeTensorsCataloger(t *testing.T) {
}, },
}, },
}, },
{
// the walk stops at the parent: a config.json two levels up is not used.
name: "config.json two levels up is not used",
setup: func(t *testing.T) string {
dir := t.TempDir()
model(t, filepath.Join(dir, "a", "b", "someothertensor"))
writeFile(t, filepath.Join(dir, "a", "config.json"), `{"_name_or_path":"org/TooFar"}`)
return dir
},
expectedPackages: []pkg.Package{
{
Name: "someothertensor",
Type: pkg.ModelPkg,
Metadata: wantMetadata(""),
},
},
},
{
// JSON without any HF field is not treated as a model config, so the
// lookup keeps going to the parent.
name: "non-HF config.json is skipped",
setup: func(t *testing.T) string {
dir := t.TempDir()
modelDir := filepath.Join(dir, "app", "sometensor")
model(t, modelDir)
writeFile(t, filepath.Join(modelDir, "config.json"), `{"port":8080}`)
writeFile(t, filepath.Join(dir, "app", "config.json"), `{"_name_or_path":"org/Parent"}`)
return dir
},
expectedPackages: []pkg.Package{
{
Name: "Parent",
Type: pkg.ModelPkg,
Metadata: wantMetadata(""),
},
},
},
{
// an oversized config.json is skipped entirely, while a large README
// still yields its frontmatter (only a prefix is read).
name: "oversized config.json is ignored and a large README still names the model",
setup: func(t *testing.T) string {
dir := t.TempDir()
modelDir := filepath.Join(dir, "sometensor")
model(t, modelDir)
writeFile(t, filepath.Join(modelDir, "config.json"),
`{"architectures":["X"],"_name_or_path":"`+strings.Repeat("A", maxHFConfigSize)+`"}`)
writeFile(t, filepath.Join(modelDir, "README.md"),
"---\nlicense: MIT\nbase_model: org/Readme\n---\n"+strings.Repeat("A", maxReadmePrefixSize))
return dir
},
expectedPackages: []pkg.Package{
{
Name: "Readme",
Type: pkg.ModelPkg,
Licenses: pkg.NewLicenseSet(
pkg.NewLicenseFromFields("MIT", "", nil),
),
Metadata: wantMetadata(""),
},
},
},
{
// a _name_or_path that sanitizes to nothing doesn't block README base_model.
name: "degenerate _name_or_path falls through to README base_model",
setup: func(t *testing.T) string {
dir := t.TempDir()
modelDir := filepath.Join(dir, "sometensor")
model(t, modelDir)
writeFile(t, filepath.Join(modelDir, "config.json"), `{"_name_or_path":"."}`)
writeFile(t, filepath.Join(modelDir, "README.md"), "---\nbase_model: org/Readme\n---\n")
return dir
},
expectedPackages: []pkg.Package{
{
Name: "Readme",
Type: pkg.ModelPkg,
Metadata: wantMetadata(""),
},
},
},
{ {
// grouping: independent models in one scan are grouped by their own // grouping: independent models in one scan are grouped by their own
// parent directory and each named from it. // parent directory and each named from it.
@@ -601,6 +685,60 @@ spdx-id: Apache-2.0
assertHasLicense(t, out[0], "Apache-2.0") assertHasLicense(t, out[0], "Apache-2.0")
}) })
t.Run("OCI: config precedence is stable across layer order", func(t *testing.T) {
dir := t.TempDir()
a := filepath.Join(dir, "a.json")
b := filepath.Join(dir, "b.json")
gen := filepath.Join(dir, "generation_config.json")
require.NoError(t, os.WriteFile(a, []byte(`{"_name_or_path":"org/first"}`), 0o644))
require.NoError(t, os.WriteFile(b, []byte(`{"_name_or_path":"org/second"}`), 0o644))
require.NoError(t, os.WriteFile(gen, []byte(`{"max_new_tokens":10}`), 0o644))
configMd := pkg.SafeTensorsModelInfo{Format: "safetensors", TensorCount: 1}
for _, order := range [][]string{{a, b, gen}, {gen, b, a}} {
var locs []file.Location
for _, p := range order {
locs = append(locs, file.NewLocation(p))
}
resolver := file.NewMockResolverForMediaTypes(map[string][]file.Location{
dockerAIModelFileMediaType: locs,
})
out, _, err := safeTensorsMergeProcessor(context.Background(), resolver, []pkg.Package{ociPkg(configMd)}, nil, nil)
require.NoError(t, err)
require.Len(t, out, 1)
assert.Equal(t, "first", out[0].Name)
// the non-HF generation config is not recorded as supporting evidence
assert.Len(t, out[0].Locations.ToSlice(), 3)
}
})
t.Run("dir scan: non-HF config.json in the parent is not supporting evidence", func(t *testing.T) {
dir := t.TempDir()
cfgPath := filepath.Join(dir, "config.json")
require.NoError(t, os.WriteFile(cfgPath, []byte(`{"port":8080}`), 0o644))
loc, cfg := findDirHFConfig(file.NewMockResolverForPaths(cfgPath), filepath.Join(dir, "model"))
assert.Nil(t, loc)
assert.Nil(t, cfg)
})
t.Run("OCI: a large license layer still yields its frontmatter spdx-id", func(t *testing.T) {
dir := t.TempDir()
licensePath := filepath.Join(dir, "LICENSE")
require.NoError(t, os.WriteFile(licensePath,
[]byte("---\nspdx-id: Apache-2.0\n---\n"+strings.Repeat("A", maxLicensePrefixSize)), 0o644))
assert.Equal(t, "Apache-2.0", readLicenseSPDXIDFromFrontmatter(file.NewMockResolverForPaths(licensePath), file.NewLocation(licensePath)))
})
t.Run("OCI: oversized model.file layer is rejected", func(t *testing.T) {
dir := t.TempDir()
cfgPath := filepath.Join(dir, "config.json")
require.NoError(t, os.WriteFile(cfgPath,
[]byte(`{"_name_or_path":"org/x","pad":"`+strings.Repeat("A", maxModelFileLayerSize)+`"}`), 0o644))
cfg, fm := classifyOCIModelFileLayer(file.NewMockResolverForPaths(cfgPath), file.NewLocation(cfgPath))
assert.Nil(t, cfg)
assert.Nil(t, fm)
})
t.Run("names packages and still propagates an upstream error", func(t *testing.T) { t.Run("names packages and still propagates an upstream error", func(t *testing.T) {
// A non-nil upstream error (e.g. another file in the run failed to parse) // A non-nil upstream error (e.g. another file in the run failed to parse)
// must NOT short-circuit naming. The package is still nameless at this // must NOT short-circuit naming. The package is still nameless at this
@@ -622,7 +760,7 @@ spdx-id: Apache-2.0
// (or neither, asserting the drop path). // (or neither, asserting the drop path).
// //
// Precedence (highest → lowest): // Precedence (highest → lowest):
// 1. config.json _name_or_path (path.Base applied; both dir-scan and OCI) // 1. config.json _name_or_path (sanitized to its last path element; both dir-scan and OCI)
// 2. fallback name — OCI image-ref last segment, or dir-scan parent directory // 2. fallback name — OCI image-ref last segment, or dir-scan parent directory
// base name (the merge processor computes the right one per group) // base name (the merge processor computes the right one per group)
// → drop (empty name) when nothing matches // → drop (empty name) when nothing matches
@@ -663,6 +801,31 @@ func TestSafeTensorsNamingPrecedence(t *testing.T) {
want: "tiny-llama", want: "tiny-llama",
}, },
// sanitizing
{
name: "windows-style path uses the last element",
nameOrPath: `C:\models\win-model`,
fallbackName: "fallback-name",
want: "win-model",
},
{
name: "degenerate path falls back",
nameOrPath: "..",
fallbackName: "fallback-name",
want: "fallback-name",
},
{
name: "root path falls back",
nameOrPath: "/",
fallbackName: "fallback-name",
want: "fallback-name",
},
{
name: "long names are truncated",
nameOrPath: "org/" + strings.Repeat("x", maxModelNameLength+10),
want: strings.Repeat("x", maxModelNameLength),
},
// drops // drops
{ {
name: "drop: both rungs empty", name: "drop: both rungs empty",
@@ -984,8 +1147,9 @@ func TestSafeTensorsHeader_parameterCountAndDType(t *testing.T) {
"small": {DType: "F32", Shape: []int64{16, 16}}, "small": {DType: "F32", Shape: []int64{16, 16}},
"scalar": {DType: "F32", Shape: []int64{}}, // empty shape contributes 1 "scalar": {DType: "F32", Shape: []int64{}}, // empty shape contributes 1
}} }}
assert.Equal(t, uint64(1000*16+16*16+1), h.parameterCount()) params, dtype := h.parameterStats()
assert.Equal(t, "BF16", h.dominantDType()) assert.Equal(t, uint64(1000*16+16*16+1), params)
assert.Equal(t, "BF16", dtype)
} }
func TestSafeTensorsHeader_parameterCount_nonPositiveDims(t *testing.T) { func TestSafeTensorsHeader_parameterCount_nonPositiveDims(t *testing.T) {
@@ -997,7 +1161,29 @@ func TestSafeTensorsHeader_parameterCount_nonPositiveDims(t *testing.T) {
"zero": {DType: "F32", Shape: []int64{4, 0}}, "zero": {DType: "F32", Shape: []int64{4, 0}},
"negative": {DType: "F32", Shape: []int64{-1, 8}}, "negative": {DType: "F32", Shape: []int64{-1, 8}},
}} }}
assert.Equal(t, uint64(16), h.parameterCount()) params, _ := h.parameterStats()
assert.Equal(t, uint64(16), params)
}
func TestSafeTensorsHeader_parameterStats_overflow(t *testing.T) {
// a shape whose product overflows uint64 is skipped instead of wrapping (the
// first tensor would wrap to 0) and can't flip the dominant dtype; the total
// saturates instead of wrapping.
h := &safeTensorsHeader{tensors: map[string]safeTensorsEntry{
"good": {DType: "BF16", Shape: []int64{4, 4}},
"wraps": {DType: "F32", Shape: []int64{4294967296, 4294967296}},
"overflow": {DType: "F32", Shape: []int64{math.MaxInt64, 3}},
}}
params, dtype := h.parameterStats()
assert.Equal(t, uint64(16), params)
assert.Equal(t, "BF16", dtype)
h = &safeTensorsHeader{tensors: map[string]safeTensorsEntry{
"a": {DType: "F32", Shape: []int64{math.MaxInt64, 2}},
"b": {DType: "F32", Shape: []int64{math.MaxInt64, 2}},
}}
params, _ = h.parameterStats()
assert.Equal(t, uint64(math.MaxUint64), params)
} }
func TestSafeTensorsHeader_dominantDType_tieBreak(t *testing.T) { func TestSafeTensorsHeader_dominantDType_tieBreak(t *testing.T) {
@@ -1007,7 +1193,8 @@ func TestSafeTensorsHeader_dominantDType_tieBreak(t *testing.T) {
"a": {DType: "F32", Shape: []int64{4, 4}}, "a": {DType: "F32", Shape: []int64{4, 4}},
"b": {DType: "BF16", Shape: []int64{4, 4}}, "b": {DType: "BF16", Shape: []int64{4, 4}},
}} }}
assert.Equal(t, "BF16", h.dominantDType()) _, dtype := h.parameterStats()
assert.Equal(t, "BF16", dtype)
} }
func TestNormalizeDType(t *testing.T) { func TestNormalizeDType(t *testing.T) {
@@ -1036,25 +1223,78 @@ func TestParseFrontmatter(t *testing.T) {
name string name string
input string input string
wantNil bool wantNil bool
wantLicense string wantLicenses []string
wantBaseModel []string wantBaseModel []string
}{ }{
{ {
name: "list base_model", name: "list base_model",
input: "---\nlicense: mit\nbase_model:\n - org/Model\n---\nbody", input: "---\nlicense: mit\nbase_model:\n - org/Model\n---\nbody",
wantLicense: "mit", wantLicenses: []string{"mit"},
wantBaseModel: []string{"org/Model"}, wantBaseModel: []string{"org/Model"},
}, },
{ {
name: "scalar base_model", name: "scalar base_model",
input: "---\nlicense: apache-2.0\nbase_model: org/Model\n---\n", input: "---\nlicense: apache-2.0\nbase_model: org/Model\n---\n",
wantLicense: "apache-2.0", wantLicenses: []string{"apache-2.0"},
wantBaseModel: []string{"org/Model"}, wantBaseModel: []string{"org/Model"},
}, },
{ {
name: "leading BOM", name: "list license keeps base_model",
input: "\xef\xbb\xbf---\nlicense: mit\n---\n", input: "---\nlicense: [mit, apache-2.0]\nbase_model: org/Model\n---\n",
wantLicense: "mit", wantLicenses: []string{"mit", "apache-2.0"},
wantBaseModel: []string{"org/Model"},
},
{
name: "mapping-valued fields are ignored, not fatal",
input: "---\nlicense: {name: mit}\nbase_model: org/Model\n---\n",
wantBaseModel: []string{"org/Model"},
},
{
name: "leading BOM",
input: "\xef\xbb\xbf---\nlicense: mit\n---\n",
wantLicenses: []string{"mit"},
},
{
name: "null license is ignored",
input: "---\nlicense: null\nbase_model: ~\n---\n",
wantLicenses: nil,
wantBaseModel: nil,
},
{
name: "empty block",
input: "---\n---\n",
wantNil: true,
},
{
name: "oversized block is rejected",
input: "---\nlicense: mit\n" + strings.Repeat("# pad\n", maxFrontmatterSize/6+1) + "---\n",
wantNil: true,
},
{
name: "closing delimiter at EOF with trailing whitespace",
input: "---\nlicense: mit\n--- ",
wantLicenses: []string{"mit"},
},
{
name: "list values are capped",
input: "---\nlicense: [" + strings.Repeat("mit,", maxFrontmatterValues+5) + "mit]\n---\n",
wantLicenses: slices.Repeat([]string{"mit"}, maxFrontmatterValues),
},
{
name: "CRLF delimiters",
input: "---\r\nlicense: mit\r\n---\r\nbody",
wantLicenses: []string{"mit"},
},
{
name: "opening delimiter must be exactly ---",
input: "----\nlicense: mit\n---\n",
wantNil: true,
},
{
name: "a ---x line inside the block does not close it",
input: "---\nbase_model: org/Model\n---x: 1\nlicense: mit\n---\n",
wantLicenses: []string{"mit"},
wantBaseModel: []string{"org/Model"},
}, },
{ {
name: "no frontmatter", name: "no frontmatter",
@@ -1066,6 +1306,12 @@ func TestParseFrontmatter(t *testing.T) {
input: "---\nlicense: mit\n", input: "---\nlicense: mit\n",
wantNil: true, wantNil: true,
}, },
{
// https://github.com/anchore/syft/issues/5327
name: "malformed frontmatter that panics the yaml decoder",
input: "---\n<<:\r? 0:\n---\nbody",
wantNil: true,
},
} }
for _, tt := range tests { for _, tt := range tests {
@@ -1076,7 +1322,7 @@ func TestParseFrontmatter(t *testing.T) {
return return
} }
require.NotNil(t, fm) require.NotNil(t, fm)
assert.Equal(t, tt.wantLicense, fm.License) assert.Equal(t, tt.wantLicenses, fm.Licenses)
if tt.wantBaseModel != nil { if tt.wantBaseModel != nil {
assert.Equal(t, tt.wantBaseModel, fm.BaseModel) assert.Equal(t, tt.wantBaseModel, fm.BaseModel)
} }
@@ -1157,6 +1403,12 @@ limitations:
input: "---\nspdx-id: MIT\n(never closes)\n", input: "---\nspdx-id: MIT\n(never closes)\n",
want: "", want: "",
}, },
{
// https://github.com/anchore/syft/issues/5327
name: "malformed frontmatter that panics the yaml decoder",
input: "---\n<<:\r? 0:\n---\nbody\n",
want: "",
},
} }
for _, tt := range tests { for _, tt := range tests {
@@ -1233,28 +1485,3 @@ func TestReadSafeTensorsHeader_capBoundsHeap(t *testing.T) {
assert.Less(t, bounded, uint64(8*1024*1024), assert.Less(t, bounded, uint64(8*1024*1024),
"the declared-size check has to reject before the body is read, not after") "the declared-size check has to reject before the body is read, not after")
} }
// TestCopyHeader_boundIsTheCallersLimitedReader pins where the GGUF bound actually lives. copyHeader
// io.Copy's whatever it is handed, so maxHeaderSize only holds because every caller wraps the reader
// first. A caller that stops doing that reads the whole file, and nothing in copyHeader would object.
//
// Bytes retained is asserted directly rather than through testutils.MeasureAlloc: the destination is a
// bytes.Buffer, whose doubling growth costs a multiple of what it holds and lands on a different power
// of two depending on the platform. buf.Len() is the same property without that noise.
func TestCopyHeader_boundIsTheCallersLimitedReader(t *testing.T) {
const payload = 2*maxHeaderSize + 1
file := make([]byte, 24, 24+payload)
binary.LittleEndian.PutUint32(file[0:4], ggufMagicNumber)
file = append(file, bytes.Repeat([]byte{'x'}, payload)...)
var unbounded bytes.Buffer
require.NoError(t, copyHeader(&unbounded, bytes.NewReader(file)))
require.Equal(t, len(file), unbounded.Len(),
"an unwrapped reader is copied whole; copyHeader applies no limit of its own")
var bounded bytes.Buffer
require.NoError(t, copyHeader(&bounded, &io.LimitedReader{R: bytes.NewReader(file), N: maxHeaderSize}))
assert.Equal(t, maxHeaderSize, bounded.Len(),
"a wrapped reader must cap the copy at maxHeaderSize")
}
syft/pkg/cataloger/ai/processor.go
+2 -2
View File
@@ -104,10 +104,10 @@ func mergeDirModels(ctx context.Context, resolver file.Resolver, pkgs []pkg.Pack
// groupByParentDir buckets filesystem-scanned models by the directory their // groupByParentDir buckets filesystem-scanned models by the directory their
// primary-evidence file lives in. // primary-evidence file lives in.
//
// This encodes a deliberate assumption: a directory holds one logical // This encodes a deliberate assumption: a directory holds one logical
// model, so every .safetensors file in a directory is treated as a shard of the // model, so every .safetensors file in a directory is treated as a shard of the
// same modeland merged into one package. // same model and merged into one package.
// The trade-off is that if a directory happens to contain several unrelated models, // The trade-off is that if a directory happens to contain several unrelated models,
// they are merged into one package rather than reported separately. // they are merged into one package rather than reported separately.
// We accept that because the conventional on-disk layout gives each model // We accept that because the conventional on-disk layout gives each model
syft/pkg/cataloger/ai/processor_test.go
+18
View File
@@ -1,6 +1,7 @@
package ai package ai
import ( import (
"errors"
"testing" "testing"
"github.com/stretchr/testify/assert" "github.com/stretchr/testify/assert"
@@ -61,3 +62,20 @@ func Test_ggufMergeProcessor(t *testing.T) {
}) })
} }
} }
func Test_ggufMergeProcessor_mergesDespiteError(t *testing.T) {
// one layer failing to parse must not drop the rest of the model
parseErr := errors.New("one layer failed to parse")
pkgs := []pkg.Package{
{Name: "model", Metadata: pkg.GGUFFileHeader{MetadataKeyValuesHash: "abc"}},
{Name: "", Metadata: pkg.GGUFFileHeader{MetadataKeyValuesHash: "part1"}},
}
got, _, err := ggufMergeProcessor(pkgs, nil, parseErr)
require.ErrorIs(t, err, parseErr)
require.Len(t, got, 1)
assert.Equal(t, "model", got[0].Name)
header, ok := got[0].Metadata.(pkg.GGUFFileHeader)
require.True(t, ok)
assert.Len(t, header.Parts, 1)
}
syft/pkg/cataloger/ai/test_helpers_test.go
+31 -3
View File
@@ -72,8 +72,32 @@ func (b *testGGUFBuilder) withUint32KV(key string, value uint32) *testGGUFBuilde
return b return b
} }
// withRawKV adds a KV whose value bytes are written verbatim, for arrays and malformed values.
func (b *testGGUFBuilder) withRawKV(key string, valueType uint32, raw []byte) *testGGUFBuilder {
b.kvPairs = append(b.kvPairs, testKVPair{key: key, valueType: valueType, value: raw})
return b
}
// ggufArray encodes an array value: item type, length, then the already encoded items.
func ggufArray(itemType uint32, n uint64, items []byte) []byte {
buf := new(bytes.Buffer)
binary.Write(buf, binary.LittleEndian, itemType)
binary.Write(buf, binary.LittleEndian, n)
buf.Write(items)
return buf.Bytes()
}
// writeLen writes a count or length, which is 32-bit in GGUF v1 and 64-bit after.
func (b *testGGUFBuilder) writeLen(n uint64) {
if b.version <= 1 {
binary.Write(b.buf, binary.LittleEndian, uint32(n))
return
}
binary.Write(b.buf, binary.LittleEndian, n)
}
func (b *testGGUFBuilder) writeString(s string) { func (b *testGGUFBuilder) writeString(s string) {
binary.Write(b.buf, binary.LittleEndian, uint64(len(s))) b.writeLen(uint64(len(s)))
b.buf.WriteString(s) b.buf.WriteString(s)
} }
@@ -85,10 +109,10 @@ func (b *testGGUFBuilder) build() []byte {
binary.Write(b.buf, binary.LittleEndian, b.version) binary.Write(b.buf, binary.LittleEndian, b.version)
// Write tensor count // Write tensor count
binary.Write(b.buf, binary.LittleEndian, b.tensorCount) b.writeLen(b.tensorCount)
// Write KV count // Write KV count
binary.Write(b.buf, binary.LittleEndian, uint64(len(b.kvPairs))) b.writeLen(uint64(len(b.kvPairs)))
// Write KV pairs // Write KV pairs
for _, kv := range b.kvPairs { for _, kv := range b.kvPairs {
@@ -97,6 +121,10 @@ func (b *testGGUFBuilder) build() []byte {
// Write value type // Write value type
binary.Write(b.buf, binary.LittleEndian, kv.valueType) binary.Write(b.buf, binary.LittleEndian, kv.valueType)
// Write value based on type // Write value based on type
if raw, ok := kv.value.([]byte); ok {
b.buf.Write(raw)
continue
}
switch kv.valueType { switch kv.valueType {
case ggufTypeString: case ggufTypeString:
b.writeString(kv.value.(string)) b.writeString(kv.value.(string))
syft/pkg/cataloger/ai/testdata/safetensors/extract_header.go
Vendored
+3 -3
View File
@@ -28,9 +28,9 @@ import (
const ( const (
safetensorsLayerMediaType = "application/vnd.docker.ai.safetensors" safetensorsLayerMediaType = "application/vnd.docker.ai.safetensors"
// 8 MB matches maxHeaderBytes in the OCI model source. Real model headers // matches maxSafeTensorsHeaderBytes in the OCI model source (8 MB header
// are well under 1 MB; the extra slack covers outliers. // cap plus the 8-byte length prefix). Real model headers are well under 1 MB.
fetchBytes = 8 * 1024 * 1024 fetchBytes = 8*1024*1024 + 8
) )
func main() { func main() {
syft/pkg/cataloger/alpine/parse_apk_db.go
+12
View File
@@ -270,6 +270,10 @@ func (f apkField) apply(p *parsedData, ctx *apkFileParsingContext) {
ctx.indexOfLatestDirectory = len(ctx.files) - 1 ctx.indexOfLatestDirectory = len(ctx.files) - 1
case "M": case "M":
i := ctx.indexOfLatestDirectory i := ctx.indexOfLatestDirectory
if i < 0 {
log.Debugf("APK field %q appears before any directory entry", f.name)
return
}
latest := ctx.files[i] latest := ctx.files[i]
var ok bool var ok bool
@@ -296,6 +300,10 @@ func (f apkField) apply(p *parsedData, ctx *apkFileParsingContext) {
ctx.indexOfLatestRegularFile = len(ctx.files) - 1 ctx.indexOfLatestRegularFile = len(ctx.files) - 1
case "a": case "a":
i := ctx.indexOfLatestRegularFile i := ctx.indexOfLatestRegularFile
if i < 0 {
log.Debugf("APK field %q appears before any file entry", f.name)
return
}
latest := ctx.files[i] latest := ctx.files[i]
var ok bool var ok bool
@@ -309,6 +317,10 @@ func (f apkField) apply(p *parsedData, ctx *apkFileParsingContext) {
ctx.files[i] = latest ctx.files[i] = latest
case "Z": case "Z":
i := ctx.indexOfLatestRegularFile i := ctx.indexOfLatestRegularFile
if i < 0 {
log.Debugf("APK field %q appears before any file entry", f.name)
return
}
latest := ctx.files[i] latest := ctx.files[i]
latest.Digest = processChecksum(f.value) latest.Digest = processChecksum(f.value)
syft/pkg/cataloger/alpine/parse_apk_db_test.go
+26
View File
@@ -869,3 +869,29 @@ https://foo.them.org/alpine/v3.14/community`,
}) })
} }
} }
func TestParseApkDB_attributeBeforeItsEntry(t *testing.T) {
// ACL (M:, a:) and checksum (Z:) lines describe the most recent directory or file entry, and used to index
// files[-1] when no entry had been seen yet
tests := []struct {
name string
line string
}{
{name: "directory ACL before any directory", line: "M:0:0:755"},
{name: "file ACL before any file", line: "a:0:0:755"},
{name: "checksum before any file", line: "Z:Q1Kz6TXnPVJeDQUfO0IJVxD3HnYmU="},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
contents := strings.Join([]string{"P:foo", "V:1.0-r0", tt.line, ""}, "\n")
reader := file.NewLocationReadCloser(file.NewLocation("installed"), io.NopCloser(strings.NewReader(contents)))
pkgs, _, err := parseApkDB(context.Background(), nil, new(generic.Environment), reader)
require.NoError(t, err)
require.Len(t, pkgs, 1)
assert.Equal(t, "foo", pkgs[0].Name)
assert.Empty(t, pkgs[0].Metadata.(pkg.ApkDBEntry).Files, "an orphaned attribute line should not invent a file")
})
}
}
syft/pkg/cataloger/arch/parse_alpm_db.go
+1 -6
View File
@@ -148,12 +148,7 @@ func parseAlpmDBEntry(reader io.Reader) (*parsedData, error) {
} }
func newScanner(reader io.Reader) *bufio.Scanner { func newScanner(reader io.Reader) *bufio.Scanner {
// This is taken from the apk parser scanner := internal.NewLineScanner(reader)
// https://github.com/anchore/syft/blob/v0.47.0/syft/pkg/cataloger/apkdb/parse_apk_db.go#L37
const maxScannerCapacity = 1024 * 1024
bufScan := make([]byte, maxScannerCapacity)
scanner := bufio.NewScanner(reader)
scanner.Buffer(bufScan, maxScannerCapacity)
onDoubleLF := func(data []byte, atEOF bool) (advance int, token []byte, err error) { onDoubleLF := func(data []byte, atEOF bool) (advance int, token []byte, err error) {
for i := range data { for i := range data {
if i > 0 && data[i-1] == '\n' && data[i] == '\n' { if i > 0 && data[i-1] == '\n' && data[i] == '\n' {
syft/pkg/cataloger/arch/parse_alpm_db_test.go
+38 -19
View File
@@ -492,12 +492,16 @@ func Test_parseMtree_malformedInput(t *testing.T) {
// bytes still costs multiples of its own size in heap. Asserting the error would keep passing if the // bytes still costs multiples of its own size in heap. Asserting the error would keep passing if the
// line cap were removed, because the byte cap would still reject a large enough input eventually. // line cap were removed, because the byte cap would still reject a large enough input eventually.
func Test_parseMtree_lineCapBoundsHeap(t *testing.T) { func Test_parseMtree_lineCapBoundsHeap(t *testing.T) {
// newline-only lines are the worst case: minimum bytes per entry, so the most entries per byte // the property is linear in the cap, so it is measured at a small cap and scaled to the shipped
const payloadSize = 16 * intFile.MB // one. Running it at 300k lines means an uncapped parse costing ~16GB, which under -race is most
spec := bytes.Repeat([]byte{'\n'}, payloadSize) // of a CI minute for a test with no concurrency in it.
const (
lineCap = 4096
payloadSize = 64 * lineCap
)
// well inside the byte cap, so only the line cap can stop this // newline-only lines are the worst case: minimum bytes per entry, so the most entries per byte
require.Less(t, int64(payloadSize), int64(maxMtreeSize)) spec := bytes.Repeat([]byte{'\n'}, payloadSize)
unbounded := testutils.MeasureAlloc(t, func() { unbounded := testutils.MeasureAlloc(t, func() {
// the same listing with the line cap lifted, to prove the fixture is really a bomb // the same listing with the line cap lifted, to prove the fixture is really a bomb
@@ -507,19 +511,29 @@ func Test_parseMtree_lineCapBoundsHeap(t *testing.T) {
require.Greater(t, unbounded, uint64(payloadSize), require.Greater(t, unbounded, uint64(payloadSize),
"fixture did not actually cost more than its own size; it is no longer a bomb") "fixture did not actually cost more than its own size; it is no longer a bomb")
bounded := testutils.MeasureAlloc(t, func() { boundedAt := func(limit int) uint64 {
_, err := parseMtreeWithLimits(gzipOf(t, spec), maxMtreeSize, maxMtreeLines) gz := gzipOf(t, spec)
require.ErrorIs(t, err, errTooManyMtreeLines) return testutils.MeasureAlloc(t, func() {
}) _, err := parseMtreeWithLimits(gz, maxMtreeSize, limit)
require.ErrorIs(t, err, errTooManyMtreeLines)
})
}
bounded := boundedAt(lineCap)
t.Logf("unbounded allocated %d bytes, bounded allocated %d bytes", unbounded, bounded) // the marginal cost between two caps, so fixed setup (gzip reader, parser) does not get scaled
perLine := (boundedAt(2*lineCap) - bounded) / lineCap
atShippedCap := perLine * maxMtreeLines
t.Logf("unbounded allocated %d bytes, bounded allocated %d bytes (%d per line, ~%d at the shipped cap)",
unbounded, bounded, perLine, atShippedCap)
// measured at ~238MB against ~16.5GB uncapped, so the cap is doing its job, but note what it assert.Less(t, bounded, unbounded/10,
// actually buys: 300k entries still cost a few hundred MB, and that is per concurrent cataloger.
// The budget is set to catch the cap being removed or raised by an order of magnitude, not to pin
// the exact figure.
assert.Less(t, bounded, uint64(512*intFile.MB),
"the line cap has to stop the parse before it builds an entry per line") "the line cap has to stop the parse before it builds an entry per line")
// measured at ~238MB for the shipped 300k lines, so the cap is doing its job, but note what it
// actually buys: that is still a few hundred MB, per concurrent cataloger. The budget is set to
// catch the cap being raised by an order of magnitude, not to pin the exact figure.
assert.Less(t, atShippedCap, uint64(512*intFile.MB),
"the shipped line cap allows more heap than budgeted")
} }
// Test_parseMtree_sizeCapBoundsHeap is the byte-cap counterpart to the line-cap budget above. // Test_parseMtree_sizeCapBoundsHeap is the byte-cap counterpart to the line-cap budget above.
@@ -532,9 +546,12 @@ func Test_parseMtree_lineCapBoundsHeap(t *testing.T) {
// be ordinary for the byte cap to be the thing under test, and there have to be few enough of them // be ordinary for the byte cap to be the thing under test, and there have to be few enough of them
// that the line cap does not fire first. // that the line cap does not fire first.
func Test_parseMtree_sizeCapBoundsHeap(t *testing.T) { func Test_parseMtree_sizeCapBoundsHeap(t *testing.T) {
// same reasoning as the line cap: measured at a small byte cap, since the shipped 64MB cap means
// gzipping and parsing ~200MB of fixture, which is slow under -race and proves nothing more.
const ( const (
sizeCap = intFile.MB
lineLen = 1024 lineLen = 1024
numLines = 96 * 1024 // ~96MB across ~98k lines: past the 64MB cap, well under the 300k line cap numLines = 2 * sizeCap / lineLen // twice the cap, and far under the line cap
) )
var raw bytes.Buffer var raw bytes.Buffer
@@ -542,7 +559,7 @@ func Test_parseMtree_sizeCapBoundsHeap(t *testing.T) {
for range numLines { for range numLines {
raw.Write(line) raw.Write(line)
} }
require.Greater(t, int64(raw.Len()), int64(maxMtreeSize), "fixture must exceed the byte cap") require.Greater(t, int64(raw.Len()), int64(sizeCap), "fixture must exceed the byte cap")
require.Less(t, numLines, maxMtreeLines, "and must not trip the line cap first") require.Less(t, numLines, maxMtreeLines, "and must not trip the line cap first")
unbounded := testutils.MeasureAlloc(t, func() { unbounded := testutils.MeasureAlloc(t, func() {
@@ -554,11 +571,13 @@ func Test_parseMtree_sizeCapBoundsHeap(t *testing.T) {
"fixture did not actually cost more than its own size; it is no longer a bomb") "fixture did not actually cost more than its own size; it is no longer a bomb")
bounded := testutils.MeasureAlloc(t, func() { bounded := testutils.MeasureAlloc(t, func() {
_, err := parseMtreeWithLimits(gzipOf(t, raw.Bytes()), maxMtreeSize, maxMtreeLines) _, err := parseMtreeWithLimits(gzipOf(t, raw.Bytes()), sizeCap, maxMtreeLines)
require.ErrorIs(t, err, errMtreeTooLarge) require.ErrorIs(t, err, errMtreeTooLarge)
}) })
t.Logf("unbounded allocated %d bytes, bounded allocated %d bytes", unbounded, bounded) t.Logf("unbounded allocated %d bytes, bounded allocated %d bytes", unbounded, bounded)
assert.Less(t, bounded, uint64(512*intFile.MB),
// the old budget was 512MB against the 64MB cap, so 8x the cap
assert.Less(t, bounded, uint64(8*sizeCap),
"the size cap has to stop the read at the cap, not after the member is drained") "the size cap has to stop the read at the cap, not after the member is drained")
} }
syft/pkg/cataloger/binary/classifier_cataloger_test.go
+24
View File
@@ -407,6 +407,30 @@ func Test_Cataloger_PositiveCases(t *testing.T) {
Metadata: metadata("traefik-binary"), Metadata: metadata("traefik-binary"),
}, },
}, },
{
// regression test: big-endian architectures (e.g. s390x) previously matched an unrelated
// single-NUL-terminated numeric string elsewhere in the binary instead of the real version
logicalFixture: "traefik/3.6.8/linux-s390x",
expected: pkg.Package{
Name: "traefik",
Version: "3.6.8",
Type: "binary",
PURL: "pkg:generic/traefik@3.6.8",
Locations: locations("traefik"),
Metadata: metadata("traefik-binary"),
},
},
{
logicalFixture: "traefik/2.11.1/linux-arm",
expected: pkg.Package{
Name: "traefik",
Version: "2.11.1",
Type: "binary",
PURL: "pkg:generic/traefik@2.11.1",
Locations: locations("traefik"),
Metadata: metadata("traefik-binary"),
},
},
{ {
logicalFixture: "memcached/1.6.18/linux-amd64", logicalFixture: "memcached/1.6.18/linux-amd64",
expected: pkg.Package{ expected: pkg.Package{
syft/pkg/cataloger/binary/classifiers.go
+9 -2
View File
@@ -284,11 +284,18 @@ func DefaultClassifiers() []binutils.Classifier {
{ {
Class: "traefik-binary", Class: "traefik-binary",
FileGlob: "**/traefik", FileGlob: "**/traefik",
EvidenceMatcher: m.FileContentsVersionMatcher( EvidenceMatcher: binutils.MatchAny(
// on some architectures (e.g. s390x) the real version string isn't NUL-prefixed
// and is instead terminated by two NUL bytes; checking this pattern first avoids
// matching an unrelated single-NUL-terminated numeric string elsewhere in the binary
m.FileContentsVersionMatcher(
`(?m)(?P<version>[0-9]+\.[0-9]+\.[0-9]+(-alpha[0-9]|-beta[0-9]|-rc[0-9])?)\x00\x00`),
// [NUL]v1.7.34[NUL] // [NUL]v1.7.34[NUL]
// [NUL]2.9.6[NUL] // [NUL]2.9.6[NUL]
// 3.0.4[NUL] // 3.0.4[NUL]
`(?m)(\x00v?|\x{FFFD}.?)(?P<version>[0-9]+\.[0-9]+\.[0-9]+(-alpha[0-9]|-beta[0-9]|-rc[0-9])?)\x00`), m.FileContentsVersionMatcher(
`(?m)(\x00v?|\x{FFFD}.?)(?P<version>[0-9]+\.[0-9]+\.[0-9]+(-alpha[0-9]|-beta[0-9]|-rc[0-9])?)\x00`),
),
Package: "traefik", Package: "traefik",
PURL: mustPURL("pkg:generic/traefik@version"), PURL: mustPURL("pkg:generic/traefik@version"),
CPEs: singleCPE("cpe:2.3:a:traefik:traefik:*:*:*:*:*:*:*:*", cpe.NVDDictionaryLookupSource), CPEs: singleCPE("cpe:2.3:a:traefik:traefik:*:*:*:*:*:*:*:*", cpe.NVDDictionaryLookupSource),
syft/pkg/cataloger/binary/internal/manager/internal/config/application.go
+4 -4
View File
@@ -102,10 +102,10 @@ func (c Application) Validate() error {
} }
func (c Application) GetBinaryFromImage(name, version string) *BinaryFromImage { func (c Application) GetBinaryFromImage(name, version string) *BinaryFromImage {
if strings.Contains(name, "@") && version == "" { if version == "" {
parts := strings.Split(name, "@") if n, v, ok := strings.Cut(name, "@"); ok {
name = parts[0] name, version = n, v
version = parts[1] }
} }
for _, entry := range c.FromImages { for _, entry := range c.FromImages {
if entry.Name() == name && entry.Version == version { if entry.Name() == name && entry.Version == version {
syft/pkg/cataloger/binary/testdata/classifiers/snippets/traefik/2.11.1/linux-arm/traefik
Vendored
BIN
View File
Binary file not shown.
syft/pkg/cataloger/binary/testdata/classifiers/snippets/traefik/3.6.8/linux-s390x/traefik
Vendored
BIN
View File
Binary file not shown.
syft/pkg/cataloger/debian/parse_copyright.go
+5 -2
View File
@@ -1,7 +1,6 @@
package debian package debian
import ( import (
"bufio"
"io" "io"
"regexp" "regexp"
"sort" "sort"
@@ -10,6 +9,7 @@ import (
"github.com/scylladb/go-set/strset" "github.com/scylladb/go-set/strset"
"github.com/anchore/syft/internal" "github.com/anchore/syft/internal"
"github.com/anchore/syft/internal/log"
) )
// For more information see: https://www.debian.org/doc/packaging-manuals/copyright-format/1.0/#license-syntax // For more information see: https://www.debian.org/doc/packaging-manuals/copyright-format/1.0/#license-syntax
@@ -34,7 +34,7 @@ const (
func parseLicensesFromCopyright(reader io.Reader) []string { func parseLicensesFromCopyright(reader io.Reader) []string {
findings := strset.New() findings := strset.New()
scanner := bufio.NewScanner(reader) scanner := internal.NewLineScanner(reader)
// per the DEP-5 spec, machine-readable copyright files MUST have a // per the DEP-5 spec, machine-readable copyright files MUST have a
// Format field whose value is a URI for the specification. Only files // Format field whose value is a URI for the specification. Only files
@@ -69,6 +69,9 @@ func parseLicensesFromCopyright(reader io.Reader) []string {
findings.Add(found) findings.Add(found)
} }
} }
if err := scanner.Err(); err != nil {
log.WithFields("error", err).Debug("failed to fully read debian copyright file")
}
if !formatVerified { if !formatVerified {
return nil return nil
syft/pkg/cataloger/debian/parse_deb_archive_test.go
+7 -3
View File
@@ -373,9 +373,11 @@ func Test_processDataTar_capsCopyrightFiles(t *testing.T) {
// than after the fact. The existing bounds test uses a payload small enough that buffering the whole // than after the fact. The existing bounds test uses a payload small enough that buffering the whole
// thing would pass anyway; this one is large enough that the difference shows up as bytes. // thing would pass anyway; this one is large enough that the difference shows up as bytes.
func Test_decompressionStream_failsEarlyInsteadOfBuffering(t *testing.T) { func Test_decompressionStream_failsEarlyInsteadOfBuffering(t *testing.T) {
const expanded = 512 * intFile.MB // 4x the 8MB budget below is plenty to tell bounded from unbounded. Going bigger only makes the
// fixture slower to build and inflate, which is most of this test's time under -race.
const expanded = 32 * intFile.MB
// a few KB of input expanding to 512MB, so heap is what separates a streaming bound from a // well under 1% of input expanding to 32MB, so heap is what separates a streaming bound from a
// bound checked after the read completes // bound checked after the read completes
var gz bytes.Buffer var gz bytes.Buffer
gw := gzip.NewWriter(&gz) gw := gzip.NewWriter(&gz)
@@ -386,7 +388,9 @@ func Test_decompressionStream_failsEarlyInsteadOfBuffering(t *testing.T) {
remaining -= int64(n) remaining -= int64(n)
} }
require.NoError(t, gw.Close()) require.NoError(t, gw.Close())
require.Less(t, gz.Len(), 1024*1024, "compressed payload should be tiny relative to what it expands to") // a ratio rather than a fixed size: deflate output for zeros is toolchain dependent (about 510KB on
// go1.26, about 1MB on go1.27), and only the ratio matters to the test
require.Less(t, gz.Len(), expanded/100, "compressed payload should be tiny relative to what it expands to")
// both arms use io.ReadAll on purpose. The stream itself never holds much whatever the cap says, // both arms use io.ReadAll on purpose. The stream itself never holds much whatever the cap says,
// so copying to io.Discard measures nothing; what the cap actually protects is a consumer that // so copying to io.Discard measures nothing; what the cap actually protects is a consumer that
syft/pkg/cataloger/erlang/erlang_parser.go
+25 -6
View File
@@ -15,6 +15,10 @@ type erlangNode struct {
var errSkipComments = errors.New("") var errSkipComments = errors.New("")
var errUnexpectedEnd = errors.New("unexpected end of input")
var errTooDeep = fmt.Errorf("nesting deeper than %d levels", parsing.MaxDepth)
func (e erlangNode) Slice() []erlangNode { func (e erlangNode) Slice() []erlangNode {
out, ok := e.value.([]erlangNode) out, ok := e.value.([]erlangNode)
if ok { if ok {
@@ -79,7 +83,7 @@ func parseErlang(reader io.Reader) (erlangNode, error) {
} }
func parseErlangBlock(data []byte, i *int) (erlangNode, error) { func parseErlangBlock(data []byte, i *int) (erlangNode, error) {
block, err := parseErlangNode(data, i) block, err := parseErlangNode(data, i, 0)
if err != nil { if err != nil {
return node(nil), err return node(nil), err
} }
@@ -89,13 +93,19 @@ func parseErlangBlock(data []byte, i *int) (erlangNode, error) {
return block, nil return block, nil
} }
func parseErlangNode(data []byte, i *int) (erlangNode, error) { func parseErlangNode(data []byte, i *int, depth int) (erlangNode, error) {
parsing.SkipWhitespace(data, i) parsing.SkipWhitespace(data, i)
if *i >= len(data) {
return node(nil), errUnexpectedEnd
}
c := data[*i] c := data[*i]
switch c { switch c {
case '[', '{': case '[', '{':
offset := *i + 1 offset := *i + 1
parsing.SkipWhitespace(data, &offset) parsing.SkipWhitespace(data, &offset)
if offset >= len(data) {
return node(nil), errUnexpectedEnd
}
c2 := data[offset] c2 := data[offset]
// Add support for empty lists // Add support for empty lists
@@ -104,7 +114,7 @@ func parseErlangNode(data []byte, i *int) (erlangNode, error) {
return node(nil), nil return node(nil), nil
} }
return parseErlangList(data, i) return parseErlangList(data, i, depth+1)
case '"': case '"':
fallthrough fallthrough
case '\'': case '\'':
@@ -145,6 +155,9 @@ func parseErlangAngleString(data []byte, i *int) (erlangNode, error) {
} }
func parseErlangString(data []byte, i *int) (erlangNode, error) { func parseErlangString(data []byte, i *int) (erlangNode, error) {
if *i >= len(data) {
return node(nil), errUnexpectedEnd
}
delim := data[*i] delim := data[*i]
*i++ *i++
var buf bytes.Buffer var buf bytes.Buffer
@@ -167,13 +180,16 @@ func parseErlangString(data []byte, i *int) (erlangNode, error) {
return node(nil), fmt.Errorf("unterminated string at %d", *i) return node(nil), fmt.Errorf("unterminated string at %d", *i)
} }
func parseErlangList(data []byte, i *int) (erlangNode, error) { func parseErlangList(data []byte, i *int, depth int) (erlangNode, error) {
if depth > parsing.MaxDepth {
return node(nil), errTooDeep
}
*i++ *i++
out := erlangNode{ out := erlangNode{
value: []erlangNode{}, value: []erlangNode{},
} }
for *i < len(data) { for *i < len(data) {
item, err := parseErlangNode(data, i) item, err := parseErlangNode(data, i, depth)
if err != nil { if err != nil {
if err == errSkipComments { if err == errSkipComments {
parsing.SkipWhitespace(data, i) parsing.SkipWhitespace(data, i)
@@ -183,6 +199,9 @@ func parseErlangList(data []byte, i *int) (erlangNode, error) {
} }
out.value = append(out.value.([]erlangNode), item) out.value = append(out.value.([]erlangNode), item)
parsing.SkipWhitespace(data, i) parsing.SkipWhitespace(data, i)
if *i >= len(data) {
return node(nil), errUnexpectedEnd
}
c := data[*i] c := data[*i]
switch c { switch c {
case ',': case ',':
@@ -210,7 +229,7 @@ func parseErlangComment(data []byte, i *int) {
// Rest of a line is a comment. Deals with CR, LF and CR/LF // Rest of a line is a comment. Deals with CR, LF and CR/LF
if c == '\n' { if c == '\n' {
break break
} else if c == '\r' && data[*i] == '\n' { } else if c == '\r' && *i < len(data) && data[*i] == '\n' {
*i++ *i++
break break
} }
syft/pkg/cataloger/erlang/erlang_parser_test.go
+33
View File
@@ -2,6 +2,7 @@ package erlang
import ( import (
"bytes" "bytes"
"strings"
"testing" "testing"
"github.com/stretchr/testify/assert" "github.com/stretchr/testify/assert"
@@ -128,3 +129,35 @@ func Test_parseErlangString_escapedByte(t *testing.T) {
require.NoError(t, err) require.NoError(t, err)
assert.Equal(t, `a"b`, got.String()) assert.Equal(t, `a"b`, got.String())
} }
func TestParseErlang_deepNesting(t *testing.T) {
// 4MB of '[' overflowed the goroutine stack (fatal, not recoverable) before nesting was bounded
_, err := parseErlang(strings.NewReader(strings.Repeat("[a,", 4<<20)))
require.ErrorIs(t, err, errTooDeep)
// the error echoes a clipped window of the single-line input, not the whole thing
assert.Less(t, len(err.Error()), 1024)
}
func Test_parseErlang_truncated(t *testing.T) {
// each of these read past the end of the input before end-of-input was checked
tests := []struct {
name string
content string
wantErr require.ErrorAssertionFunc
}{
{name: "whitespace only", content: " ", wantErr: require.Error},
{name: "unterminated list", content: "[a", wantErr: require.Error},
{name: "unterminated empty list", content: "[ ", wantErr: require.Error},
{name: "lone binary string opener", content: "<", wantErr: require.Error},
{name: "comment ending in CR", content: "%\r", wantErr: require.NoError},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
var err error
require.NotPanics(t, func() {
_, err = parseErlang(strings.NewReader(tt.content))
})
tt.wantErr(t, err)
})
}
}
syft/pkg/cataloger/generic/cataloger.go
+25 -2
View File
@@ -2,6 +2,8 @@ package generic
import ( import (
"context" "context"
"fmt"
"runtime/debug"
"github.com/anchore/go-logger" "github.com/anchore/go-logger"
"github.com/anchore/go-sync" "github.com/anchore/go-sync"
@@ -221,12 +223,24 @@ func (c *Cataloger) Catalog(ctx context.Context, resolver file.Resolver) ([]pkg.
func (c *Cataloger) process(ctx context.Context, resolver file.Resolver, pkgs []pkg.Package, rels []artifact.Relationship, err error) ([]pkg.Package, []artifact.Relationship, error) { func (c *Cataloger) process(ctx context.Context, resolver file.Resolver, pkgs []pkg.Package, rels []artifact.Relationship, err error) ([]pkg.Package, []artifact.Relationship, error) {
for _, p := range c.processors { for _, p := range c.processors {
pkgs, rels, err = p.process(ctx, resolver, pkgs, rels, err) pkgs, rels, err = c.processSafely(ctx, p, resolver, pkgs, rels, err)
} }
return pkgs, rels, err return pkgs, rels, err
} }
func invokeParser(ctx context.Context, resolver file.Resolver, location file.Location, logger logger.Logger, parser Parser, env *Environment) ([]pkg.Package, []artifact.Relationship, error) { func (c *Cataloger) processSafely(ctx context.Context, p processExecutor, resolver file.Resolver, pkgs []pkg.Package, rels []artifact.Relationship, err error) (outPkgs []pkg.Package, outRels []artifact.Relationship, outErr error) {
defer func() {
if r := recover(); r != nil {
// keep the unprocessed results; there is no single location to attach an unknown to
log.WithFields("cataloger", c.upstreamCataloger, "panic", r).Warn("recovered from panic in package processor")
log.Tracef("processor panic stack:\n%s", debug.Stack())
outPkgs, outRels, outErr = pkgs, rels, err
}
}()
return p.process(ctx, resolver, pkgs, rels, err)
}
func invokeParser(ctx context.Context, resolver file.Resolver, location file.Location, logger logger.Logger, parser Parser, env *Environment) (pkgs []pkg.Package, rels []artifact.Relationship, err error) {
contentReader, err := resolver.FileContentsByLocation(location) contentReader, err := resolver.FileContentsByLocation(location)
if err != nil { if err != nil {
logger.WithFields("location", location.RealPath, "error", err).Debug("unable to fetch contents") logger.WithFields("location", location.RealPath, "error", err).Debug("unable to fetch contents")
@@ -234,6 +248,15 @@ func invokeParser(ctx context.Context, resolver file.Resolver, location file.Loc
} }
defer internal.CloseAndLogError(contentReader, location.AccessPath) defer internal.CloseAndLogError(contentReader, location.AccessPath)
defer func() {
if r := recover(); r != nil {
// a malformed file must not take down the whole SBOM; the caller records this as an unknown
logger.WithFields("location", location.RealPath, "panic", r).Debug("recovered from panic while parsing file")
logger.Tracef("parser panic stack:\n%s", debug.Stack())
pkgs, rels, err = nil, nil, fmt.Errorf("recovered from panic while parsing file: %v", r)
}
}()
discoveredPackages, discoveredRelationships, err := parser(ctx, resolver, env, file.NewLocationReadCloser(location, contentReader)) discoveredPackages, discoveredRelationships, err := parser(ctx, resolver, env, file.NewLocationReadCloser(location, contentReader))
if err != nil { if err != nil {
// these errors are propagated up, and are likely to be coordinate errors // these errors are propagated up, and are likely to be coordinate errors

Some files were not shown because too many files have changed in this diff Show More