mirror of
https://github.com/anchore/syft.git
synced 2026-10-15 15:47:23 +02:00
Compare commits
+3
-3
@@ -2,7 +2,7 @@ tools:
|
||||
# we want to use a pinned version of binny to manage the toolchain (so binny manages itself!)
|
||||
- name: binny
|
||||
version:
|
||||
want: v0.11.1
|
||||
want: v0.11.3
|
||||
method: github-release
|
||||
with:
|
||||
repo: anchore/binny
|
||||
@@ -26,7 +26,7 @@ tools:
|
||||
# used for linting
|
||||
- name: golangci-lint
|
||||
version:
|
||||
want: v2.8.0
|
||||
want: v2.9.0
|
||||
method: github-release
|
||||
with:
|
||||
repo: golangci/golangci-lint
|
||||
@@ -114,7 +114,7 @@ tools:
|
||||
# used to upload test fixture cache
|
||||
- name: yq
|
||||
version:
|
||||
want: v4.52.2
|
||||
want: v4.52.4
|
||||
method: github-release
|
||||
with:
|
||||
repo: mikefarah/yq
|
||||
|
||||
@@ -5,7 +5,7 @@ inputs:
|
||||
go-version:
|
||||
description: "Go version to install"
|
||||
required: true
|
||||
default: ">= 1.24"
|
||||
default: "1.25.x"
|
||||
go-dependencies:
|
||||
description: "Download go dependencies"
|
||||
required: true
|
||||
@@ -33,11 +33,12 @@ runs:
|
||||
if: inputs.go-version != ''
|
||||
with:
|
||||
go-version: ${{ inputs.go-version }}
|
||||
check-latest: true
|
||||
|
||||
- name: Restore tool cache
|
||||
if: inputs.tools == 'true'
|
||||
id: tool-cache
|
||||
uses: actions/cache@9255dc7a253b0ccc959486e2bca901246202afeb # v5.0.1
|
||||
uses: actions/cache@cdf6c1fa76f9f475f3d7449005a359c84ca0f306 # v5.0.3
|
||||
with:
|
||||
path: ${{ github.workspace }}/.tool
|
||||
key: ${{ inputs.cache-key-prefix }}-${{ runner.os }}-tool-${{ hashFiles('.binny.yaml') }}
|
||||
@@ -66,7 +67,7 @@ runs:
|
||||
|
||||
- name: Restore ORAS cache from github actions
|
||||
if: inputs.download-test-fixture-cache == 'true'
|
||||
uses: actions/cache@9255dc7a253b0ccc959486e2bca901246202afeb # v5.0.1
|
||||
uses: actions/cache@cdf6c1fa76f9f475f3d7449005a359c84ca0f306 # v5.0.3
|
||||
with:
|
||||
path: ${{ github.workspace }}/.tmp/oras-cache
|
||||
key: ${{ inputs.cache-key-prefix }}-oras-cache
|
||||
|
||||
@@ -44,6 +44,7 @@ jobs:
|
||||
uses: actions/setup-go@7a3fe6cf4cb3a834922a1244abfce67bcef6a0c5 #v6.2.0
|
||||
with:
|
||||
go-version-file: go.mod
|
||||
check-latest: true
|
||||
|
||||
# Initializes the CodeQL tools for scanning.
|
||||
- name: Initialize CodeQL
|
||||
|
||||
@@ -150,13 +150,13 @@ jobs:
|
||||
uses: ./.github/actions/bootstrap
|
||||
|
||||
- name: Login to Docker Hub
|
||||
uses: docker/login-action@5e57cd118135c172c3672efd75eb46360885c0ef #v3.6.0
|
||||
uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 #v3.7.0
|
||||
with:
|
||||
username: ${{ secrets.ANCHOREOSSWRITE_DH_USERNAME }}
|
||||
password: ${{ secrets.ANCHOREOSSWRITE_DH_PAT }}
|
||||
|
||||
- name: Login to GitHub Container Registry
|
||||
uses: docker/login-action@5e57cd118135c172c3672efd75eb46360885c0ef #v3.6.0
|
||||
uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 #v3.7.0
|
||||
with:
|
||||
registry: ghcr.io
|
||||
username: ${{ github.actor }}
|
||||
@@ -186,7 +186,7 @@ jobs:
|
||||
# for updating brew formula in anchore/homebrew-syft
|
||||
GITHUB_BREW_TOKEN: ${{ secrets.ANCHOREOPS_GITHUB_OSS_WRITE_TOKEN }}
|
||||
|
||||
- uses: anchore/sbom-action@0b82b0b1a22399a1c542d4d656f70cd903571b5c #v0.21.1
|
||||
- uses: anchore/sbom-action@28d71544de8eaf1b958d335707167c5f783590ad #v0.22.2
|
||||
continue-on-error: true
|
||||
with:
|
||||
file: go.mod
|
||||
|
||||
@@ -28,7 +28,7 @@ jobs:
|
||||
persist-credentials: false
|
||||
|
||||
- name: "Run zizmor"
|
||||
uses: zizmorcore/zizmor-action@135698455da5c3b3e55f73f4419e481ab68cdd95 # v0.4.1
|
||||
uses: zizmorcore/zizmor-action@0dce2577a4760a2749d8cfb7a84b7d5585ebcb7d # v0.5.0
|
||||
with:
|
||||
config: .github/zizmor.yml
|
||||
# Disable SARIF upload so the step is a simple pass/fail gate
|
||||
|
||||
+1
-2
@@ -50,10 +50,9 @@ linters:
|
||||
- legacy
|
||||
- std-error-handling
|
||||
rules:
|
||||
# internal/os contains OS feature detection logic; the name reflects its purpose
|
||||
# we have multiple packages in syft that might overlap with the stblib; their names reflect their purpose
|
||||
- linters:
|
||||
- revive
|
||||
path: internal/os/
|
||||
text: "var-naming: avoid package names that conflict"
|
||||
paths:
|
||||
- third_party$
|
||||
|
||||
+2
-1
@@ -545,7 +545,8 @@ tasks:
|
||||
- "go generate ./internal/capabilities/..."
|
||||
- "gofmt -s -w ./internal/capabilities"
|
||||
# now that we have the latest capabilities, run completeness tests to ensure this is self-consistent
|
||||
- "SYFT_ENABLE_COMPLETENESS_TESTS=true go test ./internal/capabilities/... -count=1"
|
||||
# note: -p 1 forces sequential package execution to avoid race between generate (writes) and internal (reads)
|
||||
- "SYFT_ENABLE_COMPLETENESS_TESTS=true go test -p 1 ./internal/capabilities/... -count=1"
|
||||
|
||||
|
||||
## Build-related targets #################################
|
||||
|
||||
@@ -59,7 +59,9 @@ const (
|
||||
nonImageSchemeHelp = ` {{.appName}} {{.command}} dir:path/to/yourproject read directly from a path on disk (any directory)
|
||||
{{.appName}} {{.command}} file:path/to/yourproject/file read directly from a path on disk (any single file)
|
||||
`
|
||||
scanSchemeHelp = "\n " + schemeHelpHeader + "\n" + imageSchemeHelp + nonImageSchemeHelp
|
||||
modelSchemeHelp = ` {{.appName}} {{.command}} oci-model-registry:ai/llama3.2 scan an OCI model artifact from a registry (e.g. Docker Hub AI models)
|
||||
`
|
||||
scanSchemeHelp = "\n " + schemeHelpHeader + "\n" + imageSchemeHelp + modelSchemeHelp + nonImageSchemeHelp
|
||||
|
||||
scanHelp = scanExample + scanSchemeHelp
|
||||
)
|
||||
|
||||
@@ -35,7 +35,8 @@ func TestYarnPackageLockDirectory(t *testing.T) {
|
||||
sbom, _ := catalogDirectory(t, "test-fixtures/yarn-lock")
|
||||
|
||||
foundPackages := strset.New()
|
||||
expectedPackages := strset.New("async@0.9.2", "async@3.2.3", "merge-objects@1.0.5", "should-type@1.3.0", "@4lolo/resize-observer-polyfill@1.5.2")
|
||||
// merge-objects and should-type are devDependencies in package.json and are excluded by default
|
||||
expectedPackages := strset.New("async@0.9.2", "async@3.2.3", "@4lolo/resize-observer-polyfill@1.5.2")
|
||||
|
||||
for actualPkg := range sbom.Artifacts.Packages.Enumerate(pkg.NpmPkg) {
|
||||
for _, actualLocation := range actualPkg.Locations.ToSlice() {
|
||||
|
||||
@@ -1,10 +1,10 @@
|
||||
module github.com/anchore/syft
|
||||
|
||||
go 1.24.6
|
||||
go 1.25
|
||||
|
||||
require (
|
||||
github.com/BurntSushi/toml v1.6.0
|
||||
github.com/CycloneDX/cyclonedx-go v0.9.3
|
||||
github.com/CycloneDX/cyclonedx-go v0.10.0
|
||||
github.com/Masterminds/semver/v3 v3.4.0
|
||||
github.com/Masterminds/sprig/v3 v3.3.0
|
||||
github.com/OneOfOne/xxhash v1.2.8
|
||||
@@ -23,13 +23,13 @@ require (
|
||||
github.com/anchore/go-testutils v0.0.0-20200925183923-d5f45b0d3c04
|
||||
github.com/anchore/go-version v1.2.2-0.20200701162849-18adb9c92b9b
|
||||
github.com/anchore/packageurl-go v0.1.1-0.20250220190351-d62adb6e1115
|
||||
github.com/anchore/stereoscope v0.1.19
|
||||
github.com/anchore/stereoscope v0.1.20
|
||||
github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be
|
||||
github.com/aquasecurity/go-pep440-version v0.0.1
|
||||
github.com/bitnami/go-version v0.0.0-20250131085805-b1f57a8634ef
|
||||
github.com/blakesmith/ar v0.0.0-20190502131153-809d4375e1fb
|
||||
github.com/bmatcuk/doublestar/v4 v4.9.2
|
||||
github.com/charmbracelet/bubbles v0.21.0
|
||||
github.com/bmatcuk/doublestar/v4 v4.10.0
|
||||
github.com/charmbracelet/bubbles v0.21.1
|
||||
github.com/charmbracelet/bubbletea v1.3.10
|
||||
github.com/charmbracelet/lipgloss v1.1.0
|
||||
github.com/dave/jennifer v1.7.1
|
||||
@@ -42,7 +42,7 @@ require (
|
||||
github.com/github/go-spdx/v2 v2.3.6
|
||||
github.com/gkampitakis/go-snaps v0.5.19
|
||||
github.com/go-git/go-billy/v5 v5.7.0
|
||||
github.com/go-git/go-git/v5 v5.16.4
|
||||
github.com/go-git/go-git/v5 v5.16.5
|
||||
github.com/go-test/deep v1.1.1
|
||||
github.com/go-viper/mapstructure/v2 v2.5.0
|
||||
github.com/gohugoio/hashstructure v0.6.0
|
||||
@@ -121,11 +121,11 @@ require (
|
||||
github.com/bodgit/plumbing v1.3.0 // indirect
|
||||
github.com/bodgit/sevenzip v1.6.1 // indirect
|
||||
github.com/bodgit/windows v1.0.1 // indirect
|
||||
github.com/charmbracelet/colorprofile v0.2.3-0.20250311203215-f60798e515dc // indirect
|
||||
github.com/charmbracelet/colorprofile v0.4.1 // indirect
|
||||
github.com/charmbracelet/harmonica v0.2.0 // indirect
|
||||
github.com/charmbracelet/x/ansi v0.10.1 // indirect
|
||||
github.com/charmbracelet/x/cellbuf v0.0.13-0.20250311204145-2c3ea96c31dd // indirect
|
||||
github.com/charmbracelet/x/term v0.2.1 // indirect
|
||||
github.com/charmbracelet/x/ansi v0.11.5 // indirect
|
||||
github.com/charmbracelet/x/cellbuf v0.0.15 // indirect
|
||||
github.com/charmbracelet/x/term v0.2.2 // indirect
|
||||
github.com/cloudflare/circl v1.6.1 // indirect
|
||||
github.com/containerd/containerd/api v1.10.0 // indirect
|
||||
github.com/containerd/continuity v0.4.5 // indirect
|
||||
@@ -139,7 +139,7 @@ require (
|
||||
github.com/containerd/typeurl/v2 v2.2.3 // indirect
|
||||
github.com/cyphar/filepath-securejoin v0.6.0 // indirect
|
||||
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
|
||||
github.com/docker/cli v29.1.5+incompatible // indirect
|
||||
github.com/docker/cli v29.2.0+incompatible // indirect
|
||||
github.com/docker/distribution v2.8.3+incompatible // indirect
|
||||
github.com/docker/docker v28.5.2+incompatible // indirect
|
||||
github.com/docker/docker-credential-helpers v0.9.4 // indirect
|
||||
@@ -177,7 +177,7 @@ require (
|
||||
github.com/kr/pretty v0.3.1 // indirect
|
||||
github.com/kr/text v0.2.0 // indirect
|
||||
github.com/logrusorgru/aurora v2.0.3+incompatible // indirect
|
||||
github.com/lucasb-eyer/go-colorful v1.2.0 // indirect
|
||||
github.com/lucasb-eyer/go-colorful v1.3.0 // indirect
|
||||
github.com/maruel/natural v1.1.1 // indirect
|
||||
github.com/mattn/go-colorable v0.1.14 // indirect
|
||||
github.com/mattn/go-isatty v0.0.20 // indirect
|
||||
@@ -307,9 +307,9 @@ require (
|
||||
github.com/aws/smithy-go v1.24.0 // indirect
|
||||
github.com/bahlo/generic-list-go v0.2.0 // indirect
|
||||
github.com/buger/jsonparser v1.1.1 // indirect
|
||||
github.com/clipperhouse/displaywidth v0.6.2 // indirect
|
||||
github.com/clipperhouse/displaywidth v0.9.0 // indirect
|
||||
github.com/clipperhouse/stringish v0.1.1 // indirect
|
||||
github.com/clipperhouse/uax29/v2 v2.3.0 // indirect
|
||||
github.com/clipperhouse/uax29/v2 v2.5.0 // indirect
|
||||
github.com/cncf/xds/go v0.0.0-20250501225837-2ac532fd4443 // indirect
|
||||
github.com/containerd/cgroups/v3 v3.1.2 // indirect
|
||||
github.com/containerd/containerd/v2 v2.2.1 // indirect
|
||||
|
||||
@@ -81,8 +81,8 @@ github.com/BurntSushi/toml v0.4.1/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbi
|
||||
github.com/BurntSushi/toml v1.6.0 h1:dRaEfpa2VI55EwlIW72hMRHdWouJeRF7TPYhI+AUQjk=
|
||||
github.com/BurntSushi/toml v1.6.0/go.mod h1:ukJfTF/6rtPPRCnwkur4qwRxa8vTRFBF0uk2lLoLwho=
|
||||
github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
|
||||
github.com/CycloneDX/cyclonedx-go v0.9.3 h1:Pyk/lwavPz7AaZNvugKFkdWOm93MzaIyWmBwmBo3aUI=
|
||||
github.com/CycloneDX/cyclonedx-go v0.9.3/go.mod h1:vcK6pKgO1WanCdd61qx4bFnSsDJQ6SbM2ZuMIgq86Jg=
|
||||
github.com/CycloneDX/cyclonedx-go v0.10.0 h1:7xyklU7YD+CUyGzSFIARG18NYLsKVn4QFg04qSsu+7Y=
|
||||
github.com/CycloneDX/cyclonedx-go v0.10.0/go.mod h1:vUvbCXQsEm48OI6oOlanxstwNByXjCZ2wuleUlwGEO8=
|
||||
github.com/DataDog/datadog-go v3.2.0+incompatible/go.mod h1:LButxg5PwREeZtORoXG3tL4fMGNddJ+vMq1mwgfaqoQ=
|
||||
github.com/DataDog/zstd v1.5.5 h1:oWf5W7GtOLgp6bciQYDmhHHjdhYkALu6S/5Ni9ZgSvQ=
|
||||
github.com/DataDog/zstd v1.5.5/go.mod h1:g4AWEaM3yOg3HYfnJ3YIawPnVdXJh9QME85blwSAmyw=
|
||||
@@ -152,8 +152,8 @@ github.com/anchore/go-version v1.2.2-0.20200701162849-18adb9c92b9b h1:e1bmaoJfZV
|
||||
github.com/anchore/go-version v1.2.2-0.20200701162849-18adb9c92b9b/go.mod h1:Bkc+JYWjMCF8OyZ340IMSIi2Ebf3uwByOk6ho4wne1E=
|
||||
github.com/anchore/packageurl-go v0.1.1-0.20250220190351-d62adb6e1115 h1:ZyRCmiEjnoGJZ1+Ah0ZZ/mKKqNhGcUZBl0s7PTTDzvY=
|
||||
github.com/anchore/packageurl-go v0.1.1-0.20250220190351-d62adb6e1115/go.mod h1:KoYIv7tdP5+CC9VGkeZV4/vGCKsY55VvoG+5dadg4YI=
|
||||
github.com/anchore/stereoscope v0.1.19 h1:1G5LVmRN1Sz6qNezpVAEeN7QfWwCE9zw9TJK1ZGnkvw=
|
||||
github.com/anchore/stereoscope v0.1.19/go.mod h1:+laNHlk05xA2YqgEzq8mxkFzclL3NRdeNIsiQQVeZZ4=
|
||||
github.com/anchore/stereoscope v0.1.20 h1:32720yZ/YtvzF5tvsoRL/ibdAJzOdIaR444fDXW4arQ=
|
||||
github.com/anchore/stereoscope v0.1.20/go.mod h1:6Ef0xQAuN2Ito7eV9A9pYjD1x/0cX5fy56MwgEGyrB4=
|
||||
github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883/go.mod h1:rCTlJbsFo29Kk6CurOXKm700vrz8f0KW0JNfpkRJY/8=
|
||||
github.com/andybalholm/brotli v1.2.0 h1:ukwgCxwYrmACq68yiUqwIWnGY0cTPox/M94sVwToPjQ=
|
||||
github.com/andybalholm/brotli v1.2.0/go.mod h1:rzTDkvFWvIrjDXZHkuS16NPggd91W3kUSvPlQ1pLaKY=
|
||||
@@ -215,8 +215,8 @@ github.com/aws/smithy-go v1.24.0 h1:LpilSUItNPFr1eY85RYgTIg5eIEPtvFbskaFcmmIUnk=
|
||||
github.com/aws/smithy-go v1.24.0/go.mod h1:LEj2LM3rBRQJxPZTB4KuzZkaZYnZPnvgIhb4pu07mx0=
|
||||
github.com/aymanbagabas/go-osc52/v2 v2.0.1 h1:HwpRHbFMcZLEVr42D4p7XBqjyuxQH5SMiErDT4WkJ2k=
|
||||
github.com/aymanbagabas/go-osc52/v2 v2.0.1/go.mod h1:uYgXzlJ7ZpABp8OJ+exZzJJhRNQ2ASbcXHWsFqH8hp8=
|
||||
github.com/aymanbagabas/go-udiff v0.2.0 h1:TK0fH4MteXUDspT88n8CKzvK0X9O2xu9yQjWpi6yML8=
|
||||
github.com/aymanbagabas/go-udiff v0.2.0/go.mod h1:RE4Ex0qsGkTAJoQdQQCA0uG+nAzJO/pI/QwceO5fgrA=
|
||||
github.com/aymanbagabas/go-udiff v0.3.1 h1:LV+qyBQ2pqe0u42ZsUEtPiCaUoqgA9gYRDs3vj1nolY=
|
||||
github.com/aymanbagabas/go-udiff v0.3.1/go.mod h1:G0fsKmG+P6ylD0r6N/KgQD/nWzgfnl8ZBcNLgcbrw8E=
|
||||
github.com/bahlo/generic-list-go v0.2.0 h1:5sz/EEAK+ls5wF+NeqDpk5+iNdMDXrh3z3nPnH1Wvgk=
|
||||
github.com/bahlo/generic-list-go v0.2.0/go.mod h1:2KvAjgMlE5NNynlg/5iLrrCCZ2+5xWbdbCW3pNTGyYg=
|
||||
github.com/becheran/wildmatch-go v1.0.0 h1:mE3dGGkTmpKtT4Z+88t8RStG40yN9T+kFEGj2PZFSzA=
|
||||
@@ -231,8 +231,8 @@ github.com/bitnami/go-version v0.0.0-20250131085805-b1f57a8634ef h1:TSFnfbbu2oAO
|
||||
github.com/bitnami/go-version v0.0.0-20250131085805-b1f57a8634ef/go.mod h1:9iglf1GG4oNRJ39bZ5AZrjgAFD2RwQbXw6Qf7Cs47wo=
|
||||
github.com/blakesmith/ar v0.0.0-20190502131153-809d4375e1fb h1:m935MPodAbYS46DG4pJSv7WO+VECIWUQ7OJYSoTrMh4=
|
||||
github.com/blakesmith/ar v0.0.0-20190502131153-809d4375e1fb/go.mod h1:PkYb9DJNAwrSvRx5DYA+gUcOIgTGVMNkfSCbZM8cWpI=
|
||||
github.com/bmatcuk/doublestar/v4 v4.9.2 h1:b0mc6WyRSYLjzofB2v/0cuDUZ+MqoGyH3r0dVij35GI=
|
||||
github.com/bmatcuk/doublestar/v4 v4.9.2/go.mod h1:xBQ8jztBU6kakFMg+8WGxn0c6z1fTSPVIjEY1Wr7jzc=
|
||||
github.com/bmatcuk/doublestar/v4 v4.10.0 h1:zU9WiOla1YA122oLM6i4EXvGW62DvKZVxIe6TYWexEs=
|
||||
github.com/bmatcuk/doublestar/v4 v4.10.0/go.mod h1:xBQ8jztBU6kakFMg+8WGxn0c6z1fTSPVIjEY1Wr7jzc=
|
||||
github.com/bodgit/plumbing v1.3.0 h1:pf9Itz1JOQgn7vEOE7v7nlEfBykYqvUYioC61TwWCFU=
|
||||
github.com/bodgit/plumbing v1.3.0/go.mod h1:JOTb4XiRu5xfnmdnDJo6GmSbSbtSyufrsyZFByMtKEs=
|
||||
github.com/bodgit/sevenzip v1.6.1 h1:kikg2pUMYC9ljU7W9SaqHXhym5HyKm8/M/jd31fYan4=
|
||||
@@ -252,24 +252,24 @@ github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XL
|
||||
github.com/cespare/xxhash/v2 v2.1.2/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
|
||||
github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs=
|
||||
github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
|
||||
github.com/charmbracelet/bubbles v0.21.0 h1:9TdC97SdRVg/1aaXNVWfFH3nnLAwOXr8Fn6u6mfQdFs=
|
||||
github.com/charmbracelet/bubbles v0.21.0/go.mod h1:HF+v6QUR4HkEpz62dx7ym2xc71/KBHg+zKwJtMw+qtg=
|
||||
github.com/charmbracelet/bubbles v0.21.1 h1:nj0decPiixaZeL9diI4uzzQTkkz1kYY8+jgzCZXSmW0=
|
||||
github.com/charmbracelet/bubbles v0.21.1/go.mod h1:HHvIYRCpbkCJw2yo0vNX1O5loCwSr9/mWS8GYSg50Sk=
|
||||
github.com/charmbracelet/bubbletea v1.3.10 h1:otUDHWMMzQSB0Pkc87rm691KZ3SWa4KUlvF9nRvCICw=
|
||||
github.com/charmbracelet/bubbletea v1.3.10/go.mod h1:ORQfo0fk8U+po9VaNvnV95UPWA1BitP1E0N6xJPlHr4=
|
||||
github.com/charmbracelet/colorprofile v0.2.3-0.20250311203215-f60798e515dc h1:4pZI35227imm7yK2bGPcfpFEmuY1gc2YSTShr4iJBfs=
|
||||
github.com/charmbracelet/colorprofile v0.2.3-0.20250311203215-f60798e515dc/go.mod h1:X4/0JoqgTIPSFcRA/P6INZzIuyqdFY5rm8tb41s9okk=
|
||||
github.com/charmbracelet/colorprofile v0.4.1 h1:a1lO03qTrSIRaK8c3JRxJDZOvhvIeSco3ej+ngLk1kk=
|
||||
github.com/charmbracelet/colorprofile v0.4.1/go.mod h1:U1d9Dljmdf9DLegaJ0nGZNJvoXAhayhmidOdcBwAvKk=
|
||||
github.com/charmbracelet/harmonica v0.2.0 h1:8NxJWRWg/bzKqqEaaeFNipOu77YR5t8aSwG4pgaUBiQ=
|
||||
github.com/charmbracelet/harmonica v0.2.0/go.mod h1:KSri/1RMQOZLbw7AHqgcBycp8pgJnQMYYT8QZRqZ1Ao=
|
||||
github.com/charmbracelet/lipgloss v1.1.0 h1:vYXsiLHVkK7fp74RkV7b2kq9+zDLoEU4MZoFqR/noCY=
|
||||
github.com/charmbracelet/lipgloss v1.1.0/go.mod h1:/6Q8FR2o+kj8rz4Dq0zQc3vYf7X+B0binUUBwA0aL30=
|
||||
github.com/charmbracelet/x/ansi v0.10.1 h1:rL3Koar5XvX0pHGfovN03f5cxLbCF2YvLeyz7D2jVDQ=
|
||||
github.com/charmbracelet/x/ansi v0.10.1/go.mod h1:3RQDQ6lDnROptfpWuUVIUG64bD2g2BgntdxH0Ya5TeE=
|
||||
github.com/charmbracelet/x/cellbuf v0.0.13-0.20250311204145-2c3ea96c31dd h1:vy0GVL4jeHEwG5YOXDmi86oYw2yuYUGqz6a8sLwg0X8=
|
||||
github.com/charmbracelet/x/cellbuf v0.0.13-0.20250311204145-2c3ea96c31dd/go.mod h1:xe0nKWGd3eJgtqZRaN9RjMtK7xUYchjzPr7q6kcvCCs=
|
||||
github.com/charmbracelet/x/ansi v0.11.5 h1:NBWeBpj/lJPE3Q5l+Lusa4+mH6v7487OP8K0r1IhRg4=
|
||||
github.com/charmbracelet/x/ansi v0.11.5/go.mod h1:2JNYLgQUsyqaiLovhU2Rv/pb8r6ydXKS3NIttu3VGZQ=
|
||||
github.com/charmbracelet/x/cellbuf v0.0.15 h1:ur3pZy0o6z/R7EylET877CBxaiE1Sp1GMxoFPAIztPI=
|
||||
github.com/charmbracelet/x/cellbuf v0.0.15/go.mod h1:J1YVbR7MUuEGIFPCaaZ96KDl5NoS0DAWkskup+mOY+Q=
|
||||
github.com/charmbracelet/x/exp/golden v0.0.0-20241011142426-46044092ad91 h1:payRxjMjKgx2PaCWLZ4p3ro9y97+TVLZNaRZgJwSVDQ=
|
||||
github.com/charmbracelet/x/exp/golden v0.0.0-20241011142426-46044092ad91/go.mod h1:wDlXFlCrmJ8J+swcL/MnGUuYnqgQdW9rhSD61oNMb6U=
|
||||
github.com/charmbracelet/x/term v0.2.1 h1:AQeHeLZ1OqSXhrAWpYUtZyX1T3zVxfpZuEQMIQaGIAQ=
|
||||
github.com/charmbracelet/x/term v0.2.1/go.mod h1:oQ4enTYFV7QN4m0i9mzHrViD7TQKvNEEkHUMCmsxdUg=
|
||||
github.com/charmbracelet/x/term v0.2.2 h1:xVRT/S2ZcKdhhOuSP4t5cLi5o+JxklsoEObBSgfgZRk=
|
||||
github.com/charmbracelet/x/term v0.2.2/go.mod h1:kF8CY5RddLWrsgVwpw4kAa6TESp6EB5y3uxGLeCqzAI=
|
||||
github.com/chromedp/cdproto v0.0.0-20230802225258-3cf4e6d46a89/go.mod h1:GKljq0VrfU4D5yc+2qA6OVr8pmO/MBbPEWqWQ/oqGEs=
|
||||
github.com/chromedp/chromedp v0.9.2/go.mod h1:LkSXJKONWTCHAfQasKFUZI+mxqS4tZqhmtGzzhLsnLs=
|
||||
github.com/chromedp/sysutil v1.0.0/go.mod h1:kgWmDdq8fTzXYcKIBqIYvRRTnYb9aNS9moAV0xufSww=
|
||||
@@ -282,12 +282,12 @@ github.com/chzyer/test v1.0.0/go.mod h1:2JlltgoNkt4TW/z9V/IzDdFaMTM2JPIi26O1pF38
|
||||
github.com/circonus-labs/circonus-gometrics v2.3.1+incompatible/go.mod h1:nmEj6Dob7S7YxXgwXpfOuvO54S+tGdZdw9fuRZt25Ag=
|
||||
github.com/circonus-labs/circonusllhist v0.1.3/go.mod h1:kMXHVDlOchFAehlya5ePtbp5jckzBHf4XRpQvBOLI+I=
|
||||
github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
|
||||
github.com/clipperhouse/displaywidth v0.6.2 h1:ZDpTkFfpHOKte4RG5O/BOyf3ysnvFswpyYrV7z2uAKo=
|
||||
github.com/clipperhouse/displaywidth v0.6.2/go.mod h1:R+kHuzaYWFkTm7xoMmK1lFydbci4X2CicfbGstSGg0o=
|
||||
github.com/clipperhouse/displaywidth v0.9.0 h1:Qb4KOhYwRiN3viMv1v/3cTBlz3AcAZX3+y9OLhMtAtA=
|
||||
github.com/clipperhouse/displaywidth v0.9.0/go.mod h1:aCAAqTlh4GIVkhQnJpbL0T/WfcrJXHcj8C0yjYcjOZA=
|
||||
github.com/clipperhouse/stringish v0.1.1 h1:+NSqMOr3GR6k1FdRhhnXrLfztGzuG+VuFDfatpWHKCs=
|
||||
github.com/clipperhouse/stringish v0.1.1/go.mod h1:v/WhFtE1q0ovMta2+m+UbpZ+2/HEXNWYXQgCt4hdOzA=
|
||||
github.com/clipperhouse/uax29/v2 v2.3.0 h1:SNdx9DVUqMoBuBoW3iLOj4FQv3dN5mDtuqwuhIGpJy4=
|
||||
github.com/clipperhouse/uax29/v2 v2.3.0/go.mod h1:Wn1g7MK6OoeDT0vL+Q0SQLDz/KpfsVRgg6W7ihQeh4g=
|
||||
github.com/clipperhouse/uax29/v2 v2.5.0 h1:x7T0T4eTHDONxFJsL94uKNKPHrclyFI0lm7+w94cO8U=
|
||||
github.com/clipperhouse/uax29/v2 v2.5.0/go.mod h1:Wn1g7MK6OoeDT0vL+Q0SQLDz/KpfsVRgg6W7ihQeh4g=
|
||||
github.com/cloudflare/circl v1.6.1 h1:zqIqSPIndyBh1bjLVVDHMPpVKqp8Su/V+6MeDzzQBQ0=
|
||||
github.com/cloudflare/circl v1.6.1/go.mod h1:uddAzsPgqdMAYatqJ0lsjX1oECcQLIlRpzZh3pJrofs=
|
||||
github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc=
|
||||
@@ -351,8 +351,8 @@ github.com/distribution/reference v0.6.0 h1:0IXCQ5g4/QMHHkarYzh5l+u8T3t73zM5Qvfr
|
||||
github.com/distribution/reference v0.6.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E=
|
||||
github.com/djherbis/times v1.6.0 h1:w2ctJ92J8fBvWPxugmXIv7Nz7Q3iDMKNx9v5ocVH20c=
|
||||
github.com/djherbis/times v1.6.0/go.mod h1:gOHeRAz2h+VJNZ5Gmc/o7iD9k4wW7NMVqieYCY99oc0=
|
||||
github.com/docker/cli v29.1.5+incompatible h1:GckbANUt3j+lsnQ6eCcQd70mNSOismSHWt8vk2AX8ao=
|
||||
github.com/docker/cli v29.1.5+incompatible/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8=
|
||||
github.com/docker/cli v29.2.0+incompatible h1:9oBd9+YM7rxjZLfyMGxjraKBKE4/nVyvVfN4qNl9XRM=
|
||||
github.com/docker/cli v29.2.0+incompatible/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8=
|
||||
github.com/docker/distribution v2.8.3+incompatible h1:AtKxIZ36LoNK51+Z6RpzLpddBirtxJnzDrHLEKxTAYk=
|
||||
github.com/docker/distribution v2.8.3+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w=
|
||||
github.com/docker/docker v28.5.2+incompatible h1:DBX0Y0zAjZbSrm1uzOkdr1onVghKaftjlSWt4AFexzM=
|
||||
@@ -436,8 +436,8 @@ github.com/go-git/go-billy/v5 v5.7.0 h1:83lBUJhGWhYp0ngzCMSgllhUSuoHP1iEWYjsPl9n
|
||||
github.com/go-git/go-billy/v5 v5.7.0/go.mod h1:/1IUejTKH8xipsAcdfcSAlUlo2J7lkYV8GTKxAT/L3E=
|
||||
github.com/go-git/go-git-fixtures/v4 v4.3.2-0.20231010084843-55a94097c399 h1:eMje31YglSBqCdIqdhKBW8lokaMrL3uTkpGYlE2OOT4=
|
||||
github.com/go-git/go-git-fixtures/v4 v4.3.2-0.20231010084843-55a94097c399/go.mod h1:1OCfN199q1Jm3HZlxleg+Dw/mwps2Wbk9frAWm+4FII=
|
||||
github.com/go-git/go-git/v5 v5.16.4 h1:7ajIEZHZJULcyJebDLo99bGgS0jRrOxzZG4uCk2Yb2Y=
|
||||
github.com/go-git/go-git/v5 v5.16.4/go.mod h1:4Ge4alE/5gPs30F2H1esi2gPd69R0C39lolkucHBOp8=
|
||||
github.com/go-git/go-git/v5 v5.16.5 h1:mdkuqblwr57kVfXri5TTH+nMFLNUxIj9Z7F5ykFbw5s=
|
||||
github.com/go-git/go-git/v5 v5.16.5/go.mod h1:QOMLpNf1qxuSY4StA/ArOdfFR2TrKEjJiye2kel2m+M=
|
||||
github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU=
|
||||
github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
|
||||
github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
|
||||
@@ -683,8 +683,8 @@ github.com/ledongthuc/pdf v0.0.0-20220302134840-0c2507a12d80/go.mod h1:imJHygn/1
|
||||
github.com/logrusorgru/aurora v0.0.0-20200102142835-e9ef32dff381/go.mod h1:7rIyQOR62GCctdiQpZ/zOJlFyk6y+94wXzv6RNZgaR4=
|
||||
github.com/logrusorgru/aurora v2.0.3+incompatible h1:tOpm7WcpBTn4fjmVfgpQq0EfczGlG91VSDkswnjF5A8=
|
||||
github.com/logrusorgru/aurora v2.0.3+incompatible/go.mod h1:7rIyQOR62GCctdiQpZ/zOJlFyk6y+94wXzv6RNZgaR4=
|
||||
github.com/lucasb-eyer/go-colorful v1.2.0 h1:1nnpGOrhyZZuNyfu1QjKiUICQ74+3FNCN69Aj6K7nkY=
|
||||
github.com/lucasb-eyer/go-colorful v1.2.0/go.mod h1:R4dSotOR9KMtayYi1e77YzuveK+i7ruzyGqttikkLy0=
|
||||
github.com/lucasb-eyer/go-colorful v1.3.0 h1:2/yBRLdWBZKrf7gB40FoiKfAWYQ0lqNcbuQwVHXptag=
|
||||
github.com/lucasb-eyer/go-colorful v1.3.0/go.mod h1:R4dSotOR9KMtayYi1e77YzuveK+i7ruzyGqttikkLy0=
|
||||
github.com/lyft/protoc-gen-star v0.5.3/go.mod h1:V0xaHgaf5oCCqmcxYcWiDfTiKsZsRc87/1qhoTACD8w=
|
||||
github.com/magiconair/properties v1.8.5/go.mod h1:y3VJvCyxH9uVvJTWEGAELF3aiYNyPKd5NZ3oSwXrF60=
|
||||
github.com/magiconair/properties v1.8.10 h1:s31yESBquKXCV9a/ScB3ESkOjUYYv+X0rg8SYxI99mE=
|
||||
|
||||
@@ -3,10 +3,12 @@ package internal
|
||||
const (
|
||||
// JSONSchemaVersion is the current schema version output by the JSON encoder
|
||||
// This is roughly following the "SchemaVer" guidelines for versioning the JSON schema. Please see schema/json/README.md for details on how to increment.
|
||||
JSONSchemaVersion = "16.1.2"
|
||||
JSONSchemaVersion = "16.1.3"
|
||||
|
||||
// Changelog
|
||||
// 16.1.0 - reformulated the python pdm fields (added "URL" and removed the unused "path" field).
|
||||
// 16.1.1 - correct elf package osCpe field according to the document of systemd (also add appCpe field)
|
||||
// 16.1.2 - placeholder for 16.1.2 changelog
|
||||
// 16.1.3 - add GGUFFileParts to GGUFFileHeader metadata
|
||||
|
||||
)
|
||||
|
||||
@@ -82,12 +82,29 @@ func assembleTypeContainer(items []any) (any, map[string]string) {
|
||||
return reflect.New(structType).Elem().Interface(), mapping
|
||||
}
|
||||
|
||||
//nolint:funlen
|
||||
func build() *jsonschema.Schema {
|
||||
// create metadata mapping first so we can use it in the Namer function for self-referential types
|
||||
pkgMetadataContainer, pkgMetadataMapping := assembleTypeContainer(packagemetadata.AllTypes())
|
||||
pkgMetadataContainerType := reflect.TypeOf(pkgMetadataContainer)
|
||||
|
||||
// create a set of valid metadata display names for lookup
|
||||
// (since Namer now returns display names, the schema definitions use display names as keys)
|
||||
pkgMetadataDisplayNames := make(map[string]struct{}, len(pkgMetadataMapping))
|
||||
for _, displayName := range pkgMetadataMapping {
|
||||
pkgMetadataDisplayNames[displayName] = struct{}{}
|
||||
}
|
||||
|
||||
reflector := &jsonschema.Reflector{
|
||||
BaseSchemaID: schemaID(),
|
||||
AllowAdditionalProperties: true,
|
||||
Namer: func(r reflect.Type) string {
|
||||
return strings.TrimPrefix(r.Name(), "JSON")
|
||||
name := strings.TrimPrefix(r.Name(), "JSON")
|
||||
// if this is a metadata type, use the mapped name for consistent references
|
||||
if mappedName, ok := pkgMetadataMapping[name]; ok {
|
||||
return mappedName
|
||||
}
|
||||
return name
|
||||
},
|
||||
CommentMap: make(map[string]string),
|
||||
}
|
||||
@@ -123,9 +140,6 @@ func build() *jsonschema.Schema {
|
||||
copyAliasFieldComments(reflector.CommentMap, repoRoot)
|
||||
}
|
||||
|
||||
pkgMetadataContainer, pkgMetadataMapping := assembleTypeContainer(packagemetadata.AllTypes())
|
||||
pkgMetadataContainerType := reflect.TypeOf(pkgMetadataContainer)
|
||||
|
||||
// srcMetadataContainer := assembleTypeContainer(sourcemetadata.AllTypes())
|
||||
// srcMetadataContainerType := reflect.TypeOf(srcMetadataContainer)
|
||||
|
||||
@@ -144,11 +158,10 @@ func build() *jsonschema.Schema {
|
||||
continue
|
||||
}
|
||||
|
||||
displayName, ok := pkgMetadataMapping[typeName]
|
||||
if ok {
|
||||
// this is a package metadata type...
|
||||
documentSchema.Definitions[displayName] = definition
|
||||
metadataNames = append(metadataNames, displayName)
|
||||
if _, ok := pkgMetadataDisplayNames[typeName]; ok {
|
||||
// this is a package metadata type (typeName is already the display name from Namer)
|
||||
documentSchema.Definitions[typeName] = definition
|
||||
metadataNames = append(metadataNames, typeName)
|
||||
} else {
|
||||
// this is a type that the metadata type uses (e.g. DpkgFileRecord)
|
||||
documentSchema.Definitions[typeName] = definition
|
||||
|
||||
@@ -26,6 +26,7 @@ var knownNonMetadataTypeNames = strset.New(
|
||||
// known to be metadata types themselves. Adding to this list will prevent the removal of the type from the schema.
|
||||
var knownMetadataTypeNames = strset.New(
|
||||
"DotnetPortableExecutableEntry",
|
||||
"GGUFFileHeader",
|
||||
)
|
||||
|
||||
func DiscoverTypeNames() ([]string, error) {
|
||||
|
||||
@@ -94,25 +94,33 @@ func findMetadataDefinitionNamesInFile(path string) ([]string, []string, error)
|
||||
|
||||
// loop over all types declared in the type declaration
|
||||
for _, typ := range spec.Specs {
|
||||
// check if the type is a struct type
|
||||
spec, ok := typ.(*ast.TypeSpec)
|
||||
if !ok || spec.Type == nil {
|
||||
typeSpec, ok := typ.(*ast.TypeSpec)
|
||||
if !ok || typeSpec.Type == nil {
|
||||
continue
|
||||
}
|
||||
|
||||
structType, ok := spec.Type.(*ast.StructType)
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
|
||||
// check if the struct type ends with "Metadata"
|
||||
name := spec.Name.String()
|
||||
name := typeSpec.Name.String()
|
||||
|
||||
// only look for exported types that end with "Metadata"
|
||||
if isMetadataTypeCandidate(name) {
|
||||
// print the full declaration of the struct type
|
||||
metadataDefinitions = append(metadataDefinitions, name)
|
||||
if !isMetadataTypeCandidate(name) {
|
||||
continue
|
||||
}
|
||||
|
||||
metadataDefinitions = append(metadataDefinitions, name)
|
||||
|
||||
// handle struct types (e.g., "type FooMetadata struct {...}")
|
||||
if structType, ok := typeSpec.Type.(*ast.StructType); ok {
|
||||
usedTypeNames = append(usedTypeNames, typeNamesUsedInStruct(structType)...)
|
||||
continue
|
||||
}
|
||||
|
||||
// handle type definitions from another type (e.g., "type FooMetadata BarMetadata")
|
||||
// if the base type is NOT a metadata candidate, track it as used
|
||||
// (e.g., we want both ImageMetadata and OCIModelMetadata which is an alias to it)
|
||||
if ident, ok := typeSpec.Type.(*ast.Ident); ok {
|
||||
if !isMetadataTypeCandidate(ident.Name) {
|
||||
usedTypeNames = append(usedTypeNames, ident.Name)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -6,5 +6,5 @@ import "github.com/anchore/syft/syft/source"
|
||||
|
||||
// AllTypes returns a list of all source metadata types that syft supports (that are represented in the source.Description.Metadata field).
|
||||
func AllTypes() []any {
|
||||
return []any{source.DirectoryMetadata{}, source.FileMetadata{}, source.ImageMetadata{}, source.SnapMetadata{}}
|
||||
return []any{source.DirectoryMetadata{}, source.FileMetadata{}, source.ImageMetadata{}, source.OCIModelMetadata{}, source.SnapMetadata{}}
|
||||
}
|
||||
|
||||
@@ -12,6 +12,7 @@ var jsonNameFromType = map[reflect.Type][]string{
|
||||
reflect.TypeOf(source.FileMetadata{}): {"file"},
|
||||
reflect.TypeOf(source.ImageMetadata{}): {"image"},
|
||||
reflect.TypeOf(source.SnapMetadata{}): {"snap"},
|
||||
reflect.TypeOf(source.OCIModelMetadata{}): {"oci-model"},
|
||||
}
|
||||
|
||||
func AllTypeNames() []string {
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
||||
"$id": "anchore.io/schema/syft/json/16.1.2/document",
|
||||
"$id": "anchore.io/schema/syft/json/16.1.3/document",
|
||||
"$ref": "#/$defs/Document",
|
||||
"$defs": {
|
||||
"AlpmDbEntry": {
|
||||
@@ -1478,6 +1478,13 @@
|
||||
"metadataHash": {
|
||||
"type": "string",
|
||||
"description": "MetadataKeyValuesHash is a xx64 hash of all key-value pairs from the GGUF header metadata.\nThis hash is computed over the complete header metadata (including the fields extracted\ninto typed fields above) and provides a stable identifier for the model configuration\nacross different file locations or remotes. It allows matching identical models even\nwhen stored in different repositories or with different filenames."
|
||||
},
|
||||
"parts": {
|
||||
"items": {
|
||||
"$ref": "#/$defs/GgufFileHeader"
|
||||
},
|
||||
"type": "array",
|
||||
"description": "Parts contains headers from additional GGUF files that were merged\ninto this package during post-processing (e.g., from OCI layers without model names)."
|
||||
}
|
||||
},
|
||||
"type": "object",
|
||||
|
||||
@@ -477,7 +477,7 @@ func (c *CreateSBOMConfig) Create(ctx context.Context, src source.Source) (*sbom
|
||||
|
||||
func findDefaultTags(src source.Description) ([]string, error) {
|
||||
switch m := src.Metadata.(type) {
|
||||
case source.ImageMetadata:
|
||||
case source.ImageMetadata, source.OCIModelMetadata:
|
||||
return []string{pkgcataloging.ImageTag, filecataloging.FileTag}, nil
|
||||
case source.FileMetadata, source.DirectoryMetadata:
|
||||
return []string{pkgcataloging.DirectoryTag, filecataloging.FileTag}, nil
|
||||
|
||||
@@ -13,16 +13,18 @@ import (
|
||||
)
|
||||
|
||||
var _ Resolver = (*MockResolver)(nil)
|
||||
var _ OCIMediaTypeResolver = (*MockResolver)(nil)
|
||||
|
||||
// MockResolver implements the FileResolver interface and is intended for use *only in test code*.
|
||||
// It provides an implementation that can resolve local filesystem paths using only a provided discrete list of file
|
||||
// paths, which are typically paths to test fixtures.
|
||||
type MockResolver struct {
|
||||
locations []Location
|
||||
metadata map[Coordinates]Metadata
|
||||
mimeTypeIndex map[string][]Location
|
||||
extension map[string][]Location
|
||||
basename map[string][]Location
|
||||
locations []Location
|
||||
metadata map[Coordinates]Metadata
|
||||
mimeTypeIndex map[string][]Location
|
||||
mediaTypeIndex map[string][]Location
|
||||
extension map[string][]Location
|
||||
basename map[string][]Location
|
||||
}
|
||||
|
||||
// NewMockResolverForPaths creates a new MockResolver, where the only resolvable
|
||||
@@ -72,6 +74,34 @@ func NewMockResolverForPathsWithMetadata(metadata map[Coordinates]Metadata) *Moc
|
||||
}
|
||||
}
|
||||
|
||||
// NewMockResolverForMediaTypes creates a MockResolver that can resolve files by media type.
|
||||
// The mediaTypes map specifies which locations should be returned for each media type.
|
||||
func NewMockResolverForMediaTypes(mediaTypes map[string][]Location) *MockResolver {
|
||||
var locations []Location
|
||||
mediaTypeIndex := make(map[string][]Location)
|
||||
extension := make(map[string][]Location)
|
||||
basename := make(map[string][]Location)
|
||||
|
||||
for mediaType, locs := range mediaTypes {
|
||||
mediaTypeIndex[mediaType] = append(mediaTypeIndex[mediaType], locs...)
|
||||
for _, l := range locs {
|
||||
locations = append(locations, l)
|
||||
ext := path.Ext(l.RealPath)
|
||||
extension[ext] = append(extension[ext], l)
|
||||
bn := path.Base(l.RealPath)
|
||||
basename[bn] = append(basename[bn], l)
|
||||
}
|
||||
}
|
||||
|
||||
return &MockResolver{
|
||||
locations: locations,
|
||||
metadata: make(map[Coordinates]Metadata),
|
||||
mediaTypeIndex: mediaTypeIndex,
|
||||
extension: extension,
|
||||
basename: basename,
|
||||
}
|
||||
}
|
||||
|
||||
// HasPath indicates if the given path exists in the underlying source.
|
||||
func (r MockResolver) HasPath(path string) bool {
|
||||
for _, l := range r.locations {
|
||||
@@ -189,6 +219,14 @@ func (r MockResolver) FilesByMIMEType(types ...string) ([]Location, error) {
|
||||
return locations, nil
|
||||
}
|
||||
|
||||
func (r MockResolver) FilesByMediaType(types ...string) ([]Location, error) {
|
||||
var locations []Location
|
||||
for _, ty := range types {
|
||||
locations = append(locations, r.mediaTypeIndex[ty]...)
|
||||
}
|
||||
return locations, nil
|
||||
}
|
||||
|
||||
func (r MockResolver) FilesByExtension(extensions ...string) ([]Location, error) {
|
||||
var results []Location
|
||||
for _, ext := range extensions {
|
||||
|
||||
@@ -52,6 +52,17 @@ type PathResolver interface {
|
||||
RelativeFileByPath(_ Location, path string) *Location
|
||||
}
|
||||
|
||||
// OCIMediaTypeResolver resolves single files as a layer in an OCI artifact for a given media type.
|
||||
type OCIMediaTypeResolver interface {
|
||||
// FilesByMediaType fetches a set of file references which the contents have been classified as one of the given Media Types.
|
||||
// The implementation for this may vary, however, this was first implemented to classify ai globs stored in OCI images.
|
||||
// The following considerations should be made when implementing:
|
||||
// - only return locations to files (NOT directories)
|
||||
// - locations for the implementer should be "/" and the fsid should be the layer digest the glob was found
|
||||
// - locations should be used with the FileContents API to return readers to the temporary data
|
||||
FilesByMediaType(types ...string) ([]Location, error)
|
||||
}
|
||||
|
||||
// LocationResolver provides iteration over all file locations in a source.
|
||||
type LocationResolver interface {
|
||||
// AllLocations returns a channel of all file references from the underlying source.
|
||||
|
||||
@@ -35,6 +35,7 @@ const (
|
||||
spdxPrimaryPurposeOther = "OTHER"
|
||||
|
||||
prefixImage = "Image"
|
||||
prefixOCIModel = "OCIModel"
|
||||
prefixDirectory = "Directory"
|
||||
prefixFile = "File"
|
||||
prefixSnap = "Snap"
|
||||
@@ -215,6 +216,36 @@ func toRootPackage(s source.Description) *spdx.Package {
|
||||
}
|
||||
}
|
||||
|
||||
case source.OCIModelMetadata:
|
||||
prefix = prefixOCIModel
|
||||
purpose = spdxPrimaryPurposeContainer
|
||||
|
||||
qualifiers := packageurl.Qualifiers{
|
||||
{
|
||||
Key: "arch",
|
||||
Value: m.Architecture,
|
||||
},
|
||||
}
|
||||
|
||||
ref, _ := reference.Parse(m.UserInput)
|
||||
if ref, ok := ref.(reference.NamedTagged); ok {
|
||||
qualifiers = append(qualifiers, packageurl.Qualifier{
|
||||
Key: "tag",
|
||||
Value: ref.Tag(),
|
||||
})
|
||||
}
|
||||
|
||||
c := toChecksum(m.ManifestDigest)
|
||||
if c != nil {
|
||||
checksums = append(checksums, *c)
|
||||
purl = &packageurl.PackageURL{
|
||||
Type: "oci",
|
||||
Name: s.Name,
|
||||
Version: m.ManifestDigest,
|
||||
Qualifiers: qualifiers,
|
||||
}
|
||||
}
|
||||
|
||||
case source.DirectoryMetadata:
|
||||
prefix = prefixDirectory
|
||||
purpose = spdxPrimaryPurposeFile
|
||||
|
||||
@@ -316,6 +316,81 @@ func Test_toFormatModel(t *testing.T) {
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "oci-model",
|
||||
in: sbom.SBOM{
|
||||
Source: source.Description{
|
||||
Name: "llama",
|
||||
Version: "sha256:d34db33f",
|
||||
Supplier: "Model Provider",
|
||||
Metadata: source.OCIModelMetadata{
|
||||
UserInput: "model-repo/llama:latest",
|
||||
ManifestDigest: "sha256:d34db33f",
|
||||
},
|
||||
},
|
||||
Artifacts: sbom.Artifacts{
|
||||
Packages: pkg.NewCollection(pkg.Package{
|
||||
Name: "pkg-1",
|
||||
Version: "version-1",
|
||||
}),
|
||||
},
|
||||
},
|
||||
expected: &spdx.Document{
|
||||
SPDXIdentifier: "DOCUMENT",
|
||||
SPDXVersion: spdx.Version,
|
||||
DataLicense: spdx.DataLicense,
|
||||
DocumentName: "llama",
|
||||
Packages: []*spdx.Package{
|
||||
{
|
||||
PackageSPDXIdentifier: "Package-pkg-1-pkg-1",
|
||||
PackageName: "pkg-1",
|
||||
PackageVersion: "version-1",
|
||||
PackageSupplier: &spdx.Supplier{
|
||||
Supplier: "Model Provider",
|
||||
SupplierType: "Organization",
|
||||
},
|
||||
},
|
||||
{
|
||||
PackageSPDXIdentifier: "DocumentRoot-OCIModel-llama",
|
||||
PackageName: "llama",
|
||||
PackageVersion: "sha256:d34db33f",
|
||||
PrimaryPackagePurpose: "CONTAINER",
|
||||
PackageChecksums: []spdx.Checksum{{Algorithm: "SHA256", Value: "d34db33f"}},
|
||||
PackageExternalReferences: []*v2_3.PackageExternalReference{
|
||||
{
|
||||
Category: "PACKAGE-MANAGER",
|
||||
RefType: "purl",
|
||||
Locator: "pkg:oci/llama@sha256%3Ad34db33f?arch=&tag=latest",
|
||||
},
|
||||
},
|
||||
PackageSupplier: &spdx.Supplier{
|
||||
Supplier: "Model Provider",
|
||||
SupplierType: "Organization",
|
||||
},
|
||||
},
|
||||
},
|
||||
Relationships: []*spdx.Relationship{
|
||||
{
|
||||
RefA: spdx.DocElementID{
|
||||
ElementRefID: "DocumentRoot-OCIModel-llama",
|
||||
},
|
||||
RefB: spdx.DocElementID{
|
||||
ElementRefID: "Package-pkg-1-pkg-1",
|
||||
},
|
||||
Relationship: spdx.RelationshipContains,
|
||||
},
|
||||
{
|
||||
RefA: spdx.DocElementID{
|
||||
ElementRefID: "DOCUMENT",
|
||||
},
|
||||
RefB: spdx.DocElementID{
|
||||
ElementRefID: "DocumentRoot-OCIModel-llama",
|
||||
},
|
||||
Relationship: spdx.RelationshipDescribes,
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
for _, test := range tests {
|
||||
|
||||
@@ -122,6 +122,9 @@ func toPath(s source.Description, p pkg.Package) string {
|
||||
case source.ImageMetadata:
|
||||
image := strings.ReplaceAll(metadata.UserInput, ":/", "//")
|
||||
return fmt.Sprintf("%s:/%s", image, packagePath)
|
||||
case source.OCIModelMetadata:
|
||||
image := strings.ReplaceAll(metadata.UserInput, ":/", "//")
|
||||
return fmt.Sprintf("%s:/%s", image, packagePath)
|
||||
case source.FileMetadata:
|
||||
path := trimRelative(metadata.Path)
|
||||
if isArchive(metadata.Path) {
|
||||
|
||||
@@ -178,6 +178,11 @@ func Test_toGithubModel(t *testing.T) {
|
||||
metadata: source.SnapMetadata{},
|
||||
testPath: "name:/etc",
|
||||
},
|
||||
{
|
||||
name: "oci-model",
|
||||
metadata: source.OCIModelMetadata{UserInput: "model-repo/llama:latest"},
|
||||
testPath: "model-repo/llama:latest:/etc",
|
||||
},
|
||||
}
|
||||
|
||||
for _, test := range tests {
|
||||
|
||||
@@ -12,6 +12,8 @@ func DocumentName(src source.Description) string {
|
||||
switch metadata := src.Metadata.(type) {
|
||||
case source.ImageMetadata:
|
||||
return metadata.UserInput
|
||||
case source.OCIModelMetadata:
|
||||
return metadata.UserInput
|
||||
case source.DirectoryMetadata:
|
||||
return metadata.Path
|
||||
case source.FileMetadata:
|
||||
|
||||
@@ -54,6 +54,17 @@ func Test_DocumentName(t *testing.T) {
|
||||
},
|
||||
expected: "some/name",
|
||||
},
|
||||
{
|
||||
name: "oci-model",
|
||||
srcMetadata: source.Description{
|
||||
Metadata: source.OCIModelMetadata{
|
||||
UserInput: "model-repo/name:tag",
|
||||
ID: "id",
|
||||
ManifestDigest: "digest",
|
||||
},
|
||||
},
|
||||
expected: "model-repo/name:tag",
|
||||
},
|
||||
{
|
||||
name: "named",
|
||||
srcMetadata: source.Description{
|
||||
|
||||
@@ -14,6 +14,7 @@ import (
|
||||
|
||||
const (
|
||||
InputImage = "image"
|
||||
InputOCIModel = "oci-model"
|
||||
InputDirectory = "dir"
|
||||
InputFile = "file"
|
||||
InputSnap = "snap"
|
||||
@@ -30,6 +31,8 @@ func DocumentNamespace(name string, src source.Description, desc sbom.Descriptor
|
||||
switch src.Metadata.(type) {
|
||||
case source.ImageMetadata:
|
||||
input = InputImage
|
||||
case source.OCIModelMetadata:
|
||||
input = InputOCIModel
|
||||
case source.DirectoryMetadata:
|
||||
input = InputDirectory
|
||||
case source.FileMetadata:
|
||||
|
||||
@@ -61,6 +61,18 @@ func Test_DocumentNamespace(t *testing.T) {
|
||||
},
|
||||
expected: "https://anchore.com/syft/snap/my-name-",
|
||||
},
|
||||
{
|
||||
name: "oci-model",
|
||||
inputName: "my-name",
|
||||
src: source.Description{
|
||||
Metadata: source.OCIModelMetadata{
|
||||
UserInput: "model-repo/name:tag",
|
||||
ID: "id",
|
||||
ManifestDigest: "digest",
|
||||
},
|
||||
},
|
||||
expected: "https://anchore.com/syft/oci-model/my-name-",
|
||||
},
|
||||
}
|
||||
for _, test := range tests {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
|
||||
@@ -83,7 +83,7 @@ func SourceInfo(p pkg.Package) string {
|
||||
case pkg.TerraformPkg:
|
||||
answer = "acquired package info from Terraform dependency lock file"
|
||||
case pkg.ModelPkg:
|
||||
answer = "acquired package info from AI artifact (e.g. GGUF File"
|
||||
answer = "acquired package info from AI artifact (e.g. GGUF File)"
|
||||
default:
|
||||
answer = "acquired package info from the following paths"
|
||||
}
|
||||
|
||||
@@ -190,6 +190,37 @@ func TestSource_UnmarshalJSON(t *testing.T) {
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "oci-model",
|
||||
input: []byte(`{
|
||||
"id": "foobar",
|
||||
"type": "oci-model",
|
||||
"metadata": {
|
||||
"userInput": "model-repo/llama:latest",
|
||||
"imageID": "sha256:e7b300aee9f9bf3433d32bc9305bfdd22183beb59d933b48d77ab56ba53a197a",
|
||||
"manifestDigest": "sha256:e515aad2ed234a5072c4d2ef86a1cb77d5bfe4b11aa865d9214875734c4eeb3c",
|
||||
"mediaType": "application/vnd.oci.image.manifest.v1+json",
|
||||
"tags": [],
|
||||
"imageSize": 5576169,
|
||||
"layers": [],
|
||||
"repoDigests": []
|
||||
}
|
||||
}`),
|
||||
expected: &Source{
|
||||
ID: "foobar",
|
||||
Type: "oci-model",
|
||||
Metadata: source.OCIModelMetadata{
|
||||
UserInput: "model-repo/llama:latest",
|
||||
ID: "sha256:e7b300aee9f9bf3433d32bc9305bfdd22183beb59d933b48d77ab56ba53a197a",
|
||||
ManifestDigest: "sha256:e515aad2ed234a5072c4d2ef86a1cb77d5bfe4b11aa865d9214875734c4eeb3c",
|
||||
MediaType: "application/vnd.oci.image.manifest.v1+json",
|
||||
Tags: []string{},
|
||||
Size: 5576169,
|
||||
Layers: []source.LayerMetadata{},
|
||||
RepoDigests: []string{},
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "unknown source type",
|
||||
input: []byte(`{
|
||||
|
||||
@@ -325,7 +325,17 @@ func toSourceModel(src source.Description) model.Source {
|
||||
Metadata: src.Metadata,
|
||||
}
|
||||
|
||||
if metadata, ok := src.Metadata.(source.ImageMetadata); ok {
|
||||
switch metadata := src.Metadata.(type) {
|
||||
case source.ImageMetadata:
|
||||
// ensure that empty collections are not shown as null
|
||||
if metadata.RepoDigests == nil {
|
||||
metadata.RepoDigests = []string{}
|
||||
}
|
||||
if metadata.Tags == nil {
|
||||
metadata.Tags = []string{}
|
||||
}
|
||||
m.Metadata = metadata
|
||||
case source.OCIModelMetadata:
|
||||
// ensure that empty collections are not shown as null
|
||||
if metadata.RepoDigests == nil {
|
||||
metadata.RepoDigests = []string{}
|
||||
|
||||
@@ -161,6 +161,34 @@ func Test_toSourceModel(t *testing.T) {
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "oci-model",
|
||||
src: source.Description{
|
||||
ID: "test-id",
|
||||
Name: "some-name",
|
||||
Version: "some-version",
|
||||
Metadata: source.OCIModelMetadata{
|
||||
UserInput: "user-input",
|
||||
ID: "id...",
|
||||
ManifestDigest: "digest...",
|
||||
MediaType: "type...",
|
||||
},
|
||||
},
|
||||
expected: model.Source{
|
||||
ID: "test-id",
|
||||
Name: "some-name",
|
||||
Version: "some-version",
|
||||
Type: "oci-model",
|
||||
Metadata: source.OCIModelMetadata{
|
||||
UserInput: "user-input",
|
||||
ID: "id...",
|
||||
ManifestDigest: "digest...",
|
||||
MediaType: "type...",
|
||||
RepoDigests: []string{},
|
||||
Tags: []string{},
|
||||
},
|
||||
},
|
||||
},
|
||||
// below are regression tests for when the name/version are not provided
|
||||
// historically we've hoisted up the name/version from the metadata, now it is a simple pass-through
|
||||
{
|
||||
@@ -225,6 +253,30 @@ func Test_toSourceModel(t *testing.T) {
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "oci-model - no name/version",
|
||||
src: source.Description{
|
||||
ID: "test-id",
|
||||
Metadata: source.OCIModelMetadata{
|
||||
UserInput: "user-input",
|
||||
ID: "id...",
|
||||
ManifestDigest: "digest...",
|
||||
MediaType: "type...",
|
||||
},
|
||||
},
|
||||
expected: model.Source{
|
||||
ID: "test-id",
|
||||
Type: "oci-model",
|
||||
Metadata: source.OCIModelMetadata{
|
||||
UserInput: "user-input",
|
||||
ID: "id...",
|
||||
ManifestDigest: "digest...",
|
||||
MediaType: "type...",
|
||||
RepoDigests: []string{},
|
||||
Tags: []string{},
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
for _, test := range tests {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
|
||||
@@ -130,6 +130,32 @@ func Test_toSyftSourceData(t *testing.T) {
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "oci-model",
|
||||
src: model.Source{
|
||||
ID: "the-id",
|
||||
Name: "some-name",
|
||||
Version: "some-version",
|
||||
Type: "oci-model",
|
||||
Metadata: source.OCIModelMetadata{
|
||||
UserInput: "user-input",
|
||||
ID: "id...",
|
||||
ManifestDigest: "digest...",
|
||||
MediaType: "type...",
|
||||
},
|
||||
},
|
||||
expected: &source.Description{
|
||||
ID: "the-id",
|
||||
Name: "some-name",
|
||||
Version: "some-version",
|
||||
Metadata: source.OCIModelMetadata{
|
||||
UserInput: "user-input",
|
||||
ID: "id...",
|
||||
ManifestDigest: "digest...",
|
||||
MediaType: "type...",
|
||||
},
|
||||
},
|
||||
},
|
||||
// below are regression tests for when the name/version are not provided
|
||||
// historically we've hoisted up the name/version from the metadata, now it is a simple pass-through
|
||||
{
|
||||
@@ -192,6 +218,28 @@ func Test_toSyftSourceData(t *testing.T) {
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "oci-model - no name/version",
|
||||
src: model.Source{
|
||||
ID: "the-id",
|
||||
Type: "oci-model",
|
||||
Metadata: source.OCIModelMetadata{
|
||||
UserInput: "user-input",
|
||||
ID: "id...",
|
||||
ManifestDigest: "digest...",
|
||||
MediaType: "type...",
|
||||
},
|
||||
},
|
||||
expected: &source.Description{
|
||||
ID: "the-id",
|
||||
Metadata: source.OCIModelMetadata{
|
||||
UserInput: "user-input",
|
||||
ID: "id...",
|
||||
ManifestDigest: "digest...",
|
||||
MediaType: "type...",
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
for _, test := range tests {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
|
||||
@@ -65,6 +65,8 @@ func validateSourcePlatform(src source.Source, cfg *GetSourceConfig) error {
|
||||
switch meta.(type) {
|
||||
case *source.ImageMetadata, source.ImageMetadata:
|
||||
return nil
|
||||
case *source.OCIModelMetadata, source.OCIModelMetadata:
|
||||
return nil
|
||||
case *source.SnapMetadata, source.SnapMetadata:
|
||||
return nil
|
||||
default:
|
||||
|
||||
@@ -31,7 +31,8 @@ func TestGetProviders_Sources(t *testing.T) {
|
||||
t.Errorf("Expected no error for Sources parameter, got: %v", err)
|
||||
}
|
||||
|
||||
if len(providers) != 1 {
|
||||
t.Errorf("Expected 1 providers, got %d", len(providers))
|
||||
// Registry tag has two providers: OCIModel and Image
|
||||
if len(providers) != 2 {
|
||||
t.Errorf("Expected 2 providers, got %d", len(providers))
|
||||
}
|
||||
}
|
||||
|
||||
@@ -111,6 +111,10 @@ func TestValidateSourcePlatform_SupportedMetadataTypes(t *testing.T) {
|
||||
metadata: source.FileMetadata{},
|
||||
wantErr: require.Error,
|
||||
},
|
||||
{
|
||||
name: "oci-model",
|
||||
metadata: source.OCIModelMetadata{},
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
|
||||
@@ -0,0 +1,141 @@
|
||||
package fileresolver
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
|
||||
"github.com/anchore/syft/syft/file"
|
||||
)
|
||||
|
||||
var _ file.Resolver = (*ContainerImageModel)(nil)
|
||||
var _ file.OCIMediaTypeResolver = (*ContainerImageModel)(nil)
|
||||
|
||||
// LayerInfo holds information about an OCI model layer file stored on disk.
|
||||
type LayerInfo struct {
|
||||
TempPath string // Path to the temp file on disk
|
||||
MediaType string // OCI media type of the layer
|
||||
}
|
||||
|
||||
// ContainerImageModel is a file.Resolver implementation that provides access to
|
||||
// GGUF header data fetched from OCI model artifacts via range-GET requests.
|
||||
// This does not fetch the entire model from the registry, only a sliver of it.
|
||||
type ContainerImageModel struct {
|
||||
tempDir string // temp directory containing all layer files
|
||||
layerFiles map[string]LayerInfo // digest -> layer info (temp path + media type)
|
||||
locations map[string]file.Location // digest -> location
|
||||
}
|
||||
|
||||
// NewContainerImageModel creates a new resolver with the given temp directory and layer files.
|
||||
func NewContainerImageModel(tempDir string, layerFiles map[string]LayerInfo) *ContainerImageModel {
|
||||
// Create locations for all layer files
|
||||
// Each location has RealPath="/", FileSystemID=digest, AccessPath="/"
|
||||
locations := make(map[string]file.Location, len(layerFiles))
|
||||
for digest := range layerFiles {
|
||||
// Use NewVirtualLocationFromCoordinates with digest as FileSystemID
|
||||
coords := file.NewCoordinates("/", digest)
|
||||
locations[digest] = file.NewVirtualLocationFromCoordinates(coords, "/")
|
||||
}
|
||||
|
||||
return &ContainerImageModel{
|
||||
tempDir: tempDir,
|
||||
layerFiles: layerFiles,
|
||||
locations: locations,
|
||||
}
|
||||
}
|
||||
|
||||
// FilesByMediaType returns locations for layers matching the given media type patterns.
|
||||
// Patterns support glob-style matching (e.g., "application/vnd.docker.ai*").
|
||||
func (r *ContainerImageModel) FilesByMediaType(types ...string) ([]file.Location, error) {
|
||||
var matches []file.Location
|
||||
|
||||
for digest, info := range r.layerFiles {
|
||||
for _, pattern := range types {
|
||||
matched, err := filepath.Match(pattern, info.MediaType)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("invalid media type pattern %q: %w", pattern, err)
|
||||
}
|
||||
if matched {
|
||||
if loc, ok := r.locations[digest]; ok {
|
||||
matches = append(matches, loc)
|
||||
}
|
||||
break // Don't add the same location twice
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return matches, nil
|
||||
}
|
||||
|
||||
// FileContentsByLocation returns the contents of the file at the given location.
|
||||
// The location's FileSystemID contains the layer digest, which is used to look up the temp file.
|
||||
// This method is used as part of the content selection in the generic cataloger when locations
|
||||
// are returned by searching for contents by media type.
|
||||
func (r *ContainerImageModel) FileContentsByLocation(location file.Location) (io.ReadCloser, error) {
|
||||
// Look up the temp file path using the digest stored in FileSystemID
|
||||
digest := location.FileSystemID
|
||||
info, ok := r.layerFiles[digest]
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("no file found for digest %q", digest)
|
||||
}
|
||||
return os.Open(info.TempPath)
|
||||
}
|
||||
|
||||
// FileMetadataByLocation returns metadata for the file at the given location.
|
||||
func (r *ContainerImageModel) FileMetadataByLocation(_ file.Location) (m file.Metadata, err error) {
|
||||
return m, nil
|
||||
}
|
||||
|
||||
// HasPath checks if the given path exists in the resolver.
|
||||
func (r *ContainerImageModel) HasPath(path string) bool {
|
||||
// The virtual path is "/" for all files
|
||||
if path == "/" && len(r.layerFiles) > 0 {
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// FilesByPath returns locations for files matching the given paths.
|
||||
func (r *ContainerImageModel) FilesByPath(_ ...string) ([]file.Location, error) {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
// FilesByGlob returns locations for files matching the given glob patterns.
|
||||
func (r *ContainerImageModel) FilesByGlob(_ ...string) ([]file.Location, error) {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
// FilesByMIMEType returns locations for files with the given MIME types.
|
||||
// This is not implemented for OCI model artifacts as we don't have MIME type detection.
|
||||
func (r *ContainerImageModel) FilesByMIMEType(_ ...string) ([]file.Location, error) {
|
||||
// Not implemented - OCI model artifacts don't have MIME type detection
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
// RelativeFileByPath returns a file at the given path relative to the reference location.
|
||||
// This is not applicable for OCI model artifacts.
|
||||
func (r *ContainerImageModel) RelativeFileByPath(_ file.Location, _ string) *file.Location {
|
||||
// Not implemented - no layer hierarchy in OCI model artifacts
|
||||
return nil
|
||||
}
|
||||
|
||||
// AllLocations returns all file locations in the resolver.
|
||||
func (r *ContainerImageModel) AllLocations(ctx context.Context) <-chan file.Location {
|
||||
ch := make(chan file.Location)
|
||||
|
||||
go func() {
|
||||
defer close(ch)
|
||||
|
||||
for _, loc := range r.locations {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case ch <- loc:
|
||||
}
|
||||
}
|
||||
}()
|
||||
|
||||
return ch
|
||||
}
|
||||
@@ -0,0 +1,130 @@
|
||||
package fileresolver
|
||||
|
||||
import (
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"github.com/anchore/syft/syft/file"
|
||||
)
|
||||
|
||||
const ggufLayerMediaType = "application/vnd.docker.ai.gguf.v3"
|
||||
|
||||
func TestOCIModelResolver_FilesByMediaType(t *testing.T) {
|
||||
tempDir := t.TempDir()
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
layerFiles map[string]LayerInfo
|
||||
patterns []string
|
||||
expected int
|
||||
}{
|
||||
{
|
||||
name: "exact match GGUF",
|
||||
layerFiles: map[string]LayerInfo{
|
||||
"sha256:abc123": {TempPath: filepath.Join(tempDir, "f1"), MediaType: ggufLayerMediaType},
|
||||
},
|
||||
patterns: []string{ggufLayerMediaType},
|
||||
expected: 1,
|
||||
},
|
||||
{
|
||||
name: "glob match docker ai",
|
||||
layerFiles: map[string]LayerInfo{
|
||||
"sha256:abc123": {TempPath: filepath.Join(tempDir, "f1"), MediaType: ggufLayerMediaType},
|
||||
},
|
||||
patterns: []string{"application/vnd.docker.ai*"},
|
||||
expected: 1,
|
||||
},
|
||||
{
|
||||
name: "no match",
|
||||
layerFiles: map[string]LayerInfo{
|
||||
"sha256:abc123": {TempPath: filepath.Join(tempDir, "f1"), MediaType: ggufLayerMediaType},
|
||||
},
|
||||
patterns: []string{"application/json"},
|
||||
expected: 0,
|
||||
},
|
||||
{
|
||||
name: "multiple patterns match multiple files",
|
||||
layerFiles: map[string]LayerInfo{
|
||||
"sha256:abc123": {TempPath: filepath.Join(tempDir, "f1"), MediaType: ggufLayerMediaType},
|
||||
"sha256:def456": {TempPath: filepath.Join(tempDir, "f2"), MediaType: "application/octet-stream"},
|
||||
},
|
||||
patterns: []string{ggufLayerMediaType, "application/octet-stream"},
|
||||
expected: 2,
|
||||
},
|
||||
}
|
||||
|
||||
for _, test := range tests {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
resolver := NewContainerImageModel(tempDir, test.layerFiles)
|
||||
|
||||
locations, err := resolver.FilesByMediaType(test.patterns...)
|
||||
require.NoError(t, err)
|
||||
assert.Len(t, locations, test.expected)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestOCIModelResolver_FileContentsByLocation(t *testing.T) {
|
||||
tempDir := t.TempDir()
|
||||
content := []byte("test gguf content")
|
||||
|
||||
tempFile := filepath.Join(tempDir, "test.gguf")
|
||||
require.NoError(t, os.WriteFile(tempFile, content, 0600))
|
||||
|
||||
digest := "sha256:abc123"
|
||||
layerFiles := map[string]LayerInfo{
|
||||
digest: {TempPath: tempFile, MediaType: ggufLayerMediaType},
|
||||
}
|
||||
|
||||
resolver := NewContainerImageModel(tempDir, layerFiles)
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
digest string
|
||||
wantErr bool
|
||||
wantData []byte
|
||||
errSubstr string
|
||||
}{
|
||||
{
|
||||
name: "valid location returns content",
|
||||
digest: digest,
|
||||
wantErr: false,
|
||||
wantData: content,
|
||||
},
|
||||
{
|
||||
name: "invalid digest returns error",
|
||||
digest: "sha256:invalid",
|
||||
wantErr: true,
|
||||
errSubstr: "no file found for digest",
|
||||
},
|
||||
}
|
||||
|
||||
for _, test := range tests {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
loc := file.NewVirtualLocationFromCoordinates(
|
||||
file.NewCoordinates("/", test.digest),
|
||||
"/",
|
||||
)
|
||||
|
||||
reader, err := resolver.FileContentsByLocation(loc)
|
||||
|
||||
if test.wantErr {
|
||||
require.Error(t, err)
|
||||
assert.Contains(t, err.Error(), test.errSubstr)
|
||||
return
|
||||
}
|
||||
|
||||
require.NoError(t, err)
|
||||
defer reader.Close()
|
||||
|
||||
data, err := io.ReadAll(reader)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, test.wantData, data)
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -9,8 +9,17 @@ import (
|
||||
"github.com/anchore/syft/syft/pkg/cataloger/generic"
|
||||
)
|
||||
|
||||
const (
|
||||
catalogerName = "gguf-cataloger"
|
||||
ggufLayerMediaType = "application/vnd.docker.ai*"
|
||||
)
|
||||
|
||||
// NewGGUFCataloger returns a new cataloger instance for GGUF model files.
|
||||
// It supports both traditional file-based discovery and OCI layer-aware discovery
|
||||
// when the source for the SBOM is the oci model source
|
||||
func NewGGUFCataloger() pkg.Cataloger {
|
||||
return generic.NewCataloger("gguf-cataloger").
|
||||
WithParserByGlobs(parseGGUFModel, "**/*.gguf")
|
||||
return generic.NewCataloger(catalogerName).
|
||||
WithParserByGlobs(parseGGUFModel, "**/*.gguf").
|
||||
WithParserByMediaType(parseGGUFModel, ggufLayerMediaType).
|
||||
WithProcessors(ggufMergeProcessor)
|
||||
}
|
||||
|
||||
@@ -122,6 +122,10 @@ func extractVersion(kvs gguf_parser.GGUFMetadataKVs) string {
|
||||
|
||||
// extractModelNameFromPath extracts the model name from the file path
|
||||
func extractModelNameFromPath(path string) string {
|
||||
// we do not want to return a name from filepath if it's not a distinct gguf file
|
||||
if !strings.Contains(path, ".gguf") {
|
||||
return ""
|
||||
}
|
||||
// Get the base filename
|
||||
base := filepath.Base(path)
|
||||
|
||||
|
||||
@@ -0,0 +1,59 @@
|
||||
package ai
|
||||
|
||||
import (
|
||||
"github.com/anchore/syft/syft/artifact"
|
||||
"github.com/anchore/syft/syft/pkg"
|
||||
)
|
||||
|
||||
// ggufMergeProcessor consolidates multiple GGUF packages into a single package
|
||||
// representing the AI model. When scanning OCI images with multiple layers,
|
||||
// each layer may produce a separate package. This processor finds the package
|
||||
// with a name and merges metadata from nameless packages into its GGUFFileParts field.
|
||||
// Only packages with a non-empty name are returned in the final result.
|
||||
func ggufMergeProcessor(pkgs []pkg.Package, rels []artifact.Relationship, err error) ([]pkg.Package, []artifact.Relationship, error) {
|
||||
if err != nil {
|
||||
return pkgs, rels, err
|
||||
}
|
||||
|
||||
if len(pkgs) == 0 {
|
||||
return pkgs, rels, err
|
||||
}
|
||||
|
||||
// Separate packages with names from those without
|
||||
var namedPkgs []pkg.Package
|
||||
var namelessHeaders []pkg.GGUFFileHeader
|
||||
|
||||
for _, p := range pkgs {
|
||||
if p.Name != "" {
|
||||
namedPkgs = append(namedPkgs, p)
|
||||
} else {
|
||||
if header, ok := p.Metadata.(pkg.GGUFFileHeader); ok {
|
||||
// We do not want a kv hash for nameless headers
|
||||
header.MetadataKeyValuesHash = ""
|
||||
namelessHeaders = append(namelessHeaders, header)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// If there are no named packages, return nothing
|
||||
if len(namedPkgs) == 0 {
|
||||
return nil, rels, err
|
||||
}
|
||||
|
||||
// merge nameless headers into a single named package;
|
||||
// if there are multiple named packages, return them without trying to merge headers.
|
||||
// we cannot determine which nameless headers belong to which package
|
||||
// this is because the order we receive the gguf headers in is not guaranteed
|
||||
// to match the layer order in the original oci image
|
||||
if len(namedPkgs) == 1 && len(namelessHeaders) > 0 {
|
||||
winner := &namedPkgs[0]
|
||||
if header, ok := winner.Metadata.(pkg.GGUFFileHeader); ok {
|
||||
header.Parts = namelessHeaders
|
||||
winner.Metadata = header
|
||||
}
|
||||
}
|
||||
|
||||
// Largest number of key value
|
||||
|
||||
return namedPkgs, rels, err
|
||||
}
|
||||
@@ -0,0 +1,63 @@
|
||||
package ai
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"github.com/anchore/syft/syft/pkg"
|
||||
)
|
||||
|
||||
func Test_ggufMergeProcessor(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
pkgs []pkg.Package
|
||||
wantPkgCount int
|
||||
wantFilePartCount int
|
||||
}{
|
||||
{
|
||||
name: "single named package merges nameless headers",
|
||||
pkgs: []pkg.Package{
|
||||
{Name: "model", Metadata: pkg.GGUFFileHeader{MetadataKeyValuesHash: "abc"}},
|
||||
{Name: "", Metadata: pkg.GGUFFileHeader{MetadataKeyValuesHash: "part1"}},
|
||||
{Name: "", Metadata: pkg.GGUFFileHeader{MetadataKeyValuesHash: "part2"}},
|
||||
},
|
||||
wantPkgCount: 1,
|
||||
wantFilePartCount: 2,
|
||||
},
|
||||
{
|
||||
name: "multiple named packages returns all without merging",
|
||||
pkgs: []pkg.Package{
|
||||
{Name: "model1", Metadata: pkg.GGUFFileHeader{}},
|
||||
{Name: "model2", Metadata: pkg.GGUFFileHeader{}},
|
||||
{Name: "", Metadata: pkg.GGUFFileHeader{}},
|
||||
},
|
||||
wantPkgCount: 2,
|
||||
wantFilePartCount: 0,
|
||||
},
|
||||
{
|
||||
name: "no named packages returns empty result",
|
||||
pkgs: []pkg.Package{
|
||||
{Name: "", Metadata: pkg.GGUFFileHeader{}},
|
||||
{Name: "", Metadata: pkg.GGUFFileHeader{}},
|
||||
},
|
||||
wantPkgCount: 0,
|
||||
wantFilePartCount: 0,
|
||||
},
|
||||
}
|
||||
|
||||
for _, test := range tests {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
got, _, err := ggufMergeProcessor(test.pkgs, nil, nil)
|
||||
require.NoError(t, err)
|
||||
assert.Len(t, got, test.wantPkgCount)
|
||||
|
||||
if test.wantPkgCount == 1 && test.wantFilePartCount > 0 {
|
||||
header, ok := got[0].Metadata.(pkg.GGUFFileHeader)
|
||||
require.True(t, ok)
|
||||
assert.Len(t, header.Parts, test.wantFilePartCount)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -2164,6 +2164,11 @@ func (p *panicyResolver) FilesByMIMEType(_ ...string) ([]file.Location, error) {
|
||||
return nil, errors.New("not implemented")
|
||||
}
|
||||
|
||||
func (p *panicyResolver) FilesByMediaType(_ ...string) ([]file.Location, error) {
|
||||
p.searchCalled = true
|
||||
return nil, errors.New("not implemented")
|
||||
}
|
||||
|
||||
func (p *panicyResolver) RelativeFileByPath(_ file.Location, _ string) *file.Location {
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
package debian
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"io"
|
||||
"regexp"
|
||||
"sort"
|
||||
@@ -14,23 +15,32 @@ import (
|
||||
// For more information see: https://www.debian.org/doc/packaging-manuals/copyright-format/1.0/#license-syntax
|
||||
|
||||
var (
|
||||
licensePattern = regexp.MustCompile(`^License: (?P<license>\S*)`)
|
||||
commonLicensePathPattern = regexp.MustCompile(`/usr/share/common-licenses/(?P<license>[0-9A-Za-z_.\-]+)`)
|
||||
licenseFirstSentenceAfterHeadingPattern = regexp.MustCompile(`(?is)^[^\n]+?\n[-]+?\n+(?P<license>.*?\.)`)
|
||||
licenseAgreementHeadingPattern = regexp.MustCompile(`(?i)^\s*(?P<license>LICENSE AGREEMENT(?: FOR .+?)?)\s*$`)
|
||||
licensePattern = regexp.MustCompile(`^License: (?P<license>\S*)`)
|
||||
commonLicensePathPattern = regexp.MustCompile(`/usr/share/common-licenses/(?P<license>[0-9A-Za-z_.\-]+)`)
|
||||
licenseAgreementHeadingPattern = regexp.MustCompile(`(?i)^\s*(?P<license>LICENSE AGREEMENT(?: FOR .+?)?)\s*$`)
|
||||
)
|
||||
|
||||
func parseLicensesFromCopyright(reader io.Reader) []string {
|
||||
findings := strset.New()
|
||||
data, err := io.ReadAll(reader)
|
||||
if err != nil {
|
||||
// Fail-safe: return nothing if unable to read
|
||||
return []string{}
|
||||
}
|
||||
scanner := bufio.NewScanner(reader)
|
||||
|
||||
content := string(data)
|
||||
lines := strings.Split(content, "\n")
|
||||
for _, line := range lines {
|
||||
// State machine replacing licenseFirstSentenceAfterHeadingPattern.
|
||||
// That regex only matched at the start of the file: a non-empty heading,
|
||||
// a line of dashes, blank lines, then text up to the first period.
|
||||
const (
|
||||
expectHeading = iota
|
||||
expectDashes
|
||||
skipBlanks
|
||||
captureLicense
|
||||
headingDone // matched or impossible — stop checking
|
||||
)
|
||||
headingState := expectHeading
|
||||
var licenseText strings.Builder
|
||||
|
||||
for scanner.Scan() {
|
||||
line := scanner.Text()
|
||||
|
||||
// per-line regex checks (applied to every line)
|
||||
if value := findLicenseClause(licensePattern, line); value != "" {
|
||||
findings.Add(value)
|
||||
}
|
||||
@@ -40,13 +50,39 @@ func parseLicensesFromCopyright(reader io.Reader) []string {
|
||||
if value := findLicenseClause(licenseAgreementHeadingPattern, line); value != "" {
|
||||
findings.Add(value)
|
||||
}
|
||||
}
|
||||
|
||||
// some copyright files have a license declaration after the heading ex:
|
||||
// End User License Agreement\n--------------------------
|
||||
// we want to try and find these multi-line license declarations and make exceptions for them
|
||||
if value := findLicenseClause(licenseFirstSentenceAfterHeadingPattern, content); value != "" {
|
||||
findings.Add(value)
|
||||
// multi-line heading detection (only at start of file)
|
||||
switch headingState {
|
||||
case expectHeading:
|
||||
if strings.TrimSpace(line) != "" {
|
||||
headingState = expectDashes
|
||||
} else {
|
||||
headingState = headingDone
|
||||
}
|
||||
case expectDashes:
|
||||
trimmed := strings.TrimSpace(line)
|
||||
if len(trimmed) > 0 && strings.Trim(trimmed, "-") == "" {
|
||||
headingState = skipBlanks
|
||||
} else {
|
||||
headingState = headingDone
|
||||
}
|
||||
case skipBlanks:
|
||||
if strings.TrimSpace(line) != "" {
|
||||
headingState = captureLicense
|
||||
licenseText.WriteString(line)
|
||||
if value := extractUpToFirstPeriod(licenseText.String()); value != "" {
|
||||
findings.Add(value)
|
||||
headingState = headingDone
|
||||
}
|
||||
}
|
||||
case captureLicense:
|
||||
licenseText.WriteString(" ")
|
||||
licenseText.WriteString(line)
|
||||
if value := extractUpToFirstPeriod(licenseText.String()); value != "" {
|
||||
findings.Add(value)
|
||||
headingState = headingDone
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
results := findings.List()
|
||||
@@ -55,6 +91,15 @@ func parseLicensesFromCopyright(reader io.Reader) []string {
|
||||
return results
|
||||
}
|
||||
|
||||
// extractUpToFirstPeriod returns the license text up to the first period,
|
||||
// processed through ensureIsSingleLicense, or "" if no period found yet.
|
||||
func extractUpToFirstPeriod(s string) string {
|
||||
if idx := strings.Index(s, "."); idx >= 0 {
|
||||
return ensureIsSingleLicense(s[:idx+1])
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func findLicenseClause(pattern *regexp.Regexp, line string) string {
|
||||
valueGroup := "license"
|
||||
matchesByGroup := internal.MatchNamedCaptureGroups(pattern, line)
|
||||
|
||||
@@ -2,7 +2,6 @@ package debian
|
||||
|
||||
import (
|
||||
"archive/tar"
|
||||
"bytes"
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
@@ -107,41 +106,56 @@ func processDataTar(dcReader io.ReadCloser) ([]string, error) {
|
||||
func processControlTar(dcReader io.ReadCloser) (*pkg.DpkgArchiveEntry, error) {
|
||||
defer internal.CloseAndLogError(dcReader, "")
|
||||
|
||||
// Extract control, md5sums, and conffiles files from control.tar
|
||||
tarReader := tar.NewReader(dcReader)
|
||||
controlFileContent, md5Content, confContent, err := readControlFiles(tarReader)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to read control files: %w", err)
|
||||
|
||||
var metadata *pkg.DpkgArchiveEntry
|
||||
var files []pkg.DpkgFileRecord
|
||||
var confFileRecords []pkg.DpkgFileRecord
|
||||
|
||||
for {
|
||||
header, err := tarReader.Next()
|
||||
if err == io.EOF {
|
||||
break
|
||||
}
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to read control tar: %w", err)
|
||||
}
|
||||
|
||||
switch filepath.Base(header.Name) {
|
||||
case "control":
|
||||
// parseDpkgStatus already streams via bufio.Reader
|
||||
entries, err := parseDpkgStatus(tarReader)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to parse control file: %w", err)
|
||||
}
|
||||
if len(entries) == 0 {
|
||||
return nil, fmt.Errorf("no package entries found in control file")
|
||||
}
|
||||
entry := pkg.DpkgArchiveEntry(entries[0])
|
||||
metadata = &entry
|
||||
case "md5sums":
|
||||
// parseDpkgMD5Info already streams via bufio.Scanner
|
||||
files = parseDpkgMD5Info(tarReader)
|
||||
case "conffiles":
|
||||
// parseDpkgConffileInfo already streams via bufio.Scanner
|
||||
confFileRecords = parseDpkgConffileInfo(tarReader)
|
||||
}
|
||||
}
|
||||
|
||||
if controlFileContent == nil {
|
||||
if metadata == nil {
|
||||
return nil, fmt.Errorf("control file not found in archive")
|
||||
}
|
||||
|
||||
metadata, err := newDpkgArchiveMetadata(controlFileContent, md5Content, confContent)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to create package metadata: %w", err)
|
||||
}
|
||||
|
||||
return &metadata, nil
|
||||
}
|
||||
|
||||
func newDpkgArchiveMetadata(controlFile, md5sums, confFiles []byte) (pkg.DpkgArchiveEntry, error) {
|
||||
// parse the control file to get package metadata
|
||||
metadata, err := parseControlFile(string(controlFile))
|
||||
if err != nil {
|
||||
return pkg.DpkgArchiveEntry{}, fmt.Errorf("failed to parse control file: %w", err)
|
||||
}
|
||||
|
||||
// parse MD5 sums to get file records
|
||||
var files []pkg.DpkgFileRecord
|
||||
if len(md5sums) > 0 {
|
||||
files = parseDpkgMD5Info(bytes.NewReader(md5sums))
|
||||
}
|
||||
|
||||
// mark config files
|
||||
if len(confFiles) > 0 {
|
||||
markConfigFiles(confFiles, files)
|
||||
if len(confFileRecords) > 0 && len(files) > 0 {
|
||||
configPaths := make(map[string]struct{}, len(confFileRecords))
|
||||
for _, cf := range confFileRecords {
|
||||
configPaths[cf.Path] = struct{}{}
|
||||
}
|
||||
for i, f := range files {
|
||||
if _, isConfig := configPaths[f.Path]; isConfig {
|
||||
files[i].IsConfigFile = true
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
metadata.Files = files
|
||||
@@ -166,73 +180,3 @@ func decompressionStream(ctx context.Context, r io.Reader, filePath string) (io.
|
||||
|
||||
return rc, nil
|
||||
}
|
||||
|
||||
// readControlFiles extracts important files from the control.tar archive
|
||||
func readControlFiles(tarReader *tar.Reader) (controlFile, md5sums, conffiles []byte, err error) {
|
||||
for {
|
||||
header, err := tarReader.Next()
|
||||
if err == io.EOF {
|
||||
break
|
||||
}
|
||||
if err != nil {
|
||||
return nil, nil, nil, err
|
||||
}
|
||||
|
||||
switch filepath.Base(header.Name) {
|
||||
case "control":
|
||||
controlFile, err = io.ReadAll(tarReader)
|
||||
if err != nil {
|
||||
return nil, nil, nil, err
|
||||
}
|
||||
case "md5sums":
|
||||
md5sums, err = io.ReadAll(tarReader)
|
||||
if err != nil {
|
||||
return nil, nil, nil, err
|
||||
}
|
||||
case "conffiles":
|
||||
conffiles, err = io.ReadAll(tarReader)
|
||||
if err != nil {
|
||||
return nil, nil, nil, err
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return controlFile, md5sums, conffiles, nil
|
||||
}
|
||||
|
||||
// parseControlFile parses the content of a debian control file into package metadata
|
||||
func parseControlFile(controlFileContent string) (pkg.DpkgArchiveEntry, error) {
|
||||
// Reuse the existing dpkg status file parsing logic
|
||||
reader := strings.NewReader(controlFileContent)
|
||||
|
||||
entries, err := parseDpkgStatus(reader)
|
||||
if err != nil {
|
||||
return pkg.DpkgArchiveEntry{}, fmt.Errorf("failed to parse control file: %w", err)
|
||||
}
|
||||
|
||||
if len(entries) == 0 {
|
||||
return pkg.DpkgArchiveEntry{}, fmt.Errorf("no package entries found in control file")
|
||||
}
|
||||
|
||||
// We expect only one entry from a .deb control file
|
||||
return pkg.DpkgArchiveEntry(entries[0]), nil
|
||||
}
|
||||
|
||||
// markConfigFiles marks files that are listed in conffiles as configuration files
|
||||
func markConfigFiles(conffilesContent []byte, files []pkg.DpkgFileRecord) {
|
||||
// Parse the conffiles content into DpkgFileRecord entries
|
||||
confFiles := parseDpkgConffileInfo(bytes.NewReader(conffilesContent))
|
||||
|
||||
// Create a map for quick lookup of config files by path
|
||||
configPathMap := make(map[string]struct{})
|
||||
for _, confFile := range confFiles {
|
||||
configPathMap[confFile.Path] = struct{}{}
|
||||
}
|
||||
|
||||
// Mark files as config files if they're in the conffiles list
|
||||
for i := range files {
|
||||
if _, exists := configPathMap[files[i].Path]; exists {
|
||||
files[i].IsConfigFile = true
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -3,31 +3,58 @@ package debian
|
||||
import (
|
||||
"archive/tar"
|
||||
"bytes"
|
||||
"io"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"github.com/anchore/syft/syft/file"
|
||||
"github.com/anchore/syft/syft/pkg"
|
||||
)
|
||||
|
||||
func TestReadControlFiles(t *testing.T) {
|
||||
func TestProcessControlTar(t *testing.T) {
|
||||
tarBytes := createTestTarWithControlFiles(t)
|
||||
|
||||
tarReader := bytes.NewReader(tarBytes)
|
||||
reader := tar.NewReader(tarReader)
|
||||
|
||||
controlFile, md5sums, conffiles, err := readControlFiles(reader)
|
||||
metadata, err := processControlTar(io.NopCloser(bytes.NewReader(tarBytes)))
|
||||
|
||||
require.NoError(t, err)
|
||||
assert.NotNil(t, controlFile, "expected control file to be found")
|
||||
assert.NotNil(t, md5sums, "expected md5sums file to be found")
|
||||
assert.NotNil(t, conffiles, "expected conffiles file to be found")
|
||||
require.NotNil(t, metadata)
|
||||
|
||||
assert.Contains(t, string(controlFile), "Package: test-package")
|
||||
assert.Contains(t, string(md5sums), "d41d8cd98f00b204e9800998ecf8427e")
|
||||
assert.Contains(t, string(conffiles), "/etc/test")
|
||||
assert.Equal(t, "test-package", metadata.Package)
|
||||
assert.Equal(t, "1.0.0", metadata.Version)
|
||||
|
||||
// md5sums should have been parsed into file records
|
||||
require.Len(t, metadata.Files, 1)
|
||||
assert.Equal(t, "/usr/bin/test-command", metadata.Files[0].Path)
|
||||
assert.Equal(t, "d41d8cd98f00b204e9800998ecf8427e", metadata.Files[0].Digest.Value)
|
||||
|
||||
// conffiles should have marked config files
|
||||
assert.True(t, metadata.Files[0].IsConfigFile, "file listed in conffiles should be marked as config")
|
||||
}
|
||||
|
||||
func TestProcessControlTar_ConfigFileMarking(t *testing.T) {
|
||||
// Create a tar where conffiles lists paths that overlap with md5sums entries
|
||||
var buf bytes.Buffer
|
||||
tw := tar.NewWriter(&buf)
|
||||
|
||||
controlContent := "Package: test-package\nVersion: 1.0.0\nArchitecture: all\n"
|
||||
writeTarEntry(t, tw, "control", controlContent)
|
||||
|
||||
md5Content := "d41d8cd98f00b204e9800998ecf8427e usr/bin/test-command\n" +
|
||||
"d41d8cd98f00b204e9800998ecf8427e etc/test/config.conf\n" +
|
||||
"d41d8cd98f00b204e9800998ecf8427e usr/bin/other-command\n"
|
||||
writeTarEntry(t, tw, "md5sums", md5Content)
|
||||
|
||||
conffilesContent := "/usr/bin/test-command\n/etc/test/config.conf\n"
|
||||
writeTarEntry(t, tw, "conffiles", conffilesContent)
|
||||
|
||||
require.NoError(t, tw.Close())
|
||||
|
||||
metadata, err := processControlTar(io.NopCloser(bytes.NewReader(buf.Bytes())))
|
||||
require.NoError(t, err)
|
||||
require.Len(t, metadata.Files, 3)
|
||||
|
||||
assert.True(t, metadata.Files[0].IsConfigFile, "first file should be marked as config file")
|
||||
assert.True(t, metadata.Files[1].IsConfigFile, "second file should be marked as config file")
|
||||
assert.False(t, metadata.Files[2].IsConfigFile, "third file should not be marked as config file")
|
||||
}
|
||||
|
||||
// createTestTarWithControlFiles creates a simple in-memory tar file with test control files
|
||||
@@ -35,108 +62,26 @@ func createTestTarWithControlFiles(t *testing.T) []byte {
|
||||
var buf bytes.Buffer
|
||||
tw := tar.NewWriter(&buf)
|
||||
|
||||
// Add control file
|
||||
controlContent := `Package: test-package
|
||||
Version: 1.0.0
|
||||
Architecture: all
|
||||
Maintainer: Test <test@example.com>
|
||||
Description: Test package
|
||||
`
|
||||
err := tw.WriteHeader(&tar.Header{
|
||||
Name: "control",
|
||||
Mode: 0644,
|
||||
Size: int64(len(controlContent)),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
_, err = tw.Write([]byte(controlContent))
|
||||
require.NoError(t, err)
|
||||
controlContent := "Package: test-package\nVersion: 1.0.0\nArchitecture: all\nMaintainer: Test <test@example.com>\nDescription: Test package\n"
|
||||
writeTarEntry(t, tw, "control", controlContent)
|
||||
|
||||
// Add md5sums file
|
||||
md5Content := "d41d8cd98f00b204e9800998ecf8427e usr/bin/test-command\n"
|
||||
err = tw.WriteHeader(&tar.Header{
|
||||
Name: "md5sums",
|
||||
Mode: 0644,
|
||||
Size: int64(len(md5Content)),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
_, err = tw.Write([]byte(md5Content))
|
||||
require.NoError(t, err)
|
||||
writeTarEntry(t, tw, "md5sums", md5Content)
|
||||
|
||||
// Add conffiles file
|
||||
conffilesContent := "/etc/test/config.conf\n"
|
||||
err = tw.WriteHeader(&tar.Header{
|
||||
Name: "conffiles",
|
||||
Mode: 0644,
|
||||
Size: int64(len(conffilesContent)),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
_, err = tw.Write([]byte(conffilesContent))
|
||||
require.NoError(t, err)
|
||||
|
||||
// Close the tar writer
|
||||
err = tw.Close()
|
||||
require.NoError(t, err)
|
||||
conffilesContent := "/usr/bin/test-command\n"
|
||||
writeTarEntry(t, tw, "conffiles", conffilesContent)
|
||||
|
||||
require.NoError(t, tw.Close())
|
||||
return buf.Bytes()
|
||||
}
|
||||
|
||||
func TestMarkConfigFiles(t *testing.T) {
|
||||
// Create test data
|
||||
conffilesContent := []byte("/usr/bin/test-command\n/etc/test/config.conf\n")
|
||||
|
||||
files := []pkg.DpkgFileRecord{
|
||||
{
|
||||
Path: "/usr/bin/test-command",
|
||||
Digest: &file.Digest{
|
||||
Algorithm: "md5",
|
||||
Value: "d41d8cd98f00b204e9800998ecf8427e",
|
||||
},
|
||||
},
|
||||
{
|
||||
Path: "/etc/test/config.conf",
|
||||
Digest: &file.Digest{
|
||||
Algorithm: "md5",
|
||||
Value: "d41d8cd98f00b204e9800998ecf8427e",
|
||||
},
|
||||
},
|
||||
{
|
||||
Path: "/usr/bin/other-command",
|
||||
Digest: &file.Digest{
|
||||
Algorithm: "md5",
|
||||
Value: "d41d8cd98f00b204e9800998ecf8427e",
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
markConfigFiles(conffilesContent, files)
|
||||
|
||||
assert.True(t, files[0].IsConfigFile, "first file should be marked as config file")
|
||||
assert.True(t, files[1].IsConfigFile, "second file should be marked as config file")
|
||||
assert.False(t, files[2].IsConfigFile, "third file should not be marked as config file")
|
||||
}
|
||||
|
||||
func TestParseControlFile(t *testing.T) {
|
||||
controlContent := `Package: test-package
|
||||
Version: 1.2.3-4
|
||||
Architecture: amd64
|
||||
Maintainer: Test User <test@example.com>
|
||||
Installed-Size: 1234
|
||||
Depends: libc6, libtest
|
||||
Description: This is a test package
|
||||
More description text
|
||||
And even more details
|
||||
`
|
||||
|
||||
metadata, err := parseControlFile(controlContent)
|
||||
|
||||
func writeTarEntry(t *testing.T, tw *tar.Writer, name, content string) {
|
||||
t.Helper()
|
||||
require.NoError(t, tw.WriteHeader(&tar.Header{
|
||||
Name: name,
|
||||
Mode: 0644,
|
||||
Size: int64(len(content)),
|
||||
}))
|
||||
_, err := tw.Write([]byte(content))
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "test-package", metadata.Package)
|
||||
assert.Equal(t, "1.2.3-4", metadata.Version)
|
||||
assert.Equal(t, "amd64", metadata.Architecture)
|
||||
assert.Equal(t, "Test User <test@example.com>", metadata.Maintainer)
|
||||
assert.Equal(t, 1234, metadata.InstalledSize)
|
||||
assert.Contains(t, metadata.Description, "This is a test package")
|
||||
assert.Len(t, metadata.Depends, 2)
|
||||
assert.Contains(t, metadata.Depends, "libc6")
|
||||
assert.Contains(t, metadata.Depends, "libtest")
|
||||
}
|
||||
|
||||
@@ -114,6 +114,26 @@ func (c *Cataloger) WithParserByPath(parser Parser, paths ...string) *Cataloger
|
||||
return c
|
||||
}
|
||||
|
||||
func (c *Cataloger) WithParserByMediaType(parser Parser, types ...string) *Cataloger {
|
||||
c.requesters = append(c.requesters,
|
||||
func(resolver file.Resolver, _ Environment) []request {
|
||||
var requests []request
|
||||
log.WithFields("mediatypes", types).Trace("searching content matching mediatypes")
|
||||
ociResolver, ok := resolver.(file.OCIMediaTypeResolver)
|
||||
if !ok {
|
||||
return nil
|
||||
}
|
||||
matches, err := ociResolver.FilesByMediaType(types...)
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
requests = append(requests, makeRequests(parser, matches)...)
|
||||
return requests
|
||||
},
|
||||
)
|
||||
return c
|
||||
}
|
||||
|
||||
func (c *Cataloger) WithProcessors(processors ...Processor) *Cataloger {
|
||||
for _, p := range processors {
|
||||
c.processors = append(c.processors, processorWrapper{Processor: p})
|
||||
|
||||
@@ -138,6 +138,10 @@ func (m spyReturningFileResolver) FilesByMIMEType(types ...string) ([]file.Locat
|
||||
return m.m.FilesByMIMEType(types...)
|
||||
}
|
||||
|
||||
func (m spyReturningFileResolver) FilesByMediaType(types ...string) ([]file.Location, error) {
|
||||
return m.m.FilesByMediaType(types...)
|
||||
}
|
||||
|
||||
func (m spyReturningFileResolver) RelativeFileByPath(f file.Location, path string) *file.Location {
|
||||
return m.m.RelativeFileByPath(f, path)
|
||||
}
|
||||
@@ -189,6 +193,55 @@ func TestClosesFileOnParserPanic(t *testing.T) {
|
||||
require.True(t, spy.closed)
|
||||
}
|
||||
|
||||
func Test_CatalogerWithParserByMediaType(t *testing.T) {
|
||||
allParsedPaths := make(map[string]bool)
|
||||
parser := func(_ context.Context, resolver file.Resolver, env *Environment, reader file.LocationReadCloser) ([]pkg.Package, []artifact.Relationship, error) {
|
||||
allParsedPaths[reader.Path()] = true
|
||||
contents, err := io.ReadAll(reader)
|
||||
require.NoError(t, err)
|
||||
|
||||
if len(contents) == 0 {
|
||||
return nil, nil, nil
|
||||
}
|
||||
|
||||
p := pkg.Package{
|
||||
Name: string(contents),
|
||||
Locations: file.NewLocationSet(reader.Location),
|
||||
}
|
||||
|
||||
return []pkg.Package{p}, nil, nil
|
||||
}
|
||||
|
||||
upstream := "media-type-cataloger"
|
||||
|
||||
// Create locations with test fixtures that exist on disk
|
||||
loc1 := file.NewLocation("test-fixtures/a-path.txt")
|
||||
loc2 := file.NewLocation("test-fixtures/another-path.txt")
|
||||
|
||||
// Create a mock resolver that maps media types to locations
|
||||
resolver := file.NewMockResolverForMediaTypes(map[string][]file.Location{
|
||||
"application/vnd.test.model": {loc1, loc2},
|
||||
})
|
||||
|
||||
cataloger := NewCataloger(upstream).
|
||||
WithParserByMediaType(parser, "application/vnd.test.model")
|
||||
|
||||
actualPkgs, _, err := cataloger.Catalog(context.Background(), resolver)
|
||||
assert.NoError(t, err)
|
||||
|
||||
// Verify both files were parsed
|
||||
assert.True(t, allParsedPaths["test-fixtures/a-path.txt"], "expected a-path.txt to be parsed")
|
||||
assert.True(t, allParsedPaths["test-fixtures/another-path.txt"], "expected another-path.txt to be parsed")
|
||||
|
||||
// Verify packages were created
|
||||
assert.Len(t, actualPkgs, 2)
|
||||
|
||||
// Verify FoundBy is set correctly
|
||||
for _, p := range actualPkgs {
|
||||
assert.Equal(t, upstream, p.FoundBy)
|
||||
}
|
||||
}
|
||||
|
||||
func Test_genericCatalogerReturnsErrors(t *testing.T) {
|
||||
genericErrorReturning := NewCataloger("error returning").WithParserByGlobs(func(ctx context.Context, resolver file.Resolver, environment *Environment, locationReader file.LocationReadCloser) ([]pkg.Package, []artifact.Relationship, error) {
|
||||
return []pkg.Package{
|
||||
|
||||
@@ -97,6 +97,15 @@ func Test_candidateVendorsForAPK(t *testing.T) {
|
||||
},
|
||||
expected: []string{"rake", "ruby-lang"},
|
||||
},
|
||||
{
|
||||
name: "libavif",
|
||||
pkg: pkg.Package{
|
||||
Metadata: pkg.ApkDBEntry{
|
||||
Package: "libavif",
|
||||
},
|
||||
},
|
||||
expected: []string{"aomedia", "libavif"},
|
||||
},
|
||||
}
|
||||
for _, test := range tests {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
|
||||
@@ -410,6 +410,11 @@ var defaultCandidateAdditions = buildCandidateLookup(
|
||||
candidateKey{PkgName: "git"},
|
||||
candidateAddition{AdditionalVendors: []string{"git-scm"}},
|
||||
},
|
||||
{
|
||||
pkg.ApkPkg,
|
||||
candidateKey{PkgName: "libavif"},
|
||||
candidateAddition{AdditionalVendors: []string{"aomedia"}},
|
||||
},
|
||||
{
|
||||
pkg.ApkPkg,
|
||||
candidateKey{PkgName: "bind"},
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
+20
-3
@@ -10,6 +10,23 @@ import (
|
||||
|
||||
const cacheDir = ".cpe-cache"
|
||||
|
||||
// parseNVDTimestamp parses timestamps from NVD API which may or may not include timezone
|
||||
func parseNVDTimestamp(ts string) (time.Time, error) {
|
||||
// try RFC3339 first (with timezone)
|
||||
if t, err := time.Parse(time.RFC3339, ts); err == nil {
|
||||
return t, nil
|
||||
}
|
||||
// try without timezone (NVD sometimes returns this format)
|
||||
if t, err := time.Parse("2006-01-02T15:04:05.000", ts); err == nil {
|
||||
return t, nil
|
||||
}
|
||||
// try without milliseconds
|
||||
if t, err := time.Parse("2006-01-02T15:04:05", ts); err == nil {
|
||||
return t, nil
|
||||
}
|
||||
return time.Time{}, fmt.Errorf("unable to parse timestamp: %s", ts)
|
||||
}
|
||||
|
||||
// IncrementMetadata tracks a single fetch increment for a monthly batch
|
||||
type IncrementMetadata struct {
|
||||
FetchedAt time.Time `json:"fetchedAt"`
|
||||
@@ -215,7 +232,7 @@ func groupProductsByMonth(products []NVDProduct) (map[string][]NVDProduct, error
|
||||
productsByMonth := make(map[string][]NVDProduct)
|
||||
|
||||
for _, product := range products {
|
||||
lastMod, err := time.Parse(time.RFC3339, product.CPE.LastModified)
|
||||
lastMod, err := parseNVDTimestamp(product.CPE.LastModified)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to parse lastModified for %s: %w", product.CPE.CPENameID, err)
|
||||
}
|
||||
@@ -342,8 +359,8 @@ func (m *CacheManager) LoadAllProducts() ([]NVDProduct, error) {
|
||||
}
|
||||
|
||||
// compare lastModified timestamps to keep the newer one
|
||||
newMod, _ := time.Parse(time.RFC3339, p.CPE.LastModified)
|
||||
existingMod, _ := time.Parse(time.RFC3339, existing.CPE.LastModified)
|
||||
newMod, _ := parseNVDTimestamp(p.CPE.LastModified)
|
||||
existingMod, _ := parseNVDTimestamp(existing.CPE.LastModified)
|
||||
|
||||
if newMod.After(existingMod) {
|
||||
productMap[p.CPE.CPENameID] = p
|
||||
|
||||
+56
@@ -317,3 +317,59 @@ func TestCacheManager_SaveProducts(t *testing.T) {
|
||||
assert.Equal(t, 1, metadata.MonthlyBatches["2024-12"].TotalProducts)
|
||||
})
|
||||
}
|
||||
|
||||
func TestParseNVDTimestamp(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
input string
|
||||
expectError bool
|
||||
expectedStr string // expected time formatted as 2006-01-02T15:04:05
|
||||
}{
|
||||
{
|
||||
name: "RFC3339 with Z timezone",
|
||||
input: "2024-11-15T10:00:00.000Z",
|
||||
expectError: false,
|
||||
expectedStr: "2024-11-15T10:00:00",
|
||||
},
|
||||
{
|
||||
name: "RFC3339 with offset timezone",
|
||||
input: "2024-11-15T10:00:00+00:00",
|
||||
expectError: false,
|
||||
expectedStr: "2024-11-15T10:00:00",
|
||||
},
|
||||
{
|
||||
name: "without timezone (NVD format)",
|
||||
input: "2026-01-27T15:53:34.823",
|
||||
expectError: false,
|
||||
expectedStr: "2026-01-27T15:53:34",
|
||||
},
|
||||
{
|
||||
name: "without timezone or milliseconds",
|
||||
input: "2024-11-15T10:00:00",
|
||||
expectError: false,
|
||||
expectedStr: "2024-11-15T10:00:00",
|
||||
},
|
||||
{
|
||||
name: "invalid format",
|
||||
input: "not-a-timestamp",
|
||||
expectError: true,
|
||||
},
|
||||
{
|
||||
name: "empty string",
|
||||
input: "",
|
||||
expectError: true,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
result, err := parseNVDTimestamp(tt.input)
|
||||
if tt.expectError {
|
||||
require.Error(t, err)
|
||||
} else {
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, tt.expectedStr, result.Format("2006-01-02T15:04:05"))
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
@@ -53,33 +53,26 @@ func updateCache(ctx context.Context, cacheManager *CacheManager, forceFullRefre
|
||||
|
||||
lastModStartDate, isFullRefresh := determineUpdateMode(metadata, forceFullRefresh)
|
||||
|
||||
// use resume index if available
|
||||
resumeFromIndex := 0
|
||||
if !isFullRefresh && metadata.LastStartIndex > 0 {
|
||||
resumeFromIndex = metadata.LastStartIndex
|
||||
fmt.Printf("Resuming from index %d...\n", resumeFromIndex)
|
||||
}
|
||||
|
||||
allProducts, increment, err := fetchProducts(ctx, lastModStartDate, resumeFromIndex)
|
||||
products, increment, err := fetchProducts(ctx, lastModStartDate)
|
||||
if err != nil {
|
||||
// if we have partial products, save them before returning error
|
||||
if len(allProducts) > 0 {
|
||||
fmt.Printf("\nError occurred but saving %d products fetched so far...\n", len(allProducts))
|
||||
if saveErr := saveAndReportResults(cacheManager, allProducts, isFullRefresh, metadata, increment); saveErr != nil {
|
||||
if len(products) > 0 {
|
||||
fmt.Printf("\nError occurred but saving %d products fetched so far...\n", len(products))
|
||||
if saveErr := saveAndReportResults(cacheManager, products, isFullRefresh, metadata, increment); saveErr != nil {
|
||||
fmt.Printf("WARNING: Failed to save partial progress: %v\n", saveErr)
|
||||
} else {
|
||||
fmt.Println("Partial progress saved successfully. Run again to resume from this point.")
|
||||
fmt.Println("Partial progress saved successfully.")
|
||||
}
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
if len(allProducts) == 0 {
|
||||
if len(products) == 0 {
|
||||
fmt.Println("No products fetched (already up to date)")
|
||||
return nil
|
||||
}
|
||||
|
||||
return saveAndReportResults(cacheManager, allProducts, isFullRefresh, metadata, increment)
|
||||
return saveAndReportResults(cacheManager, products, isFullRefresh, metadata, increment)
|
||||
}
|
||||
|
||||
// determineUpdateMode decides whether to do a full refresh or incremental update
|
||||
@@ -94,48 +87,25 @@ func determineUpdateMode(metadata *CacheMetadata, forceFullRefresh bool) (time.T
|
||||
}
|
||||
|
||||
// fetchProducts fetches products from the NVD API
|
||||
func fetchProducts(ctx context.Context, lastModStartDate time.Time, resumeFromIndex int) ([]NVDProduct, IncrementMetadata, error) {
|
||||
func fetchProducts(ctx context.Context, lastModStartDate time.Time) ([]NVDProduct, IncrementMetadata, error) {
|
||||
apiClient := NewNVDAPIClient()
|
||||
fmt.Println("Fetching CPE data from NVD Products API...")
|
||||
|
||||
var allProducts []NVDProduct
|
||||
var totalResults int
|
||||
var firstStartIndex, lastEndIndex int
|
||||
|
||||
onPageFetched := func(startIndex int, response NVDProductsResponse) error {
|
||||
if totalResults == 0 {
|
||||
totalResults = response.TotalResults
|
||||
firstStartIndex = startIndex
|
||||
}
|
||||
lastEndIndex = startIndex + response.ResultsPerPage
|
||||
allProducts = append(allProducts, response.Products...)
|
||||
fmt.Printf("Fetched %d/%d products...\n", len(allProducts), totalResults)
|
||||
return nil
|
||||
}
|
||||
|
||||
if err := apiClient.FetchProductsSince(ctx, lastModStartDate, resumeFromIndex, onPageFetched); err != nil {
|
||||
// return partial products with increment metadata so they can be saved
|
||||
increment := IncrementMetadata{
|
||||
FetchedAt: time.Now(),
|
||||
LastModStartDate: lastModStartDate,
|
||||
LastModEndDate: time.Now(),
|
||||
Products: len(allProducts),
|
||||
StartIndex: firstStartIndex,
|
||||
EndIndex: lastEndIndex,
|
||||
}
|
||||
return allProducts, increment, fmt.Errorf("failed to fetch products from NVD API: %w", err)
|
||||
}
|
||||
products, err := apiClient.FetchProductsSince(ctx, lastModStartDate)
|
||||
|
||||
increment := IncrementMetadata{
|
||||
FetchedAt: time.Now(),
|
||||
LastModStartDate: lastModStartDate,
|
||||
LastModEndDate: time.Now(),
|
||||
Products: len(allProducts),
|
||||
StartIndex: firstStartIndex,
|
||||
EndIndex: lastEndIndex,
|
||||
Products: len(products),
|
||||
}
|
||||
|
||||
return allProducts, increment, nil
|
||||
if err != nil {
|
||||
// return partial products so they can be saved
|
||||
return products, increment, fmt.Errorf("failed to fetch products from NVD API: %w", err)
|
||||
}
|
||||
|
||||
return products, increment, nil
|
||||
}
|
||||
|
||||
// saveAndReportResults saves products and metadata, then reports success
|
||||
|
||||
+96
-55
@@ -24,6 +24,9 @@ const (
|
||||
// retry configuration for rate limiting
|
||||
maxRetries = 5
|
||||
baseRetryDelay = 30 * time.Second // NVD uses 30-second rolling windows
|
||||
|
||||
// NVD API has a maximum date range of 120 days for queries with date filters
|
||||
maxDateRangeDays = 120
|
||||
)
|
||||
|
||||
// NVDAPIClient handles communication with the NVD Products API
|
||||
@@ -109,45 +112,98 @@ func NewNVDAPIClient() *NVDAPIClient {
|
||||
}
|
||||
}
|
||||
|
||||
// PageCallback is called after each page is successfully fetched
|
||||
// it receives the startIndex and the response for that page
|
||||
type PageCallback func(startIndex int, response NVDProductsResponse) error
|
||||
// FetchProductsSince fetches all products modified since the given date.
|
||||
// If lastModStartDate is zero, fetches all products (no date filter).
|
||||
// If lastModStartDate is set, fetches in 120-day chunks (NVD API limit) from that date to now.
|
||||
// Returns partial results on error so progress can be saved.
|
||||
func (c *NVDAPIClient) FetchProductsSince(ctx context.Context, lastModStartDate time.Time) ([]NVDProduct, error) {
|
||||
// if no date filter, fetch all products in a single pass
|
||||
if lastModStartDate.IsZero() {
|
||||
return c.fetchDateRange(ctx, time.Time{}, time.Time{})
|
||||
}
|
||||
|
||||
// FetchProductsSince fetches all products modified since the given date
|
||||
// if lastModStartDate is zero, fetches all products
|
||||
// calls onPageFetched callback after each successful page fetch for incremental saving
|
||||
// if resumeFromIndex > 0, starts fetching from that index
|
||||
func (c *NVDAPIClient) FetchProductsSince(ctx context.Context, lastModStartDate time.Time, resumeFromIndex int, onPageFetched PageCallback) error {
|
||||
startIndex := resumeFromIndex
|
||||
// fetch in 120-day chunks from lastModStartDate to now
|
||||
chunks := buildDateChunks(lastModStartDate, time.Now().UTC())
|
||||
if len(chunks) > 1 {
|
||||
fmt.Printf("Date range spans %d chunks of up to %d days each\n", len(chunks), maxDateRangeDays)
|
||||
}
|
||||
|
||||
var allProducts []NVDProduct
|
||||
for i, chunk := range chunks {
|
||||
if len(chunks) > 1 {
|
||||
fmt.Printf("Fetching chunk %d/%d: %s to %s\n", i+1, len(chunks),
|
||||
chunk.start.Format("2006-01-02"), chunk.end.Format("2006-01-02"))
|
||||
}
|
||||
|
||||
products, err := c.fetchDateRange(ctx, chunk.start, chunk.end)
|
||||
if err != nil {
|
||||
// return partial results so caller can save progress
|
||||
return allProducts, err
|
||||
}
|
||||
|
||||
allProducts = append(allProducts, products...)
|
||||
if len(chunks) > 1 {
|
||||
fmt.Printf("Chunk %d complete: %d products (total so far: %d)\n", i+1, len(products), len(allProducts))
|
||||
}
|
||||
}
|
||||
|
||||
fmt.Printf("Fetched %d products total\n", len(allProducts))
|
||||
return allProducts, nil
|
||||
}
|
||||
|
||||
// dateChunk represents a date range for fetching
|
||||
type dateChunk struct {
|
||||
start time.Time
|
||||
end time.Time
|
||||
}
|
||||
|
||||
// buildDateChunks splits a date range into chunks of maxDateRangeDays
|
||||
func buildDateChunks(start, end time.Time) []dateChunk {
|
||||
var chunks []dateChunk
|
||||
chunkStart := start
|
||||
|
||||
for chunkStart.Before(end) {
|
||||
chunkEnd := chunkStart.AddDate(0, 0, maxDateRangeDays)
|
||||
if chunkEnd.After(end) {
|
||||
chunkEnd = end
|
||||
}
|
||||
chunks = append(chunks, dateChunk{start: chunkStart, end: chunkEnd})
|
||||
chunkStart = chunkEnd
|
||||
}
|
||||
|
||||
return chunks
|
||||
}
|
||||
|
||||
// fetchDateRange fetches all products within a single date range (must be <= 120 days).
|
||||
// If start and end are both zero, fetches all products without date filtering.
|
||||
func (c *NVDAPIClient) fetchDateRange(ctx context.Context, start, end time.Time) ([]NVDProduct, error) {
|
||||
var products []NVDProduct
|
||||
startIndex := 0
|
||||
|
||||
for {
|
||||
resp, err := c.fetchPage(ctx, startIndex, lastModStartDate)
|
||||
resp, err := c.fetchPage(ctx, startIndex, start, end)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to fetch page at index %d: %w", startIndex, err)
|
||||
return products, fmt.Errorf("failed to fetch page at index %d: %w", startIndex, err)
|
||||
}
|
||||
|
||||
// call callback to save progress immediately
|
||||
if onPageFetched != nil {
|
||||
if err := onPageFetched(startIndex, resp); err != nil {
|
||||
return fmt.Errorf("callback failed at index %d: %w", startIndex, err)
|
||||
}
|
||||
}
|
||||
products = append(products, resp.Products...)
|
||||
fmt.Printf(" Fetched %d/%d products...\n", len(products), resp.TotalResults)
|
||||
|
||||
// check if we've fetched all results
|
||||
if startIndex+resp.ResultsPerPage >= resp.TotalResults {
|
||||
fmt.Printf("Fetched %d/%d products (complete)\n", resp.TotalResults, resp.TotalResults)
|
||||
break
|
||||
}
|
||||
|
||||
startIndex += resp.ResultsPerPage
|
||||
fmt.Printf("Fetched %d/%d products...\n", startIndex, resp.TotalResults)
|
||||
}
|
||||
|
||||
return nil
|
||||
return products, nil
|
||||
}
|
||||
|
||||
// fetchPage fetches a single page of results from the NVD API with retry logic for rate limiting
|
||||
func (c *NVDAPIClient) fetchPage(ctx context.Context, startIndex int, lastModStartDate time.Time) (NVDProductsResponse, error) {
|
||||
// if both start and end are zero, fetches without date filtering
|
||||
// if start and end are set, they must form a range <= 120 days (enforced by caller)
|
||||
func (c *NVDAPIClient) fetchPage(ctx context.Context, startIndex int, start, end time.Time) (NVDProductsResponse, error) {
|
||||
var lastErr error
|
||||
|
||||
for attempt := 0; attempt < maxRetries; attempt++ {
|
||||
@@ -160,10 +216,12 @@ func (c *NVDAPIClient) fetchPage(ctx context.Context, startIndex int, lastModSta
|
||||
url := fmt.Sprintf("%s?resultsPerPage=%d&startIndex=%d", nvdProductsAPIURL, resultsPerPage, startIndex)
|
||||
|
||||
// add date range if specified (incremental update)
|
||||
if !lastModStartDate.IsZero() {
|
||||
// NVD API requires RFC3339 format: 2024-01-01T00:00:00.000
|
||||
lastModStartStr := lastModStartDate.Format("2006-01-02T15:04:05.000")
|
||||
url += fmt.Sprintf("&lastModStartDate=%s", lastModStartStr)
|
||||
// NVD API requires both lastModStartDate and lastModEndDate when either is present
|
||||
if !start.IsZero() && !end.IsZero() {
|
||||
// NVD API requires this format: 2024-01-01T00:00:00.000
|
||||
startStr := start.Format("2006-01-02T15:04:05.000")
|
||||
endStr := end.Format("2006-01-02T15:04:05.000")
|
||||
url += fmt.Sprintf("&lastModStartDate=%s&lastModEndDate=%s", startStr, endStr)
|
||||
}
|
||||
|
||||
// create request
|
||||
@@ -191,14 +249,12 @@ func (c *NVDAPIClient) fetchPage(ctx context.Context, startIndex int, lastModSta
|
||||
continue // retry
|
||||
}
|
||||
|
||||
// handle HTTP status codes
|
||||
statusResponse, handled, err := c.handleHTTPStatus(httpResp, startIndex)
|
||||
if handled {
|
||||
// either error or special case (404 with empty results)
|
||||
return statusResponse, err
|
||||
// check for error status codes
|
||||
if err := checkHTTPStatus(httpResp); err != nil {
|
||||
return NVDProductsResponse{}, err
|
||||
}
|
||||
|
||||
// success - parse response
|
||||
// parse response
|
||||
var response NVDProductsResponse
|
||||
if err := json.NewDecoder(httpResp.Body).Decode(&response); err != nil {
|
||||
httpResp.Body.Close()
|
||||
@@ -235,31 +291,16 @@ func (c *NVDAPIClient) handleRateLimit(ctx context.Context, httpResp *http.Respo
|
||||
}
|
||||
}
|
||||
|
||||
// handleHTTPStatus handles non-429 HTTP status codes
|
||||
// returns (response, handled, error) where:
|
||||
// - handled=true means the status was processed (either success case like 404 or error)
|
||||
// - handled=false means continue to normal response parsing
|
||||
func (c *NVDAPIClient) handleHTTPStatus(httpResp *http.Response, startIndex int) (NVDProductsResponse, bool, error) {
|
||||
// handle 404 as "no results found" (common when querying recent dates with no updates)
|
||||
if httpResp.StatusCode == http.StatusNotFound {
|
||||
httpResp.Body.Close()
|
||||
return NVDProductsResponse{
|
||||
ResultsPerPage: 0,
|
||||
StartIndex: startIndex,
|
||||
TotalResults: 0,
|
||||
Products: []NVDProduct{},
|
||||
}, true, nil
|
||||
// checkHTTPStatus returns an error for non-200 status codes.
|
||||
// NVD API returns 200 with TotalResults=0 when there are no results,
|
||||
// so any non-200 status (including 404) indicates an actual error.
|
||||
func checkHTTPStatus(httpResp *http.Response) error {
|
||||
if httpResp.StatusCode == http.StatusOK {
|
||||
return nil
|
||||
}
|
||||
|
||||
// check for other non-200 status codes
|
||||
if httpResp.StatusCode != http.StatusOK {
|
||||
body, _ := io.ReadAll(httpResp.Body)
|
||||
httpResp.Body.Close()
|
||||
return NVDProductsResponse{}, true, fmt.Errorf("unexpected status code %d: %s", httpResp.StatusCode, string(body))
|
||||
}
|
||||
|
||||
// status OK - let caller parse response
|
||||
return NVDProductsResponse{}, false, nil
|
||||
body, _ := io.ReadAll(httpResp.Body)
|
||||
httpResp.Body.Close()
|
||||
return fmt.Errorf("NVD API error (status %d): %s", httpResp.StatusCode, string(body))
|
||||
}
|
||||
|
||||
// parseRetryAfter parses the Retry-After header from HTTP 429 responses
|
||||
|
||||
+233
@@ -0,0 +1,233 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"io"
|
||||
"net/http"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestBuildDateChunks(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
start time.Time
|
||||
end time.Time
|
||||
expectedChunks int
|
||||
validateChunks func(t *testing.T, chunks []dateChunk)
|
||||
}{
|
||||
{
|
||||
name: "single chunk when range is under 120 days",
|
||||
start: time.Date(2025, 1, 1, 0, 0, 0, 0, time.UTC),
|
||||
end: time.Date(2025, 2, 1, 0, 0, 0, 0, time.UTC), // 31 days
|
||||
expectedChunks: 1,
|
||||
validateChunks: func(t *testing.T, chunks []dateChunk) {
|
||||
assert.Equal(t, time.Date(2025, 1, 1, 0, 0, 0, 0, time.UTC), chunks[0].start)
|
||||
assert.Equal(t, time.Date(2025, 2, 1, 0, 0, 0, 0, time.UTC), chunks[0].end)
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "single chunk when range is exactly 120 days",
|
||||
start: time.Date(2025, 1, 1, 0, 0, 0, 0, time.UTC),
|
||||
end: time.Date(2025, 5, 1, 0, 0, 0, 0, time.UTC), // 120 days
|
||||
expectedChunks: 1,
|
||||
validateChunks: func(t *testing.T, chunks []dateChunk) {
|
||||
assert.Equal(t, time.Date(2025, 1, 1, 0, 0, 0, 0, time.UTC), chunks[0].start)
|
||||
assert.Equal(t, time.Date(2025, 5, 1, 0, 0, 0, 0, time.UTC), chunks[0].end)
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "two chunks when range is 121 days",
|
||||
start: time.Date(2025, 1, 1, 0, 0, 0, 0, time.UTC),
|
||||
end: time.Date(2025, 5, 2, 0, 0, 0, 0, time.UTC), // 121 days
|
||||
expectedChunks: 2,
|
||||
validateChunks: func(t *testing.T, chunks []dateChunk) {
|
||||
// first chunk: Jan 1 to May 1 (120 days)
|
||||
assert.Equal(t, time.Date(2025, 1, 1, 0, 0, 0, 0, time.UTC), chunks[0].start)
|
||||
assert.Equal(t, time.Date(2025, 5, 1, 0, 0, 0, 0, time.UTC), chunks[0].end)
|
||||
// second chunk: May 1 to May 2 (1 day)
|
||||
assert.Equal(t, time.Date(2025, 5, 1, 0, 0, 0, 0, time.UTC), chunks[1].start)
|
||||
assert.Equal(t, time.Date(2025, 5, 2, 0, 0, 0, 0, time.UTC), chunks[1].end)
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "multiple chunks for a full year",
|
||||
start: time.Date(2024, 1, 1, 0, 0, 0, 0, time.UTC),
|
||||
end: time.Date(2025, 1, 1, 0, 0, 0, 0, time.UTC), // 366 days (leap year)
|
||||
expectedChunks: 4,
|
||||
validateChunks: func(t *testing.T, chunks []dateChunk) {
|
||||
// verify chunks are contiguous (each chunk starts where previous ended)
|
||||
for i := 1; i < len(chunks); i++ {
|
||||
assert.Equal(t, chunks[i-1].end, chunks[i].start, "chunks should be contiguous")
|
||||
}
|
||||
// verify first and last
|
||||
assert.Equal(t, time.Date(2024, 1, 1, 0, 0, 0, 0, time.UTC), chunks[0].start)
|
||||
assert.Equal(t, time.Date(2025, 1, 1, 0, 0, 0, 0, time.UTC), chunks[len(chunks)-1].end)
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "empty result when start equals end",
|
||||
start: time.Date(2025, 1, 1, 0, 0, 0, 0, time.UTC),
|
||||
end: time.Date(2025, 1, 1, 0, 0, 0, 0, time.UTC),
|
||||
expectedChunks: 0,
|
||||
},
|
||||
{
|
||||
name: "empty result when start is after end",
|
||||
start: time.Date(2025, 2, 1, 0, 0, 0, 0, time.UTC),
|
||||
end: time.Date(2025, 1, 1, 0, 0, 0, 0, time.UTC),
|
||||
expectedChunks: 0,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
chunks := buildDateChunks(tt.start, tt.end)
|
||||
assert.Len(t, chunks, tt.expectedChunks)
|
||||
if tt.validateChunks != nil && len(chunks) > 0 {
|
||||
tt.validateChunks(t, chunks)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildDateChunks_ChunkSizeNeverExceeds120Days(t *testing.T) {
|
||||
// test with various date ranges to ensure no chunk exceeds 120 days
|
||||
testCases := []struct {
|
||||
start time.Time
|
||||
end time.Time
|
||||
}{
|
||||
{time.Date(2024, 1, 1, 0, 0, 0, 0, time.UTC), time.Date(2025, 1, 1, 0, 0, 0, 0, time.UTC)},
|
||||
{time.Date(2023, 6, 15, 0, 0, 0, 0, time.UTC), time.Date(2025, 2, 1, 0, 0, 0, 0, time.UTC)},
|
||||
{time.Date(2024, 1, 1, 0, 0, 0, 0, time.UTC), time.Date(2024, 12, 31, 0, 0, 0, 0, time.UTC)},
|
||||
}
|
||||
|
||||
for _, tc := range testCases {
|
||||
chunks := buildDateChunks(tc.start, tc.end)
|
||||
for i, chunk := range chunks {
|
||||
days := chunk.end.Sub(chunk.start).Hours() / 24
|
||||
assert.LessOrEqual(t, days, float64(maxDateRangeDays),
|
||||
"chunk %d exceeds max days: start=%s, end=%s, days=%.0f",
|
||||
i, chunk.start.Format("2006-01-02"), chunk.end.Format("2006-01-02"), days)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// mockResponseBody creates an io.ReadCloser from a string for testing
|
||||
type mockReadCloser struct {
|
||||
io.Reader
|
||||
closed bool
|
||||
}
|
||||
|
||||
func (m *mockReadCloser) Close() error {
|
||||
m.closed = true
|
||||
return nil
|
||||
}
|
||||
|
||||
func newMockResponse(statusCode int, body string) *http.Response {
|
||||
return &http.Response{
|
||||
StatusCode: statusCode,
|
||||
Body: &mockReadCloser{Reader: strings.NewReader(body)},
|
||||
}
|
||||
}
|
||||
|
||||
func TestCheckHTTPStatus(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
statusCode int
|
||||
body string
|
||||
expectError bool
|
||||
errorContains string
|
||||
}{
|
||||
{
|
||||
name: "200 OK returns nil",
|
||||
statusCode: http.StatusOK,
|
||||
body: `{"totalResults": 0}`,
|
||||
expectError: false,
|
||||
},
|
||||
{
|
||||
name: "404 returns error",
|
||||
statusCode: http.StatusNotFound,
|
||||
body: "",
|
||||
expectError: true,
|
||||
errorContains: "status 404",
|
||||
},
|
||||
{
|
||||
name: "400 Bad Request returns error with body",
|
||||
statusCode: http.StatusBadRequest,
|
||||
body: "Both lastModStartDate and lastModEndDate are required",
|
||||
expectError: true,
|
||||
errorContains: "lastModStartDate and lastModEndDate are required",
|
||||
},
|
||||
{
|
||||
name: "500 Internal Server Error returns error",
|
||||
statusCode: http.StatusInternalServerError,
|
||||
body: "Internal server error",
|
||||
expectError: true,
|
||||
errorContains: "status 500",
|
||||
},
|
||||
{
|
||||
name: "503 Service Unavailable returns error",
|
||||
statusCode: http.StatusServiceUnavailable,
|
||||
body: "Service temporarily unavailable",
|
||||
expectError: true,
|
||||
errorContains: "status 503",
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
resp := newMockResponse(tt.statusCode, tt.body)
|
||||
err := checkHTTPStatus(resp)
|
||||
|
||||
if tt.expectError {
|
||||
require.Error(t, err)
|
||||
assert.Contains(t, err.Error(), tt.errorContains)
|
||||
// verify body was closed on error
|
||||
assert.True(t, resp.Body.(*mockReadCloser).closed, "response body should be closed on error")
|
||||
} else {
|
||||
require.NoError(t, err)
|
||||
// verify body was NOT closed on success (caller needs to read it)
|
||||
assert.False(t, resp.Body.(*mockReadCloser).closed, "response body should not be closed on success")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseRetryAfter(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
header string
|
||||
expected time.Duration
|
||||
}{
|
||||
{
|
||||
name: "empty header returns 0",
|
||||
header: "",
|
||||
expected: 0,
|
||||
},
|
||||
{
|
||||
name: "numeric seconds",
|
||||
header: "30",
|
||||
expected: 30 * time.Second,
|
||||
},
|
||||
{
|
||||
name: "numeric seconds - larger value",
|
||||
header: "120",
|
||||
expected: 120 * time.Second,
|
||||
},
|
||||
{
|
||||
name: "invalid value returns 0",
|
||||
header: "not-a-number",
|
||||
expected: 0,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
result := parseRetryAfter(tt.header)
|
||||
assert.Equal(t, tt.expected, result)
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -208,6 +208,11 @@ func (r *ObservingResolver) FilesByMIMEType(types ...string) ([]file.Location, e
|
||||
return locs, err
|
||||
}
|
||||
|
||||
// FilesByMediaType returns files matching the given media types.
|
||||
func (r *ObservingResolver) FilesByMediaType(_ ...string) ([]file.Location, error) {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
// RelativeFileByPath returns a file at a path relative to the given location.
|
||||
func (r *ObservingResolver) RelativeFileByPath(location file.Location, path string) *file.Location {
|
||||
const methodName = "RelativeFileByPath"
|
||||
|
||||
@@ -23,20 +23,21 @@ var _ generic.Parser = parsePackageJSON
|
||||
|
||||
// packageJSON represents a JavaScript package.json file
|
||||
type packageJSON struct {
|
||||
Version string `json:"version"`
|
||||
Latest []string `json:"latest"`
|
||||
Author person `json:"author"`
|
||||
Authors people `json:"authors"`
|
||||
Contributors people `json:"contributors"`
|
||||
Maintainers people `json:"maintainers"`
|
||||
License json.RawMessage `json:"license"`
|
||||
Licenses json.RawMessage `json:"licenses"`
|
||||
Name string `json:"name"`
|
||||
Homepage string `json:"homepage"`
|
||||
Description string `json:"description"`
|
||||
Dependencies map[string]string `json:"dependencies"`
|
||||
Repository repository `json:"repository"`
|
||||
Private bool `json:"private"`
|
||||
Version string `json:"version"`
|
||||
Latest []string `json:"latest"`
|
||||
Author person `json:"author"`
|
||||
Authors people `json:"authors"`
|
||||
Contributors people `json:"contributors"`
|
||||
Maintainers people `json:"maintainers"`
|
||||
License json.RawMessage `json:"license"`
|
||||
Licenses json.RawMessage `json:"licenses"`
|
||||
Name string `json:"name"`
|
||||
Homepage string `json:"homepage"`
|
||||
Description string `json:"description"`
|
||||
Dependencies map[string]string `json:"dependencies"`
|
||||
DevDependencies map[string]string `json:"devDependencies"`
|
||||
Repository repository `json:"repository"`
|
||||
Private bool `json:"private"`
|
||||
}
|
||||
|
||||
type person struct {
|
||||
|
||||
@@ -3,6 +3,7 @@ package javascript
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"maps"
|
||||
@@ -13,6 +14,7 @@ import (
|
||||
"github.com/goccy/go-yaml"
|
||||
"github.com/scylladb/go-set/strset"
|
||||
|
||||
"github.com/anchore/syft/internal"
|
||||
"github.com/anchore/syft/internal/log"
|
||||
"github.com/anchore/syft/internal/unknown"
|
||||
"github.com/anchore/syft/syft/artifact"
|
||||
@@ -51,7 +53,7 @@ type yarnPackage struct {
|
||||
Version string
|
||||
Resolved string
|
||||
Integrity string
|
||||
Dependencies map[string]string // We don't currently support dependencies for yarn v1 lock files
|
||||
Dependencies map[string]string
|
||||
}
|
||||
|
||||
type yarnV2PackageEntry struct {
|
||||
@@ -71,6 +73,98 @@ func newGenericYarnLockAdapter(cfg CatalogerConfig) genericYarnLockAdapter {
|
||||
}
|
||||
}
|
||||
|
||||
// readPackageJSONDeps reads the package.json adjacent to the given lockfile location.
|
||||
// NOTE: in yarn workspaces, only the root package.json is consulted. Packages declared
|
||||
// as devDependencies only in a workspace package.json (not the root) will not be detected
|
||||
// as dev-only. This is a safe degradation — those packages will be included in the SBOM
|
||||
// rather than incorrectly filtered out.
|
||||
func readPackageJSONDeps(resolver file.Resolver, lockfileLocation file.Location) (prod, dev map[string]string) {
|
||||
prod = make(map[string]string)
|
||||
dev = make(map[string]string)
|
||||
|
||||
if resolver == nil {
|
||||
return prod, dev
|
||||
}
|
||||
|
||||
pkgJSONLocation := resolver.RelativeFileByPath(lockfileLocation, "package.json")
|
||||
if pkgJSONLocation == nil {
|
||||
log.WithFields("lockfile", lockfileLocation.RealPath).Debug("could not find package.json for dev dependency detection")
|
||||
return prod, dev
|
||||
}
|
||||
|
||||
reader, err := resolver.FileContentsByLocation(*pkgJSONLocation)
|
||||
if err != nil {
|
||||
log.WithFields("location", pkgJSONLocation.RealPath, "error", err).Debug("could not read package.json for dev dependency detection")
|
||||
return prod, dev
|
||||
}
|
||||
defer internal.CloseAndLogError(reader, pkgJSONLocation.RealPath)
|
||||
|
||||
var pkgJSON packageJSON
|
||||
if err := json.NewDecoder(reader).Decode(&pkgJSON); err != nil {
|
||||
log.WithFields("location", pkgJSONLocation.RealPath, "error", err).Debug("could not parse package.json for dev dependency detection")
|
||||
return prod, dev
|
||||
}
|
||||
|
||||
if pkgJSON.Dependencies != nil {
|
||||
prod = pkgJSON.Dependencies
|
||||
}
|
||||
if pkgJSON.DevDependencies != nil {
|
||||
dev = pkgJSON.DevDependencies
|
||||
}
|
||||
|
||||
return prod, dev
|
||||
}
|
||||
|
||||
// findReachable returns all package names reachable from the given roots via BFS
|
||||
// through the dependency graph.
|
||||
func findReachable(roots map[string]string, pkgByName map[string]yarnPackage) map[string]bool {
|
||||
visited := make(map[string]bool)
|
||||
queue := make([]string, 0, len(roots))
|
||||
|
||||
for name := range roots {
|
||||
queue = append(queue, name)
|
||||
}
|
||||
|
||||
for len(queue) > 0 {
|
||||
name := queue[0]
|
||||
queue = queue[1:]
|
||||
|
||||
if visited[name] {
|
||||
continue
|
||||
}
|
||||
visited[name] = true
|
||||
|
||||
if pkg, exists := pkgByName[name]; exists {
|
||||
for depName := range pkg.Dependencies {
|
||||
if !visited[depName] {
|
||||
queue = append(queue, depName)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return visited
|
||||
}
|
||||
|
||||
func findDevOnlyPkgs(yarnPkgs []yarnPackage, prodDeps, devDeps map[string]string) map[string]bool {
|
||||
pkgByName := make(map[string]yarnPackage)
|
||||
for _, p := range yarnPkgs {
|
||||
pkgByName[p.Name] = p
|
||||
}
|
||||
|
||||
prodTransitive := findReachable(prodDeps, pkgByName)
|
||||
devTransitive := findReachable(devDeps, pkgByName)
|
||||
|
||||
devOnly := make(map[string]bool)
|
||||
for name := range devTransitive {
|
||||
if !prodTransitive[name] {
|
||||
devOnly[name] = true
|
||||
}
|
||||
}
|
||||
|
||||
return devOnly
|
||||
}
|
||||
|
||||
func parseYarnV1LockFile(reader io.ReadCloser) ([]yarnPackage, error) {
|
||||
content, err := io.ReadAll(reader)
|
||||
if err != nil {
|
||||
@@ -190,9 +284,17 @@ func (a genericYarnLockAdapter) parseYarnLock(ctx context.Context, resolver file
|
||||
return nil, nil, fmt.Errorf("failed to parse yarn.lock file: %w", err)
|
||||
}
|
||||
|
||||
packages := make([]pkg.Package, len(yarnPkgs))
|
||||
for i, p := range yarnPkgs {
|
||||
packages[i] = newYarnLockPackage(ctx, a.cfg, resolver, reader.Location, p.Name, p.Version, p.Resolved, p.Integrity, p.Dependencies)
|
||||
// get dependencies from sibling package.json for dev dependency detection
|
||||
prodDeps, devDeps := readPackageJSONDeps(resolver, reader.Location)
|
||||
|
||||
devOnlyPkgs := findDevOnlyPkgs(yarnPkgs, prodDeps, devDeps)
|
||||
|
||||
packages := make([]pkg.Package, 0, len(yarnPkgs))
|
||||
for _, p := range yarnPkgs {
|
||||
if devOnlyPkgs[p.Name] && !a.cfg.IncludeDevDependencies {
|
||||
continue
|
||||
}
|
||||
packages = append(packages, newYarnLockPackage(ctx, a.cfg, resolver, reader.Location, p.Name, p.Version, p.Resolved, p.Integrity, p.Dependencies))
|
||||
}
|
||||
|
||||
pkg.Sort(packages)
|
||||
|
||||
@@ -739,6 +739,274 @@ func TestParseYarnFindPackageNames(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseYarnLock_DevDependencies(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
fixtureDir string
|
||||
includeDev bool
|
||||
expected func(file.LocationSet) ([]pkg.Package, []artifact.Relationship)
|
||||
}{
|
||||
{
|
||||
name: "v1 include dev dependencies",
|
||||
fixtureDir: "test-fixtures/yarn-dev-deps",
|
||||
includeDev: true,
|
||||
expected: func(locations file.LocationSet) ([]pkg.Package, []artifact.Relationship) {
|
||||
pkgs := []pkg.Package{
|
||||
{
|
||||
Name: "dev-only-transitive",
|
||||
Version: "1.0.0",
|
||||
Locations: locations,
|
||||
PURL: "pkg:npm/dev-only-transitive@1.0.0",
|
||||
Language: pkg.JavaScript,
|
||||
Type: pkg.NpmPkg,
|
||||
Metadata: pkg.YarnLockEntry{
|
||||
Resolved: "https://registry.yarnpkg.com/dev-only-transitive/-/dev-only-transitive-1.0.0.tgz#abc123",
|
||||
Integrity: "sha512-devonlytransitive==",
|
||||
Dependencies: map[string]string{},
|
||||
},
|
||||
},
|
||||
{
|
||||
Name: "dev-pkg",
|
||||
Version: "1.0.0",
|
||||
Locations: locations,
|
||||
PURL: "pkg:npm/dev-pkg@1.0.0",
|
||||
Language: pkg.JavaScript,
|
||||
Type: pkg.NpmPkg,
|
||||
Metadata: pkg.YarnLockEntry{
|
||||
Resolved: "https://registry.yarnpkg.com/dev-pkg/-/dev-pkg-1.0.0.tgz#def456",
|
||||
Integrity: "sha512-devpkg==",
|
||||
Dependencies: map[string]string{
|
||||
"dev-only-transitive": "^1.0.0",
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
Name: "prod-pkg",
|
||||
Version: "1.0.0",
|
||||
Locations: locations,
|
||||
PURL: "pkg:npm/prod-pkg@1.0.0",
|
||||
Language: pkg.JavaScript,
|
||||
Type: pkg.NpmPkg,
|
||||
Metadata: pkg.YarnLockEntry{
|
||||
Resolved: "https://registry.yarnpkg.com/prod-pkg/-/prod-pkg-1.0.0.tgz#ghi789",
|
||||
Integrity: "sha512-prodpkg==",
|
||||
Dependencies: map[string]string{
|
||||
"shared-pkg": "^1.0.0",
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
Name: "shared-pkg",
|
||||
Version: "1.0.0",
|
||||
Locations: locations,
|
||||
PURL: "pkg:npm/shared-pkg@1.0.0",
|
||||
Language: pkg.JavaScript,
|
||||
Type: pkg.NpmPkg,
|
||||
Metadata: pkg.YarnLockEntry{
|
||||
Resolved: "https://registry.yarnpkg.com/shared-pkg/-/shared-pkg-1.0.0.tgz#jkl012",
|
||||
Integrity: "sha512-sharedpkg==",
|
||||
Dependencies: map[string]string{},
|
||||
},
|
||||
},
|
||||
}
|
||||
rels := []artifact.Relationship{
|
||||
{
|
||||
From: pkgs[0], // dev-only-transitive
|
||||
To: pkgs[1], // dev-pkg
|
||||
Type: artifact.DependencyOfRelationship,
|
||||
},
|
||||
{
|
||||
From: pkgs[3], // shared-pkg
|
||||
To: pkgs[2], // prod-pkg
|
||||
Type: artifact.DependencyOfRelationship,
|
||||
},
|
||||
}
|
||||
return pkgs, rels
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "v1 exclude dev dependencies",
|
||||
fixtureDir: "test-fixtures/yarn-dev-deps",
|
||||
includeDev: false,
|
||||
expected: func(locations file.LocationSet) ([]pkg.Package, []artifact.Relationship) {
|
||||
pkgs := []pkg.Package{
|
||||
{
|
||||
Name: "prod-pkg",
|
||||
Version: "1.0.0",
|
||||
Locations: locations,
|
||||
PURL: "pkg:npm/prod-pkg@1.0.0",
|
||||
Language: pkg.JavaScript,
|
||||
Type: pkg.NpmPkg,
|
||||
Metadata: pkg.YarnLockEntry{
|
||||
Resolved: "https://registry.yarnpkg.com/prod-pkg/-/prod-pkg-1.0.0.tgz#ghi789",
|
||||
Integrity: "sha512-prodpkg==",
|
||||
Dependencies: map[string]string{
|
||||
"shared-pkg": "^1.0.0",
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
Name: "shared-pkg",
|
||||
Version: "1.0.0",
|
||||
Locations: locations,
|
||||
PURL: "pkg:npm/shared-pkg@1.0.0",
|
||||
Language: pkg.JavaScript,
|
||||
Type: pkg.NpmPkg,
|
||||
Metadata: pkg.YarnLockEntry{
|
||||
Resolved: "https://registry.yarnpkg.com/shared-pkg/-/shared-pkg-1.0.0.tgz#jkl012",
|
||||
Integrity: "sha512-sharedpkg==",
|
||||
Dependencies: map[string]string{},
|
||||
},
|
||||
},
|
||||
}
|
||||
rels := []artifact.Relationship{
|
||||
{
|
||||
From: pkgs[1], // shared-pkg
|
||||
To: pkgs[0], // prod-pkg
|
||||
Type: artifact.DependencyOfRelationship,
|
||||
},
|
||||
}
|
||||
return pkgs, rels
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "v2 (berry) include dev dependencies",
|
||||
fixtureDir: "test-fixtures/yarn-berry-dev-deps",
|
||||
includeDev: true,
|
||||
expected: func(locations file.LocationSet) ([]pkg.Package, []artifact.Relationship) {
|
||||
pkgs := []pkg.Package{
|
||||
{
|
||||
Name: "dev-only-transitive",
|
||||
Version: "1.0.0",
|
||||
Locations: locations,
|
||||
PURL: "pkg:npm/dev-only-transitive@1.0.0",
|
||||
Language: pkg.JavaScript,
|
||||
Type: pkg.NpmPkg,
|
||||
Metadata: pkg.YarnLockEntry{
|
||||
Resolved: "dev-only-transitive@npm:1.0.0",
|
||||
Integrity: "abc123",
|
||||
},
|
||||
},
|
||||
{
|
||||
Name: "dev-pkg",
|
||||
Version: "1.0.0",
|
||||
Locations: locations,
|
||||
PURL: "pkg:npm/dev-pkg@1.0.0",
|
||||
Language: pkg.JavaScript,
|
||||
Type: pkg.NpmPkg,
|
||||
Metadata: pkg.YarnLockEntry{
|
||||
Resolved: "dev-pkg@npm:1.0.0",
|
||||
Integrity: "def456",
|
||||
Dependencies: map[string]string{
|
||||
"dev-only-transitive": "^1.0.0",
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
Name: "prod-pkg",
|
||||
Version: "1.0.0",
|
||||
Locations: locations,
|
||||
PURL: "pkg:npm/prod-pkg@1.0.0",
|
||||
Language: pkg.JavaScript,
|
||||
Type: pkg.NpmPkg,
|
||||
Metadata: pkg.YarnLockEntry{
|
||||
Resolved: "prod-pkg@npm:1.0.0",
|
||||
Integrity: "ghi789",
|
||||
Dependencies: map[string]string{
|
||||
"shared-pkg": "^1.0.0",
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
Name: "shared-pkg",
|
||||
Version: "1.0.0",
|
||||
Locations: locations,
|
||||
PURL: "pkg:npm/shared-pkg@1.0.0",
|
||||
Language: pkg.JavaScript,
|
||||
Type: pkg.NpmPkg,
|
||||
Metadata: pkg.YarnLockEntry{
|
||||
Resolved: "shared-pkg@npm:1.0.0",
|
||||
Integrity: "jkl012",
|
||||
},
|
||||
},
|
||||
}
|
||||
rels := []artifact.Relationship{
|
||||
{
|
||||
From: pkgs[0], // dev-only-transitive
|
||||
To: pkgs[1], // dev-pkg
|
||||
Type: artifact.DependencyOfRelationship,
|
||||
},
|
||||
{
|
||||
From: pkgs[3], // shared-pkg
|
||||
To: pkgs[2], // prod-pkg
|
||||
Type: artifact.DependencyOfRelationship,
|
||||
},
|
||||
}
|
||||
return pkgs, rels
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "v2 (berry) exclude dev dependencies",
|
||||
fixtureDir: "test-fixtures/yarn-berry-dev-deps",
|
||||
includeDev: false,
|
||||
expected: func(locations file.LocationSet) ([]pkg.Package, []artifact.Relationship) {
|
||||
pkgs := []pkg.Package{
|
||||
{
|
||||
Name: "prod-pkg",
|
||||
Version: "1.0.0",
|
||||
Locations: locations,
|
||||
PURL: "pkg:npm/prod-pkg@1.0.0",
|
||||
Language: pkg.JavaScript,
|
||||
Type: pkg.NpmPkg,
|
||||
Metadata: pkg.YarnLockEntry{
|
||||
Resolved: "prod-pkg@npm:1.0.0",
|
||||
Integrity: "ghi789",
|
||||
Dependencies: map[string]string{
|
||||
"shared-pkg": "^1.0.0",
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
Name: "shared-pkg",
|
||||
Version: "1.0.0",
|
||||
Locations: locations,
|
||||
PURL: "pkg:npm/shared-pkg@1.0.0",
|
||||
Language: pkg.JavaScript,
|
||||
Type: pkg.NpmPkg,
|
||||
Metadata: pkg.YarnLockEntry{
|
||||
Resolved: "shared-pkg@npm:1.0.0",
|
||||
Integrity: "jkl012",
|
||||
},
|
||||
},
|
||||
}
|
||||
rels := []artifact.Relationship{
|
||||
{
|
||||
From: pkgs[1], // shared-pkg
|
||||
To: pkgs[0], // prod-pkg
|
||||
Type: artifact.DependencyOfRelationship,
|
||||
},
|
||||
}
|
||||
return pkgs, rels
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
fixture := tt.fixtureDir + "/yarn.lock"
|
||||
locations := file.NewLocationSet(file.NewLocation(fixture))
|
||||
expectedPkgs, expectedRels := tt.expected(locations)
|
||||
|
||||
adapter := newGenericYarnLockAdapter(CatalogerConfig{IncludeDevDependencies: tt.includeDev})
|
||||
pkgtest.NewCatalogTester().
|
||||
FromDirectory(t, tt.fixtureDir).
|
||||
FromFile(t, fixture).
|
||||
Expects(expectedPkgs, expectedRels).
|
||||
TestParser(t, adapter.parseYarnLock)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func generateMockYarnRegistryHandler(responseFixture string) func(w http.ResponseWriter, r *http.Request) {
|
||||
return func(w http.ResponseWriter, r *http.Request) {
|
||||
w.WriteHeader(http.StatusOK)
|
||||
|
||||
@@ -0,0 +1,10 @@
|
||||
{
|
||||
"name": "test-project",
|
||||
"version": "1.0.0",
|
||||
"dependencies": {
|
||||
"prod-pkg": "^1.0.0"
|
||||
},
|
||||
"devDependencies": {
|
||||
"dev-pkg": "^1.0.0"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,38 @@
|
||||
# This file is generated by running "yarn install" inside your project.
|
||||
# Manual changes might be lost - proceed with caution!
|
||||
|
||||
__metadata:
|
||||
version: 6
|
||||
cacheKey: 8
|
||||
|
||||
"dev-only-transitive@npm:^1.0.0":
|
||||
version: 1.0.0
|
||||
resolution: "dev-only-transitive@npm:1.0.0"
|
||||
checksum: abc123
|
||||
languageName: node
|
||||
linkType: hard
|
||||
|
||||
"dev-pkg@npm:^1.0.0":
|
||||
version: 1.0.0
|
||||
resolution: "dev-pkg@npm:1.0.0"
|
||||
dependencies:
|
||||
dev-only-transitive: ^1.0.0
|
||||
checksum: def456
|
||||
languageName: node
|
||||
linkType: hard
|
||||
|
||||
"prod-pkg@npm:^1.0.0":
|
||||
version: 1.0.0
|
||||
resolution: "prod-pkg@npm:1.0.0"
|
||||
dependencies:
|
||||
shared-pkg: ^1.0.0
|
||||
checksum: ghi789
|
||||
languageName: node
|
||||
linkType: hard
|
||||
|
||||
"shared-pkg@npm:^1.0.0":
|
||||
version: 1.0.0
|
||||
resolution: "shared-pkg@npm:1.0.0"
|
||||
checksum: jkl012
|
||||
languageName: node
|
||||
linkType: hard
|
||||
@@ -0,0 +1,10 @@
|
||||
{
|
||||
"name": "test-project",
|
||||
"version": "1.0.0",
|
||||
"dependencies": {
|
||||
"prod-pkg": "^1.0.0"
|
||||
},
|
||||
"devDependencies": {
|
||||
"dev-pkg": "^1.0.0"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,27 @@
|
||||
# THIS IS AN AUTOGENERATED FILE. DO NOT EDIT THIS FILE DIRECTLY.
|
||||
# yarn lockfile v1
|
||||
|
||||
|
||||
dev-only-transitive@^1.0.0:
|
||||
version "1.0.0"
|
||||
resolved "https://registry.yarnpkg.com/dev-only-transitive/-/dev-only-transitive-1.0.0.tgz#abc123"
|
||||
integrity sha512-devonlytransitive==
|
||||
|
||||
dev-pkg@^1.0.0:
|
||||
version "1.0.0"
|
||||
resolved "https://registry.yarnpkg.com/dev-pkg/-/dev-pkg-1.0.0.tgz#def456"
|
||||
integrity sha512-devpkg==
|
||||
dependencies:
|
||||
dev-only-transitive "^1.0.0"
|
||||
|
||||
prod-pkg@^1.0.0:
|
||||
version "1.0.0"
|
||||
resolved "https://registry.yarnpkg.com/prod-pkg/-/prod-pkg-1.0.0.tgz#ghi789"
|
||||
integrity sha512-prodpkg==
|
||||
dependencies:
|
||||
shared-pkg "^1.0.0"
|
||||
|
||||
shared-pkg@^1.0.0:
|
||||
version "1.0.0"
|
||||
resolved "https://registry.yarnpkg.com/shared-pkg/-/shared-pkg-1.0.0.tgz#jkl012"
|
||||
integrity sha512-sharedpkg==
|
||||
@@ -84,13 +84,13 @@ func newMetadataFromManifestLine(entry string) (*pkg.RpmDBEntry, error) {
|
||||
}, nil
|
||||
}
|
||||
|
||||
// packageURL returns the PURL for the specific RHEL package (see https://github.com/package-url/purl-spec)
|
||||
// packageURL returns the PURL for the specific RHEL or Hummingbird package (see https://github.com/package-url/purl-spec)
|
||||
func packageURL(name, arch string, epoch *int, srpm string, version, release string, distro *linux.Release) string {
|
||||
var namespace string
|
||||
if distro != nil {
|
||||
namespace = distro.ID
|
||||
}
|
||||
if namespace == "rhel" {
|
||||
if namespace == "rhel" || namespace == "hummingbird" {
|
||||
namespace = "redhat"
|
||||
}
|
||||
if strings.HasPrefix(namespace, "opensuse") {
|
||||
|
||||
@@ -55,6 +55,20 @@ func Test_packageURL(t *testing.T) {
|
||||
},
|
||||
expected: "pkg:rpm/p@v-r",
|
||||
},
|
||||
{
|
||||
name: "hummingbird distro maps to redhat namespace",
|
||||
distro: &linux.Release{
|
||||
ID: "hummingbird",
|
||||
VersionID: "1.0",
|
||||
},
|
||||
metadata: pkg.RpmDBEntry{
|
||||
Name: "p",
|
||||
Version: "v",
|
||||
Release: "r",
|
||||
Epoch: nil,
|
||||
},
|
||||
expected: "pkg:rpm/redhat/p@v-r?distro=hummingbird-1.0",
|
||||
},
|
||||
{
|
||||
name: "with upstream source rpm info",
|
||||
distro: &linux.Release{
|
||||
|
||||
@@ -80,6 +80,10 @@ func (r *rpmdbTestFileResolverMock) FilesByMIMEType(...string) ([]file.Location,
|
||||
return nil, fmt.Errorf("not implemented")
|
||||
}
|
||||
|
||||
func (r *rpmdbTestFileResolverMock) FilesByMediaType(...string) ([]file.Location, error) {
|
||||
return nil, fmt.Errorf("not implemented")
|
||||
}
|
||||
|
||||
func TestParseRpmDB(t *testing.T) {
|
||||
ctx := context.TODO()
|
||||
packagesLocation := file.NewLocation("test-fixtures/Packages")
|
||||
|
||||
@@ -1,10 +1,10 @@
|
||||
package snap
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"compress/gzip"
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"regexp"
|
||||
"strings"
|
||||
|
||||
@@ -22,16 +22,28 @@ type kernelVersionInfo struct {
|
||||
majorVersion string // e.g., "5.4"
|
||||
}
|
||||
|
||||
// parseKernelChangelog parses changelog files from kernel snaps to extract kernel version
|
||||
// parseKernelChangelog parses changelog files from kernel snaps to extract kernel version.
|
||||
// The changelog is gzip-compressed and may be very large, so we stream it line-by-line
|
||||
// rather than reading it entirely into memory.
|
||||
func parseKernelChangelog(_ context.Context, _ file.Resolver, _ *generic.Environment, reader file.LocationReadCloser) ([]pkg.Package, []artifact.Relationship, error) {
|
||||
// The file should be gzipped
|
||||
lines, err := readChangelogLines(reader)
|
||||
gzReader, err := gzip.NewReader(reader)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
return nil, nil, fmt.Errorf("failed to create gzip reader for changelog: %w", err)
|
||||
}
|
||||
defer gzReader.Close()
|
||||
|
||||
scanner := bufio.NewScanner(gzReader)
|
||||
|
||||
// read the first line to extract kernel version
|
||||
// Format: "linux (5.4.0-195.215) focal; urgency=medium"
|
||||
if !scanner.Scan() {
|
||||
if err := scanner.Err(); err != nil {
|
||||
return nil, nil, fmt.Errorf("failed to read changelog content: %w", err)
|
||||
}
|
||||
return nil, nil, fmt.Errorf("changelog file is empty")
|
||||
}
|
||||
|
||||
// pull from first line
|
||||
versionInfo, err := extractKernelVersion(lines[0])
|
||||
versionInfo, err := extractKernelVersion(scanner.Text())
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
@@ -42,38 +54,21 @@ func parseKernelChangelog(_ context.Context, _ file.Resolver, _ *generic.Environ
|
||||
|
||||
packages := createMainKernelPackage(versionInfo, snapMetadata, reader.Location)
|
||||
|
||||
// Check for base kernel package
|
||||
basePackage := findBaseKernelPackage(lines, versionInfo, snapMetadata, reader.Location)
|
||||
if basePackage != nil {
|
||||
packages = append(packages, *basePackage)
|
||||
// stream remaining lines looking for the base kernel entry
|
||||
baseKernelEntry := fmt.Sprintf("%s/linux:", strings.ReplaceAll(versionInfo.releaseVersion, ";", "/"))
|
||||
for scanner.Scan() {
|
||||
line := scanner.Text()
|
||||
if strings.Contains(line, baseKernelEntry) {
|
||||
if basePackage := parseBaseKernelLine(line, versionInfo.majorVersion, snapMetadata, reader.Location); basePackage != nil {
|
||||
packages = append(packages, *basePackage)
|
||||
}
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
return packages, nil, nil
|
||||
}
|
||||
|
||||
// readChangelogLines reads and decompresses the changelog content
|
||||
func readChangelogLines(reader file.LocationReadCloser) ([]string, error) {
|
||||
gzReader, err := gzip.NewReader(reader)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to create gzip reader for changelog: %w", err)
|
||||
}
|
||||
defer gzReader.Close()
|
||||
|
||||
content, err := io.ReadAll(gzReader)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to read changelog content: %w", err)
|
||||
}
|
||||
|
||||
lines := strings.Split(string(content), "\n")
|
||||
if len(lines) == 0 {
|
||||
return nil, fmt.Errorf("changelog file is empty")
|
||||
}
|
||||
|
||||
// Parse the first line to extract kernel version information
|
||||
// Format: "linux (5.4.0-195.215) focal; urgency=medium"
|
||||
return lines, nil
|
||||
}
|
||||
|
||||
// extractKernelVersion parses version information from the first changelog line
|
||||
func extractKernelVersion(firstLine string) (*kernelVersionInfo, error) {
|
||||
// Format: "linux (5.4.0-195.215) focal; urgency=medium"
|
||||
@@ -117,19 +112,6 @@ func createMainKernelPackage(versionInfo *kernelVersionInfo, snapMetadata pkg.Sn
|
||||
return []pkg.Package{kernelPkg}
|
||||
}
|
||||
|
||||
// findBaseKernelPackage searches for and creates base kernel package if present
|
||||
func findBaseKernelPackage(lines []string, versionInfo *kernelVersionInfo, snapMetadata pkg.SnapEntry, location file.Location) *pkg.Package {
|
||||
baseKernelEntry := fmt.Sprintf("%s/linux:", strings.ReplaceAll(versionInfo.releaseVersion, ";", "/"))
|
||||
|
||||
for _, line := range lines {
|
||||
if strings.Contains(line, baseKernelEntry) {
|
||||
return parseBaseKernelLine(line, versionInfo.majorVersion, snapMetadata, location)
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// parseBaseKernelLine extracts base kernel version from a changelog line
|
||||
func parseBaseKernelLine(line string, majorVersion string, snapMetadata pkg.SnapEntry, location file.Location) *pkg.Package {
|
||||
baseKernelRegex := regexp.MustCompile(fmt.Sprintf(`(%s-[0-9]+)\.?[0-9]*`, regexp.QuoteMeta(majorVersion)))
|
||||
|
||||
@@ -0,0 +1,312 @@
|
||||
package snap
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"compress/gzip"
|
||||
"context"
|
||||
"io"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"github.com/anchore/syft/syft/file"
|
||||
"github.com/anchore/syft/syft/pkg"
|
||||
"github.com/anchore/syft/syft/pkg/cataloger/generic"
|
||||
)
|
||||
|
||||
func gzipContent(t *testing.T, content string) []byte {
|
||||
t.Helper()
|
||||
var buf bytes.Buffer
|
||||
w := gzip.NewWriter(&buf)
|
||||
_, err := w.Write([]byte(content))
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, w.Close())
|
||||
return buf.Bytes()
|
||||
}
|
||||
|
||||
func locationReadCloser(t *testing.T, data []byte) file.LocationReadCloser {
|
||||
t.Helper()
|
||||
return file.LocationReadCloser{
|
||||
Location: file.NewLocation("test-fixtures/changelog.Debian.gz"),
|
||||
ReadCloser: io.NopCloser(bytes.NewReader(data)),
|
||||
}
|
||||
}
|
||||
|
||||
func TestExtractKernelVersion(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
firstLine string
|
||||
expected *kernelVersionInfo
|
||||
expectError string
|
||||
}{
|
||||
{
|
||||
name: "standard focal kernel",
|
||||
firstLine: "linux (5.4.0-195.215) focal; urgency=medium",
|
||||
expected: &kernelVersionInfo{
|
||||
baseVersion: "5.4.0-195",
|
||||
releaseVersion: "215",
|
||||
fullVersion: "5.4.0-195.215",
|
||||
majorVersion: "5.4",
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "noble kernel 6.x",
|
||||
firstLine: "linux (6.8.0-50.51) noble; urgency=medium",
|
||||
expected: &kernelVersionInfo{
|
||||
baseVersion: "6.8.0-50",
|
||||
releaseVersion: "51",
|
||||
fullVersion: "6.8.0-50.51",
|
||||
majorVersion: "6.8",
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "jammy kernel",
|
||||
firstLine: "linux (5.15.0-130.140) jammy; urgency=medium",
|
||||
expected: &kernelVersionInfo{
|
||||
baseVersion: "5.15.0-130",
|
||||
releaseVersion: "140",
|
||||
fullVersion: "5.15.0-130.140",
|
||||
majorVersion: "5.15",
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "empty string",
|
||||
firstLine: "",
|
||||
expectError: "could not parse kernel version from changelog",
|
||||
},
|
||||
{
|
||||
name: "no version match",
|
||||
firstLine: "not a valid changelog line",
|
||||
expectError: "could not parse kernel version from changelog",
|
||||
},
|
||||
{
|
||||
name: "missing release version",
|
||||
firstLine: "linux (5.4.0-195) focal; urgency=medium",
|
||||
expectError: "could not parse kernel version from changelog",
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
result, err := extractKernelVersion(tt.firstLine)
|
||||
if tt.expectError != "" {
|
||||
require.Error(t, err)
|
||||
assert.Contains(t, err.Error(), tt.expectError)
|
||||
return
|
||||
}
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, tt.expected.baseVersion, result.baseVersion)
|
||||
assert.Equal(t, tt.expected.releaseVersion, result.releaseVersion)
|
||||
assert.Equal(t, tt.expected.fullVersion, result.fullVersion)
|
||||
assert.Equal(t, tt.expected.majorVersion, result.majorVersion)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestCreateMainKernelPackage(t *testing.T) {
|
||||
location := file.NewLocation("test-fixtures/changelog.Debian.gz")
|
||||
versionInfo := &kernelVersionInfo{
|
||||
baseVersion: "5.4.0-195",
|
||||
releaseVersion: "215",
|
||||
fullVersion: "5.4.0-195.215",
|
||||
majorVersion: "5.4",
|
||||
}
|
||||
snapMetadata := pkg.SnapEntry{
|
||||
SnapType: pkg.SnapTypeKernel,
|
||||
}
|
||||
|
||||
packages := createMainKernelPackage(versionInfo, snapMetadata, location)
|
||||
|
||||
require.Len(t, packages, 1)
|
||||
p := packages[0]
|
||||
assert.Equal(t, "linux-image-5.4.0-195-generic", p.Name)
|
||||
assert.Equal(t, "5.4.0-195.215", p.Version)
|
||||
assert.Equal(t, pkg.DebPkg, p.Type)
|
||||
|
||||
metadata, ok := p.Metadata.(pkg.SnapEntry)
|
||||
require.True(t, ok)
|
||||
assert.Equal(t, pkg.SnapTypeKernel, metadata.SnapType)
|
||||
}
|
||||
|
||||
func TestParseBaseKernelLine(t *testing.T) {
|
||||
location := file.NewLocation("test-fixtures/changelog.Debian.gz")
|
||||
snapMetadata := pkg.SnapEntry{
|
||||
SnapType: pkg.SnapTypeKernel,
|
||||
}
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
line string
|
||||
majorVersion string
|
||||
expectNil bool
|
||||
expectedName string
|
||||
expectedVer string
|
||||
}{
|
||||
{
|
||||
name: "standard base kernel entry",
|
||||
line: " [ Ubuntu: 5.4-100.200 ]",
|
||||
majorVersion: "5.4",
|
||||
expectedName: "linux-image-5.4-100-generic",
|
||||
expectedVer: "5.4-100.200",
|
||||
},
|
||||
{
|
||||
name: "6.x base kernel entry",
|
||||
line: " [ Ubuntu: 6.8-40.41 ]",
|
||||
majorVersion: "6.8",
|
||||
expectedName: "linux-image-6.8-40-generic",
|
||||
expectedVer: "6.8-40.41",
|
||||
},
|
||||
{
|
||||
name: "no matching version",
|
||||
line: " * some random changelog text here",
|
||||
majorVersion: "5.4",
|
||||
expectNil: true,
|
||||
},
|
||||
{
|
||||
name: "empty line",
|
||||
line: "",
|
||||
majorVersion: "5.4",
|
||||
expectNil: true,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
result := parseBaseKernelLine(tt.line, tt.majorVersion, snapMetadata, location)
|
||||
if tt.expectNil {
|
||||
assert.Nil(t, result)
|
||||
return
|
||||
}
|
||||
require.NotNil(t, result)
|
||||
assert.Equal(t, tt.expectedName, result.Name)
|
||||
assert.Equal(t, tt.expectedVer, result.Version)
|
||||
assert.Equal(t, pkg.DebPkg, result.Type)
|
||||
|
||||
metadata, ok := result.Metadata.(pkg.SnapEntry)
|
||||
require.True(t, ok)
|
||||
assert.Equal(t, pkg.SnapTypeKernel, metadata.SnapType)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseKernelChangelog(t *testing.T) {
|
||||
// Realistic changelog content modeled on Ubuntu kernel changelogs.
|
||||
// The first line declares the patched kernel version.
|
||||
// Somewhere later a line references the base upstream kernel.
|
||||
fullChangelog := strings.Join([]string{
|
||||
"linux (5.4.0-195.215) focal; urgency=medium",
|
||||
"",
|
||||
" * focal/linux: 5.4.0-195.215 -proposed tracker (LP: #2083390)",
|
||||
"",
|
||||
" [ Ubuntu: 5.4-100.200 ]",
|
||||
"",
|
||||
" * Some other entry",
|
||||
"",
|
||||
" -- Ubuntu Kernel Team <kernel-team@lists.ubuntu.com> Mon, 01 Jan 2024 00:00:00 +0000",
|
||||
"",
|
||||
}, "\n")
|
||||
|
||||
// Changelog where the base kernel entry line uses the release version pattern
|
||||
// The code builds: fmt.Sprintf("%s/linux:", releaseVersion) → "215/linux:"
|
||||
changelogWithBaseEntry := strings.Join([]string{
|
||||
"linux (5.4.0-195.215) focal; urgency=medium",
|
||||
"",
|
||||
" * focal/linux: 5.4.0-195.215 -proposed tracker",
|
||||
"",
|
||||
" 215/linux: 5.4-100.200 base entry",
|
||||
"",
|
||||
" -- Ubuntu Kernel Team <kernel-team@lists.ubuntu.com> Mon, 01 Jan 2024 00:00:00 +0000",
|
||||
"",
|
||||
}, "\n")
|
||||
|
||||
// Changelog with only the header line and no base kernel match
|
||||
minimalChangelog := "linux (6.8.0-50.51) noble; urgency=medium\n"
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
input []byte
|
||||
expectedCount int
|
||||
expectedNames []string
|
||||
expectedVers []string
|
||||
expectError bool
|
||||
errorContains string
|
||||
}{
|
||||
{
|
||||
name: "full changelog with base kernel via release version pattern",
|
||||
input: gzipContent(t, changelogWithBaseEntry),
|
||||
expectedCount: 2,
|
||||
expectedNames: []string{"linux-image-5.4.0-195-generic", "linux-image-5.4-100-generic"},
|
||||
expectedVers: []string{"5.4.0-195.215", "5.4-100.200"},
|
||||
},
|
||||
{
|
||||
name: "changelog without base kernel match returns only main package",
|
||||
input: gzipContent(t, minimalChangelog),
|
||||
expectedCount: 1,
|
||||
expectedNames: []string{"linux-image-6.8.0-50-generic"},
|
||||
expectedVers: []string{"6.8.0-50.51"},
|
||||
},
|
||||
{
|
||||
name: "full changelog without matching release version pattern returns only main package",
|
||||
input: gzipContent(t, fullChangelog),
|
||||
expectedCount: 1,
|
||||
expectedNames: []string{"linux-image-5.4.0-195-generic"},
|
||||
expectedVers: []string{"5.4.0-195.215"},
|
||||
},
|
||||
{
|
||||
name: "invalid gzip data",
|
||||
input: []byte("not gzip data"),
|
||||
expectError: true,
|
||||
errorContains: "failed to create gzip reader",
|
||||
},
|
||||
{
|
||||
// The old (slurp) implementation produces "could not parse kernel version"
|
||||
// because strings.Split("", "\n") yields [""], not an empty slice.
|
||||
// The new (streaming) implementation produces "changelog file is empty"
|
||||
// because bufio.Scanner.Scan() returns false immediately.
|
||||
// Both correctly reject empty content; only the message differs.
|
||||
name: "empty gzip content",
|
||||
input: gzipContent(t, ""),
|
||||
expectError: true,
|
||||
},
|
||||
{
|
||||
name: "gzip content with unparseable first line",
|
||||
input: gzipContent(t, "this is not a valid kernel changelog\n"),
|
||||
expectError: true,
|
||||
errorContains: "could not parse kernel version from changelog",
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
reader := locationReadCloser(t, tt.input)
|
||||
|
||||
packages, relationships, err := parseKernelChangelog(
|
||||
context.Background(), nil, &generic.Environment{}, reader,
|
||||
)
|
||||
|
||||
if tt.expectError {
|
||||
require.Error(t, err)
|
||||
if tt.errorContains != "" {
|
||||
assert.Contains(t, err.Error(), tt.errorContains)
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
require.NoError(t, err)
|
||||
assert.Nil(t, relationships)
|
||||
require.Len(t, packages, tt.expectedCount)
|
||||
|
||||
for i, p := range packages {
|
||||
assert.Equal(t, tt.expectedNames[i], p.Name, "package %d name", i)
|
||||
assert.Equal(t, tt.expectedVers[i], p.Version, "package %d version", i)
|
||||
assert.Equal(t, pkg.DebPkg, p.Type, "package %d type", i)
|
||||
|
||||
metadata, ok := p.Metadata.(pkg.SnapEntry)
|
||||
require.True(t, ok, "package %d metadata type", i)
|
||||
assert.Equal(t, pkg.SnapTypeKernel, metadata.SnapType, "package %d snap type", i)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -34,4 +34,8 @@ type GGUFFileHeader struct {
|
||||
// across different file locations or remotes. It allows matching identical models even
|
||||
// when stored in different repositories or with different filenames.
|
||||
MetadataKeyValuesHash string `json:"metadataHash,omitempty" cyclonedx:"metadataHash"`
|
||||
|
||||
// Parts contains headers from additional GGUF files that were merged
|
||||
// into this package during post-processing (e.g., from OCI layers without model names).
|
||||
Parts []GGUFFileHeader `json:"parts,omitempty" cyclonedx:"parts"`
|
||||
}
|
||||
|
||||
@@ -76,6 +76,11 @@ func New(cfg Config) (source.Source, error) {
|
||||
|
||||
analysisPath, cleanupFn, err := fileAnalysisPath(cfg.Path, cfg.SkipExtractArchive)
|
||||
if err != nil {
|
||||
if cleanupFn != nil {
|
||||
if cleanupErr := cleanupFn(); cleanupErr != nil {
|
||||
log.Warnf("failed to cleanup temporary directory: %v", cleanupErr)
|
||||
}
|
||||
}
|
||||
return nil, fmt.Errorf("unable to extract file analysis path=%q: %w", cfg.Path, err)
|
||||
}
|
||||
|
||||
@@ -211,7 +216,7 @@ func fileAnalysisPath(path string, skipExtractArchive bool) (string, func() erro
|
||||
if unarchiver, ok := envelopedUnarchiver.(archives.Extractor); err == nil && ok {
|
||||
analysisPath, cleanupFn, err = unarchiveToTmp(path, unarchiver)
|
||||
if err != nil {
|
||||
return "", nil, fmt.Errorf("unable to unarchive source file: %w", err)
|
||||
return "", cleanupFn, fmt.Errorf("unable to unarchive source file: %w", err)
|
||||
}
|
||||
|
||||
log.Debugf("source path is an archive")
|
||||
|
||||
@@ -17,6 +17,7 @@ type ImageMetadata struct {
|
||||
Variant string `json:"architectureVariant,omitempty"`
|
||||
OS string `json:"os"`
|
||||
Labels map[string]string `json:"labels,omitempty"`
|
||||
Annotations map[string]string `json:"annotations,omitempty" id:"-"` // critical: do not consider annotations as an identifiable part of the source image
|
||||
}
|
||||
|
||||
// LayerMetadata represents all static metadata that defines what a container image layer is.
|
||||
|
||||
@@ -0,0 +1,66 @@
|
||||
package internal
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
|
||||
"github.com/opencontainers/go-digest"
|
||||
|
||||
"github.com/anchore/syft/syft/artifact"
|
||||
"github.com/anchore/syft/syft/source"
|
||||
)
|
||||
|
||||
// DeriveImageID derives an artifact ID from the given image metadata. The order of data precedence is:
|
||||
// 1. prefer a digest of the raw container image manifest
|
||||
// 2. if no manifest digest is available, calculate a chain ID from the image layer metadata
|
||||
// 3. if no layer metadata is available, use the user input string
|
||||
//
|
||||
// in all cases, if an alias is provided, it is additionally considered in the ID calculation. This allows for the
|
||||
// same image to be scanned multiple times with different aliases and be considered logically different.
|
||||
func DeriveImageID(alias source.Alias, metadata source.ImageMetadata) artifact.ID {
|
||||
var input string
|
||||
|
||||
if len(metadata.RawManifest) > 0 {
|
||||
input = digest.Canonical.FromBytes(metadata.RawManifest).String()
|
||||
} else {
|
||||
// calculate chain ID for image sources where manifestDigest is not available
|
||||
// https://github.com/opencontainers/image-spec/blob/main/config.md#layer-chainid
|
||||
input = calculateChainID(metadata.Layers)
|
||||
if input == "" {
|
||||
// TODO what happens here if image has no layers?
|
||||
// is this case possible?
|
||||
input = digest.Canonical.FromString(metadata.UserInput).String()
|
||||
}
|
||||
}
|
||||
|
||||
if !alias.IsEmpty() {
|
||||
// if the user provided an alias, we want to consider that in the artifact ID. This way if the user
|
||||
// scans the same item but is considered to be logically different, then ID will express that.
|
||||
aliasStr := fmt.Sprintf(":%s@%s", alias.Name, alias.Version)
|
||||
input = digest.Canonical.FromString(input + aliasStr).String()
|
||||
}
|
||||
|
||||
return ArtifactIDFromDigest(input)
|
||||
}
|
||||
|
||||
// https://github.com/opencontainers/image-spec/blob/main/config.md#layer-chainid
|
||||
func calculateChainID(lm []source.LayerMetadata) string {
|
||||
if len(lm) < 1 {
|
||||
return ""
|
||||
}
|
||||
|
||||
// DiffID(L0) = digest of layer 0
|
||||
// https://github.com/anchore/stereoscope/blob/1b1b744a919964f38d14e1416fb3f25221b761ce/pkg/image/layer_metadata.go#L19-L32
|
||||
chainID := lm[0].Digest
|
||||
id := chain(chainID, lm[1:])
|
||||
|
||||
return id
|
||||
}
|
||||
|
||||
func chain(chainID string, layers []source.LayerMetadata) string {
|
||||
if len(layers) < 1 {
|
||||
return chainID
|
||||
}
|
||||
|
||||
chainID = digest.Canonical.FromString(layers[0].Digest + " " + chainID).String()
|
||||
return chain(chainID, layers[1:])
|
||||
}
|
||||
@@ -0,0 +1,278 @@
|
||||
package internal
|
||||
|
||||
import (
|
||||
"crypto/sha256"
|
||||
"fmt"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
|
||||
"github.com/anchore/syft/syft/artifact"
|
||||
"github.com/anchore/syft/syft/source"
|
||||
)
|
||||
|
||||
func TestDeriveImageID(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
alias source.Alias
|
||||
metadata source.ImageMetadata
|
||||
want artifact.ID
|
||||
}{
|
||||
{
|
||||
name: "use raw manifest over chain ID or user input",
|
||||
metadata: source.ImageMetadata{
|
||||
UserInput: "user-input",
|
||||
Layers: []source.LayerMetadata{
|
||||
{
|
||||
Digest: "a",
|
||||
},
|
||||
{
|
||||
Digest: "b",
|
||||
},
|
||||
{
|
||||
Digest: "c",
|
||||
},
|
||||
},
|
||||
RawManifest: []byte("raw-manifest"),
|
||||
},
|
||||
want: func() artifact.ID {
|
||||
hasher := sha256.New()
|
||||
hasher.Write([]byte("raw-manifest"))
|
||||
return artifact.ID(fmt.Sprintf("%x", hasher.Sum(nil)))
|
||||
}(),
|
||||
},
|
||||
{
|
||||
name: "use chain ID over user input",
|
||||
metadata: source.ImageMetadata{
|
||||
Layers: []source.LayerMetadata{
|
||||
{
|
||||
Digest: "a",
|
||||
},
|
||||
{
|
||||
Digest: "b",
|
||||
},
|
||||
{
|
||||
Digest: "c",
|
||||
},
|
||||
},
|
||||
},
|
||||
want: func() artifact.ID {
|
||||
metadata := []source.LayerMetadata{
|
||||
{
|
||||
Digest: "a",
|
||||
},
|
||||
{
|
||||
Digest: "b",
|
||||
},
|
||||
{
|
||||
Digest: "c",
|
||||
},
|
||||
}
|
||||
return artifact.ID(strings.TrimPrefix(calculateChainID(metadata), "sha256:"))
|
||||
}(),
|
||||
},
|
||||
{
|
||||
name: "use user input last",
|
||||
metadata: source.ImageMetadata{
|
||||
UserInput: "user-input",
|
||||
},
|
||||
want: func() artifact.ID {
|
||||
hasher := sha256.New()
|
||||
hasher.Write([]byte("user-input"))
|
||||
return artifact.ID(fmt.Sprintf("%x", hasher.Sum(nil)))
|
||||
}(),
|
||||
},
|
||||
{
|
||||
name: "without alias (first)",
|
||||
metadata: source.ImageMetadata{
|
||||
UserInput: "user-input",
|
||||
Layers: []source.LayerMetadata{
|
||||
{
|
||||
Digest: "a",
|
||||
},
|
||||
{
|
||||
Digest: "b",
|
||||
},
|
||||
{
|
||||
Digest: "c",
|
||||
},
|
||||
},
|
||||
RawManifest: []byte("raw-manifest"),
|
||||
},
|
||||
want: "85298926ecd92ed57688f13039017160cd728f04dd0d2d10a10629007106f107",
|
||||
},
|
||||
{
|
||||
name: "always consider alias (first)",
|
||||
alias: source.Alias{
|
||||
Name: "alias",
|
||||
Version: "version",
|
||||
},
|
||||
metadata: source.ImageMetadata{
|
||||
UserInput: "user-input",
|
||||
Layers: []source.LayerMetadata{
|
||||
{
|
||||
Digest: "a",
|
||||
},
|
||||
{
|
||||
Digest: "b",
|
||||
},
|
||||
{
|
||||
Digest: "c",
|
||||
},
|
||||
},
|
||||
RawManifest: []byte("raw-manifest"),
|
||||
},
|
||||
want: "a8717e42449960c1dd4963f2f22bd69c7c105e7e82445be0a65aa1825d62ff0d",
|
||||
},
|
||||
{
|
||||
name: "without alias (last)",
|
||||
metadata: source.ImageMetadata{
|
||||
UserInput: "user-input",
|
||||
},
|
||||
want: "ab0dff627d80b9753193d7280bec8f45e8ec6b4cb0912c6fffcf7cd782d9739e",
|
||||
},
|
||||
{
|
||||
name: "always consider alias (last)",
|
||||
alias: source.Alias{
|
||||
Name: "alias",
|
||||
Version: "version",
|
||||
},
|
||||
metadata: source.ImageMetadata{
|
||||
UserInput: "user-input",
|
||||
},
|
||||
want: "fe86c0eecd5654d3c0c0b2176aa394aef6440347c241aa8d9b628dfdde4287cf",
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
assert.Equal(t, tt.want, DeriveImageID(tt.alias, tt.metadata))
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// ensures same metadata produces identical IDs
|
||||
// regardless of whether the source is stereoscope-based or OCI model-based. Both source types
|
||||
// use DeriveImageID with ImageMetadata
|
||||
// this test captures known-good IDs that must remain
|
||||
// stable across refactors to maintain consistency.
|
||||
//
|
||||
// IMPORTANT: If any of these tests fail after a refactor, it means the artifact ID generation
|
||||
// has changed and will break consistency between stereoscope images and OCI model sources.
|
||||
func TestDeriveImageID_CrossSourceConsistency(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
alias source.Alias
|
||||
metadata source.ImageMetadata
|
||||
wantID artifact.ID
|
||||
}{
|
||||
{
|
||||
name: "raw manifest with layers - typical container image",
|
||||
metadata: source.ImageMetadata{
|
||||
UserInput: "docker.io/library/alpine:latest",
|
||||
ManifestDigest: "sha256:abc123",
|
||||
Layers: []source.LayerMetadata{
|
||||
{Digest: "sha256:layer1", MediaType: "application/vnd.oci.image.layer.v1.tar+gzip", Size: 1000},
|
||||
{Digest: "sha256:layer2", MediaType: "application/vnd.oci.image.layer.v1.tar+gzip", Size: 2000},
|
||||
},
|
||||
RawManifest: []byte(`{"schemaVersion":2,"mediaType":"application/vnd.oci.image.manifest.v1+json"}`),
|
||||
},
|
||||
// snapshot: this ID must remain stable for stereoscope/oci-model consistency
|
||||
wantID: "b22c7289dd3b4785a3795c90e15d16bd66bd29b444b8974fe29ed0443ce50405",
|
||||
},
|
||||
{
|
||||
name: "raw manifest only - minimal image",
|
||||
metadata: source.ImageMetadata{
|
||||
RawManifest: []byte(`{"schemaVersion":2}`),
|
||||
},
|
||||
// snapshot: this ID must remain stable
|
||||
wantID: "bafebd36189ad3688b7b3915ea55d461e0bfcfbdde11e54b0a123999fb6be50f",
|
||||
},
|
||||
{
|
||||
name: "chain ID fallback - no raw manifest",
|
||||
metadata: source.ImageMetadata{
|
||||
UserInput: "some-image",
|
||||
Layers: []source.LayerMetadata{
|
||||
{Digest: "sha256:aaa111"},
|
||||
{Digest: "sha256:bbb222"},
|
||||
},
|
||||
},
|
||||
// snapshot: chain ID calculation must remain stable
|
||||
wantID: "0ba9c8d271e6708871505d362e37267c5fb7910066c04d3115b89ba4d34aa180",
|
||||
},
|
||||
{
|
||||
name: "user input fallback - no manifest or layers",
|
||||
metadata: source.ImageMetadata{
|
||||
UserInput: "registry.example.com/org/model:v1.0",
|
||||
},
|
||||
// snapshot: user input hash must remain stable
|
||||
wantID: "a5a8733a3ba3eb99a8ebebcd40c4053f9b896ea6e2217ebc6e885573f20baccf",
|
||||
},
|
||||
{
|
||||
name: "with alias - same image different logical identity",
|
||||
alias: source.Alias{
|
||||
Name: "my-custom-name",
|
||||
Version: "1.0.0",
|
||||
},
|
||||
metadata: source.ImageMetadata{
|
||||
RawManifest: []byte(`{"schemaVersion":2}`),
|
||||
},
|
||||
// snapshot: alias must affect ID deterministically
|
||||
wantID: "9eae41c0efc30023368c29089bac007f2c9d0b40a0ee034081a17c4c22f55ac6",
|
||||
},
|
||||
{
|
||||
name: "annotations has no effect on ID",
|
||||
metadata: source.ImageMetadata{
|
||||
UserInput: "registry.example.com/org/model:v1.0",
|
||||
Annotations: map[string]string{
|
||||
"annotation1": "value1",
|
||||
},
|
||||
},
|
||||
// snapshot: user input hash must remain stable
|
||||
wantID: "a5a8733a3ba3eb99a8ebebcd40c4053f9b896ea6e2217ebc6e885573f20baccf",
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got := DeriveImageID(tt.alias, tt.metadata)
|
||||
assert.Equal(t, tt.wantID, got, "ID must remain stable for cross-source consistency")
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestCalculateChainID(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
layers []source.LayerMetadata
|
||||
want string
|
||||
}{
|
||||
{
|
||||
name: "empty layers returns empty string",
|
||||
layers: []source.LayerMetadata{},
|
||||
want: "",
|
||||
},
|
||||
{
|
||||
name: "single layer returns digest",
|
||||
layers: []source.LayerMetadata{
|
||||
{Digest: "sha256:abc123"},
|
||||
},
|
||||
want: "sha256:abc123",
|
||||
},
|
||||
{
|
||||
name: "multiple layers calculates chain ID",
|
||||
layers: []source.LayerMetadata{
|
||||
{Digest: "a"},
|
||||
{Digest: "b"},
|
||||
{Digest: "c"},
|
||||
},
|
||||
// snapshot - this value should not change
|
||||
want: "sha256:1dfe230e220ef0e6bc0a8978d23d72b95769e76a62879a5f49267d8c007ab43d",
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
assert.Equal(t, tt.want, calculateChainID(tt.layers))
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,4 @@
|
||||
package source
|
||||
|
||||
// OCIModelMetadata is an AI model from an OCI registry, which is a specialized form of ImageMetadata.
|
||||
type OCIModelMetadata ImageMetadata
|
||||
@@ -0,0 +1,245 @@
|
||||
package ocimodelsource
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"sync"
|
||||
|
||||
"github.com/google/go-containerregistry/pkg/name"
|
||||
v1 "github.com/google/go-containerregistry/pkg/v1"
|
||||
|
||||
"github.com/anchore/stereoscope/pkg/image"
|
||||
"github.com/anchore/syft/internal/log"
|
||||
"github.com/anchore/syft/syft/artifact"
|
||||
"github.com/anchore/syft/syft/file"
|
||||
"github.com/anchore/syft/syft/internal/fileresolver"
|
||||
"github.com/anchore/syft/syft/source"
|
||||
"github.com/anchore/syft/syft/source/internal"
|
||||
)
|
||||
|
||||
var _ source.Source = (*ociModelSource)(nil)
|
||||
|
||||
// Config holds the input configuration for an OCI model artifact source.
|
||||
type Config struct {
|
||||
Reference string
|
||||
RegistryOptions *image.RegistryOptions
|
||||
Alias source.Alias
|
||||
}
|
||||
|
||||
// ociModelSource implements the source.Source interface for OCI model artifacts.
|
||||
type ociModelSource struct {
|
||||
id artifact.ID
|
||||
reference string
|
||||
alias source.Alias
|
||||
metadata source.OCIModelMetadata
|
||||
tempDir string
|
||||
resolver interface {
|
||||
file.Resolver
|
||||
file.OCIMediaTypeResolver
|
||||
}
|
||||
mutex *sync.Mutex
|
||||
}
|
||||
|
||||
// NewFromRegistry creates a new OCI model source by fetching the model artifact from a registry.
|
||||
func NewFromRegistry(ctx context.Context, cfg Config) (source.Source, error) {
|
||||
client := newRegistryClient(cfg.RegistryOptions)
|
||||
art, err := validateAndFetchArtifact(ctx, client, cfg.Reference)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
metadata := buildMetadata(art)
|
||||
tempDir, resolver, err := fetchAndStoreGGUFHeaders(ctx, client, art)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
id := internal.DeriveImageID(cfg.Alias, source.ImageMetadata(metadata))
|
||||
return &ociModelSource{
|
||||
id: id,
|
||||
reference: cfg.Reference,
|
||||
alias: cfg.Alias,
|
||||
metadata: metadata,
|
||||
tempDir: tempDir,
|
||||
resolver: resolver,
|
||||
mutex: &sync.Mutex{},
|
||||
}, nil
|
||||
}
|
||||
|
||||
// validateAndFetchArtifact fetches and validates a model artifact in a single registry call.
|
||||
func validateAndFetchArtifact(ctx context.Context, client *registryClient, reference string) (*modelArtifact, error) {
|
||||
art, err := client.fetchModelArtifact(ctx, reference)
|
||||
if err != nil {
|
||||
// errNotModelArtifact is wrapped, so callers can use errors.Is() to check
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if len(art.GGUFLayers) == 0 {
|
||||
return nil, fmt.Errorf("model artifact has no GGUF layers")
|
||||
}
|
||||
|
||||
return art, nil
|
||||
}
|
||||
|
||||
// fetchAndStoreGGUFHeaders fetches GGUF layer headers and stores them in temp files.
|
||||
func fetchAndStoreGGUFHeaders(ctx context.Context, client *registryClient, artifact *modelArtifact) (string, *fileresolver.ContainerImageModel, error) {
|
||||
tempDir, err := os.MkdirTemp("", "syft-oci-gguf")
|
||||
if err != nil {
|
||||
return "", nil, fmt.Errorf("failed to create temp directory: %w", err)
|
||||
}
|
||||
|
||||
layerFiles := make(map[string]fileresolver.LayerInfo)
|
||||
for _, layer := range artifact.GGUFLayers {
|
||||
li, err := fetchSingleGGUFHeader(ctx, client, artifact.Reference, layer, tempDir)
|
||||
if err != nil {
|
||||
osErr := os.RemoveAll(tempDir)
|
||||
if osErr != nil {
|
||||
log.Errorf("unable to remove temp directory (%s): %v", tempDir, err)
|
||||
}
|
||||
return "", nil, err
|
||||
}
|
||||
layerFiles[layer.Digest.String()] = li
|
||||
}
|
||||
|
||||
resolver := fileresolver.NewContainerImageModel(tempDir, layerFiles)
|
||||
|
||||
return tempDir, resolver, nil
|
||||
}
|
||||
|
||||
// fetchSingleGGUFHeader fetches a single GGUF layer header and writes it to a temp file.
|
||||
func fetchSingleGGUFHeader(ctx context.Context, client *registryClient, ref name.Reference, layer v1.Descriptor, tempDir string) (fileresolver.LayerInfo, error) {
|
||||
headerData, err := client.fetchBlobRange(ctx, ref, layer.Digest, maxHeaderBytes)
|
||||
if err != nil {
|
||||
return fileresolver.LayerInfo{}, fmt.Errorf("failed to fetch GGUF layer header: %w", err)
|
||||
}
|
||||
|
||||
digestStr := layer.Digest.String()
|
||||
safeDigest := strings.ReplaceAll(digestStr, ":", "-")
|
||||
tempPath := filepath.Join(tempDir, safeDigest+".gguf")
|
||||
if err := os.WriteFile(tempPath, headerData, 0600); err != nil {
|
||||
return fileresolver.LayerInfo{}, fmt.Errorf("failed to write temp file: %w", err)
|
||||
}
|
||||
|
||||
return fileresolver.LayerInfo{
|
||||
TempPath: tempPath,
|
||||
MediaType: string(layer.MediaType),
|
||||
}, nil
|
||||
}
|
||||
|
||||
// buildMetadata constructs OCIModelMetadata from a modelArtifact.
|
||||
func buildMetadata(artifact *modelArtifact) source.OCIModelMetadata {
|
||||
// layers
|
||||
layers := make([]source.LayerMetadata, len(artifact.Manifest.Layers))
|
||||
for i, layer := range artifact.Manifest.Layers {
|
||||
layers[i] = source.LayerMetadata{
|
||||
MediaType: string(layer.MediaType),
|
||||
Digest: layer.Digest.String(),
|
||||
Size: layer.Size,
|
||||
}
|
||||
}
|
||||
|
||||
// tags
|
||||
var tags []string
|
||||
if tagged, ok := artifact.Reference.(interface{ TagStr() string }); ok {
|
||||
if tag := tagged.TagStr(); tag != "" {
|
||||
tags = []string{tag}
|
||||
}
|
||||
}
|
||||
|
||||
// digests
|
||||
var repoDigests []string
|
||||
if artifact.ManifestDigest != "" {
|
||||
repoDigests = []string{artifact.Reference.Context().String() + "@" + artifact.ManifestDigest}
|
||||
}
|
||||
|
||||
// metadata
|
||||
return source.OCIModelMetadata{
|
||||
UserInput: artifact.Reference.String(),
|
||||
ID: artifact.ManifestDigest,
|
||||
ManifestDigest: artifact.ManifestDigest,
|
||||
MediaType: string(artifact.Manifest.MediaType),
|
||||
Tags: tags,
|
||||
Size: calculateTotalSize(layers),
|
||||
Layers: layers,
|
||||
RawManifest: artifact.RawManifest,
|
||||
RawConfig: artifact.RawConfig,
|
||||
RepoDigests: repoDigests,
|
||||
Architecture: artifact.Config.Architecture,
|
||||
Variant: artifact.Config.Variant,
|
||||
OS: artifact.Config.OS,
|
||||
Labels: artifact.Config.Config.Labels,
|
||||
Annotations: extractManifestAnnotations(artifact.Manifest),
|
||||
}
|
||||
}
|
||||
|
||||
// extractManifestAnnotations extracts annotations from the manifest.
|
||||
func extractManifestAnnotations(manifest *v1.Manifest) map[string]string {
|
||||
if manifest == nil || manifest.Annotations == nil {
|
||||
return make(map[string]string)
|
||||
}
|
||||
return manifest.Annotations
|
||||
}
|
||||
|
||||
// calculateTotalSize sums up the size of all layers.
|
||||
func calculateTotalSize(layers []source.LayerMetadata) int64 {
|
||||
var total int64
|
||||
for _, layer := range layers {
|
||||
total += layer.Size
|
||||
}
|
||||
return total
|
||||
}
|
||||
|
||||
// ID returns the artifact ID.
|
||||
func (s *ociModelSource) ID() artifact.ID {
|
||||
return s.id
|
||||
}
|
||||
|
||||
// Describe returns a description of the source.
|
||||
func (s *ociModelSource) Describe() source.Description {
|
||||
name := s.reference
|
||||
version := ""
|
||||
supplier := ""
|
||||
|
||||
if !s.alias.IsEmpty() {
|
||||
if s.alias.Name != "" {
|
||||
name = s.alias.Name
|
||||
}
|
||||
if s.alias.Version != "" {
|
||||
version = s.alias.Version
|
||||
}
|
||||
if s.alias.Supplier != "" {
|
||||
supplier = s.alias.Supplier
|
||||
}
|
||||
}
|
||||
|
||||
return source.Description{
|
||||
ID: string(s.id),
|
||||
Name: name,
|
||||
Version: version,
|
||||
Supplier: supplier,
|
||||
Metadata: s.metadata,
|
||||
}
|
||||
}
|
||||
|
||||
// FileResolver returns a file resolver for accessing header of GGUF files.
|
||||
func (s *ociModelSource) FileResolver(_ source.Scope) (file.Resolver, error) {
|
||||
return s.resolver, nil
|
||||
}
|
||||
|
||||
// Close cleans up temporary files. Safe to call multiple times.
|
||||
func (s *ociModelSource) Close() error {
|
||||
s.mutex.Lock()
|
||||
defer s.mutex.Unlock()
|
||||
|
||||
if s.tempDir == "" {
|
||||
return nil
|
||||
}
|
||||
|
||||
err := os.RemoveAll(s.tempDir)
|
||||
s.tempDir = ""
|
||||
s.resolver = nil
|
||||
return err
|
||||
}
|
||||
@@ -0,0 +1,36 @@
|
||||
package ocimodelsource
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
"github.com/anchore/stereoscope/pkg/image"
|
||||
"github.com/anchore/syft/syft/source"
|
||||
)
|
||||
|
||||
type ociModelSourceProvider struct {
|
||||
reference string
|
||||
registryOpts *image.RegistryOptions
|
||||
alias source.Alias
|
||||
}
|
||||
|
||||
// NewSourceProvider creates a new OCI model artifact source provider.
|
||||
func NewSourceProvider(reference string, registryOpts *image.RegistryOptions, alias source.Alias) source.Provider {
|
||||
return &ociModelSourceProvider{
|
||||
reference: reference,
|
||||
registryOpts: registryOpts,
|
||||
alias: alias,
|
||||
}
|
||||
}
|
||||
|
||||
func (p *ociModelSourceProvider) Name() string {
|
||||
return "oci-model"
|
||||
}
|
||||
|
||||
func (p *ociModelSourceProvider) Provide(ctx context.Context) (source.Source, error) {
|
||||
cfg := Config{
|
||||
Reference: p.reference,
|
||||
RegistryOptions: p.registryOpts,
|
||||
Alias: p.alias,
|
||||
}
|
||||
return NewFromRegistry(ctx, cfg)
|
||||
}
|
||||
@@ -0,0 +1,217 @@
|
||||
package ocimodelsource
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/tls"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"strings"
|
||||
|
||||
"github.com/google/go-containerregistry/pkg/authn"
|
||||
"github.com/google/go-containerregistry/pkg/name"
|
||||
v1 "github.com/google/go-containerregistry/pkg/v1"
|
||||
"github.com/google/go-containerregistry/pkg/v1/remote"
|
||||
|
||||
"github.com/anchore/stereoscope/pkg/image"
|
||||
)
|
||||
|
||||
// errNotModelArtifact is returned when a reference does not point to a model artifact.
|
||||
var errNotModelArtifact = errors.New("not an OCI model artifact")
|
||||
|
||||
const (
|
||||
// Model artifact media types as per Docker's OCI artifacts for AI model packaging
|
||||
// Reference: https://www.docker.com/blog/oci-artifacts-for-ai-model-packaging/
|
||||
modelConfigMediaTypePrefix = "application/vnd.docker.ai.model.config."
|
||||
ggufLayerMediaType = "application/vnd.docker.ai.gguf.v3"
|
||||
|
||||
// Maximum bytes to read/return for GGUF headers
|
||||
maxHeaderBytes = 8 * 1024 * 1024 // 8 MB
|
||||
)
|
||||
|
||||
// registryClient handles OCI registry interactions for model artifacts.
|
||||
type registryClient struct {
|
||||
options []remote.Option
|
||||
}
|
||||
|
||||
// newRegistryClient creates a new registry client with authentication from RegistryOptions.
|
||||
func newRegistryClient(registryOpts *image.RegistryOptions) *registryClient {
|
||||
opts := buildRemoteOptions(registryOpts)
|
||||
|
||||
return ®istryClient{
|
||||
options: opts,
|
||||
}
|
||||
}
|
||||
|
||||
// buildRemoteOptions converts stereoscope RegistryOptions to go-containerregistry remote.Options.
|
||||
func buildRemoteOptions(registryOpts *image.RegistryOptions) []remote.Option {
|
||||
var opts []remote.Option
|
||||
|
||||
if registryOpts == nil {
|
||||
return opts
|
||||
}
|
||||
|
||||
// Build authenticator
|
||||
authenticator := buildAuthenticator(registryOpts)
|
||||
opts = append(opts, remote.WithAuth(authenticator))
|
||||
|
||||
// Handle TLS settings
|
||||
if registryOpts.InsecureSkipTLSVerify {
|
||||
if transport, ok := remote.DefaultTransport.(*http.Transport); ok {
|
||||
transport = transport.Clone()
|
||||
if transport.TLSClientConfig == nil {
|
||||
transport.TLSClientConfig = &tls.Config{
|
||||
MinVersion: tls.VersionTLS12,
|
||||
}
|
||||
}
|
||||
transport.TLSClientConfig.InsecureSkipVerify = true //#nosec G402 -- user explicitly requested insecure TLS
|
||||
opts = append(opts, remote.WithTransport(transport))
|
||||
}
|
||||
}
|
||||
|
||||
// Handle insecure HTTP
|
||||
if registryOpts.InsecureUseHTTP {
|
||||
opts = append(opts, remote.WithTransport(http.DefaultTransport))
|
||||
}
|
||||
|
||||
return opts
|
||||
}
|
||||
|
||||
// buildAuthenticator creates an authn.Authenticator from RegistryOptions.
|
||||
func buildAuthenticator(registryOpts *image.RegistryOptions) authn.Authenticator {
|
||||
// If credentials are provided, use them
|
||||
if len(registryOpts.Credentials) > 0 {
|
||||
// Use the first credential set (we could enhance this to match by authority)
|
||||
cred := registryOpts.Credentials[0]
|
||||
|
||||
if cred.Token != "" {
|
||||
return &authn.Bearer{Token: cred.Token}
|
||||
}
|
||||
|
||||
if cred.Username != "" || cred.Password != "" {
|
||||
return &authn.Basic{
|
||||
Username: cred.Username,
|
||||
Password: cred.Password,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Fall back to anonymous authenticator
|
||||
return authn.Anonymous
|
||||
}
|
||||
|
||||
// modelArtifact represents a parsed OCI model artifact.
|
||||
type modelArtifact struct {
|
||||
Reference name.Reference
|
||||
Manifest *v1.Manifest
|
||||
Config *v1.ConfigFile
|
||||
RawManifest []byte
|
||||
RawConfig []byte
|
||||
ManifestDigest string
|
||||
GGUFLayers []v1.Descriptor
|
||||
}
|
||||
|
||||
func (c *registryClient) fetchModelArtifact(ctx context.Context, refStr string) (*modelArtifact, error) {
|
||||
ref, err := name.ParseReference(refStr)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to parse reference %q: %w", refStr, err)
|
||||
}
|
||||
|
||||
opts := c.options
|
||||
opts = append(opts, remote.WithContext(ctx))
|
||||
desc, err := remote.Get(ref, opts...)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to fetch descriptor: %w", err)
|
||||
}
|
||||
|
||||
manifest := &v1.Manifest{}
|
||||
if err := json.Unmarshal(desc.Manifest, manifest); err != nil {
|
||||
return nil, fmt.Errorf("failed to unmarshal manifest: %w", err)
|
||||
}
|
||||
|
||||
if !isModelArtifact(manifest) {
|
||||
return nil, fmt.Errorf("%w (config media type: %s)", errNotModelArtifact, manifest.Config.MediaType)
|
||||
}
|
||||
|
||||
img, err := desc.Image()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to get image: %w", err)
|
||||
}
|
||||
|
||||
configFile, err := img.ConfigFile()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to get config file: %w", err)
|
||||
}
|
||||
|
||||
rawConfig, err := img.RawConfigFile()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to get raw config: %w", err)
|
||||
}
|
||||
|
||||
ggufLayers := extractGGUFLayers(manifest)
|
||||
|
||||
return &modelArtifact{
|
||||
Reference: ref,
|
||||
Manifest: manifest,
|
||||
Config: configFile,
|
||||
RawManifest: desc.Manifest,
|
||||
RawConfig: rawConfig,
|
||||
ManifestDigest: desc.Digest.String(),
|
||||
GGUFLayers: ggufLayers,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// isModelArtifact checks if the manifest represents a model artifact.
|
||||
func isModelArtifact(manifest *v1.Manifest) bool {
|
||||
return strings.HasPrefix(string(manifest.Config.MediaType), modelConfigMediaTypePrefix)
|
||||
}
|
||||
|
||||
// extractGGUFLayers extracts GGUF layer descriptors from the manifest.
|
||||
func extractGGUFLayers(manifest *v1.Manifest) []v1.Descriptor {
|
||||
var ggufLayers []v1.Descriptor
|
||||
for _, layer := range manifest.Layers {
|
||||
if string(layer.MediaType) == ggufLayerMediaType {
|
||||
ggufLayers = append(ggufLayers, layer)
|
||||
}
|
||||
}
|
||||
return ggufLayers
|
||||
}
|
||||
|
||||
func (c *registryClient) fetchBlobRange(ctx context.Context, ref name.Reference, digest v1.Hash, maxBytes int64) ([]byte, error) {
|
||||
repo := ref.Context()
|
||||
|
||||
opts := c.options
|
||||
opts = append(opts, remote.WithContext(ctx))
|
||||
layer, err := remote.Layer(repo.Digest(digest.String()), opts...)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to fetch layer: %w", err)
|
||||
}
|
||||
|
||||
reader, err := layer.Compressed()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to get layer reader: %w", err)
|
||||
}
|
||||
// this defer is what causes the download to stop
|
||||
// 1. io.ReadFull(reader, data) reads exactly 8MB into the buffer
|
||||
// 2. The function returns with data[:n]
|
||||
// 3. defer reader.Close() executes, closing the HTTP response body
|
||||
// 4. Closing the response body closes the underlying TCP connection
|
||||
// 5. The server receives TCP FIN/RST and stops sending
|
||||
// note: some data is already in flight when we close so we will see > 8mb over the wire
|
||||
// the full image will not download given we terminate the reader early here
|
||||
defer reader.Close()
|
||||
|
||||
// Note: this is not some arbitrary number picked out of the blue.
|
||||
// This is based on the specification of header data found here:
|
||||
// https://github.com/ggml-org/ggml/blob/master/docs/gguf.md#file-structure
|
||||
data := make([]byte, maxBytes)
|
||||
n, err := io.ReadFull(reader, data)
|
||||
if err != nil && err != io.ErrUnexpectedEOF {
|
||||
// ErrUnexpectedEOF is okay - it means the file is smaller than maxBytes
|
||||
return nil, fmt.Errorf("failed to read layer data: %w", err)
|
||||
}
|
||||
|
||||
return data[:n], nil
|
||||
}
|
||||
@@ -0,0 +1,114 @@
|
||||
package ocimodelsource
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
v1 "github.com/google/go-containerregistry/pkg/v1"
|
||||
"github.com/google/go-containerregistry/pkg/v1/types"
|
||||
"github.com/stretchr/testify/assert"
|
||||
)
|
||||
|
||||
func TestIsModelArtifact(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
manifest *v1.Manifest
|
||||
expected bool
|
||||
}{
|
||||
{
|
||||
name: "valid model artifact",
|
||||
manifest: &v1.Manifest{
|
||||
Config: v1.Descriptor{
|
||||
MediaType: modelConfigMediaTypePrefix + "v1+json",
|
||||
},
|
||||
},
|
||||
expected: true,
|
||||
},
|
||||
{
|
||||
name: "container image",
|
||||
manifest: &v1.Manifest{
|
||||
Config: v1.Descriptor{
|
||||
MediaType: types.DockerConfigJSON,
|
||||
},
|
||||
},
|
||||
expected: false,
|
||||
},
|
||||
{
|
||||
name: "empty media type",
|
||||
manifest: &v1.Manifest{
|
||||
Config: v1.Descriptor{
|
||||
MediaType: "",
|
||||
},
|
||||
},
|
||||
expected: false,
|
||||
},
|
||||
}
|
||||
|
||||
for _, test := range tests {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
result := isModelArtifact(test.manifest)
|
||||
assert.Equal(t, test.expected, result)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestExtractGGUFLayers(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
manifest *v1.Manifest
|
||||
expected int
|
||||
}{
|
||||
{
|
||||
name: "single GGUF layer",
|
||||
manifest: &v1.Manifest{
|
||||
Layers: []v1.Descriptor{
|
||||
{MediaType: types.MediaType(ggufLayerMediaType), Digest: v1.Hash{Algorithm: "sha256", Hex: "abc"}},
|
||||
},
|
||||
},
|
||||
expected: 1,
|
||||
},
|
||||
{
|
||||
name: "multiple GGUF layers",
|
||||
manifest: &v1.Manifest{
|
||||
Layers: []v1.Descriptor{
|
||||
{MediaType: types.MediaType(ggufLayerMediaType), Digest: v1.Hash{Algorithm: "sha256", Hex: "abc"}},
|
||||
{MediaType: types.MediaType(ggufLayerMediaType), Digest: v1.Hash{Algorithm: "sha256", Hex: "def"}},
|
||||
},
|
||||
},
|
||||
expected: 2,
|
||||
},
|
||||
{
|
||||
name: "mixed layers",
|
||||
manifest: &v1.Manifest{
|
||||
Layers: []v1.Descriptor{
|
||||
{MediaType: types.MediaType(ggufLayerMediaType), Digest: v1.Hash{Algorithm: "sha256", Hex: "abc"}},
|
||||
{MediaType: types.DockerLayer, Digest: v1.Hash{Algorithm: "sha256", Hex: "def"}},
|
||||
{MediaType: types.MediaType(ggufLayerMediaType), Digest: v1.Hash{Algorithm: "sha256", Hex: "ghi"}},
|
||||
},
|
||||
},
|
||||
expected: 2,
|
||||
},
|
||||
{
|
||||
name: "no GGUF layers",
|
||||
manifest: &v1.Manifest{
|
||||
Layers: []v1.Descriptor{
|
||||
{MediaType: types.DockerLayer},
|
||||
},
|
||||
},
|
||||
expected: 0,
|
||||
},
|
||||
{
|
||||
name: "empty layers",
|
||||
manifest: &v1.Manifest{
|
||||
Layers: []v1.Descriptor{},
|
||||
},
|
||||
expected: 0,
|
||||
},
|
||||
}
|
||||
|
||||
for _, test := range tests {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
result := extractGGUFLayers(test.manifest)
|
||||
assert.Len(t, result, test.expected)
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -7,6 +7,7 @@ import (
|
||||
"github.com/anchore/syft/syft/source"
|
||||
"github.com/anchore/syft/syft/source/directorysource"
|
||||
"github.com/anchore/syft/syft/source/filesource"
|
||||
"github.com/anchore/syft/syft/source/ocimodelsource"
|
||||
"github.com/anchore/syft/syft/source/snapsource"
|
||||
"github.com/anchore/syft/syft/source/stereoscopesource"
|
||||
)
|
||||
@@ -43,6 +44,14 @@ func All(userInput string, cfg *Config) []collections.TaggedValue[source.Provide
|
||||
// --from docker, registry, etc.
|
||||
Join(stereoscopeProviders.Select(PullTag)...).
|
||||
|
||||
// --from oci-model, registry (for select cases only)
|
||||
// OCI model artifacts with header-only fetching
|
||||
// note: we don't want to use the "pull" tag since it's not actually pulling the full image,
|
||||
// instead we want to match on registry since these models are stored in OCI registries.
|
||||
// This does mean that this must be placed after the pull provider, which is ideal since we don't want to
|
||||
// unnecessarily pull registry headers first if the more common case is the pull providers.
|
||||
Join(tagProvider(ocimodelsource.NewSourceProvider(userInput, cfg.RegistryOptions, cfg.Alias), "registry")).
|
||||
|
||||
// --from snap (remote only)
|
||||
Join(tagProvider(snapsource.NewRemoteSourceProvider(userInput, cfg.Exclude, cfg.DigestAlgorithms, cfg.Alias), SnapTag))
|
||||
}
|
||||
|
||||
@@ -5,7 +5,6 @@ import (
|
||||
|
||||
"github.com/bmatcuk/doublestar/v4"
|
||||
"github.com/distribution/reference"
|
||||
"github.com/opencontainers/go-digest"
|
||||
|
||||
"github.com/anchore/stereoscope/pkg/image"
|
||||
"github.com/anchore/syft/internal/log"
|
||||
@@ -36,7 +35,7 @@ type stereoscopeImageSource struct {
|
||||
func New(img *image.Image, cfg ImageConfig) source.Source {
|
||||
metadata := imageMetadataFromStereoscopeImage(img, cfg.Reference)
|
||||
return &stereoscopeImageSource{
|
||||
id: deriveIDFromStereoscopeImage(cfg.Alias, metadata),
|
||||
id: internal.DeriveImageID(cfg.Alias, metadata),
|
||||
config: cfg,
|
||||
image: img,
|
||||
metadata: metadata,
|
||||
@@ -163,61 +162,6 @@ func imageMetadataFromStereoscopeImage(img *image.Image, reference string) sourc
|
||||
}
|
||||
}
|
||||
|
||||
// deriveIDFromStereoscopeImage derives an artifact ID from the given image metadata. The order of data precedence is:
|
||||
// 1. prefer a digest of the raw container image manifest
|
||||
// 2. if no manifest digest is available, calculate a chain ID from the image layer metadata
|
||||
// 3. if no layer metadata is available, use the user input string
|
||||
//
|
||||
// in all cases, if an alias is provided, it is additionally considered in the ID calculation. This allows for the
|
||||
// same image to be scanned multiple times with different aliases and be considered logically different.
|
||||
func deriveIDFromStereoscopeImage(alias source.Alias, metadata source.ImageMetadata) artifact.ID {
|
||||
var input string
|
||||
|
||||
if len(metadata.RawManifest) > 0 {
|
||||
input = digest.Canonical.FromBytes(metadata.RawManifest).String()
|
||||
} else {
|
||||
// calculate chain ID for image sources where manifestDigest is not available
|
||||
// https://github.com/opencontainers/image-spec/blob/main/config.md#layer-chainid
|
||||
input = calculateChainID(metadata.Layers)
|
||||
if input == "" {
|
||||
// TODO what happens here if image has no layers?
|
||||
// is this case possible?
|
||||
input = digest.Canonical.FromString(metadata.UserInput).String()
|
||||
}
|
||||
}
|
||||
|
||||
if !alias.IsEmpty() {
|
||||
// if the user provided an alias, we want to consider that in the artifact ID. This way if the user
|
||||
// scans the same item but is considered to be logically different, then ID will express that.
|
||||
aliasStr := fmt.Sprintf(":%s@%s", alias.Name, alias.Version)
|
||||
input = digest.Canonical.FromString(input + aliasStr).String()
|
||||
}
|
||||
|
||||
return internal.ArtifactIDFromDigest(input)
|
||||
}
|
||||
|
||||
func calculateChainID(lm []source.LayerMetadata) string {
|
||||
if len(lm) < 1 {
|
||||
return ""
|
||||
}
|
||||
|
||||
// DiffID(L0) = digest of layer 0
|
||||
// https://github.com/anchore/stereoscope/blob/1b1b744a919964f38d14e1416fb3f25221b761ce/pkg/image/layer_metadata.go#L19-L32
|
||||
chainID := lm[0].Digest
|
||||
id := chain(chainID, lm[1:])
|
||||
|
||||
return id
|
||||
}
|
||||
|
||||
func chain(chainID string, layers []source.LayerMetadata) string {
|
||||
if len(layers) < 1 {
|
||||
return chainID
|
||||
}
|
||||
|
||||
chainID = digest.Canonical.FromString(layers[0].Digest + " " + chainID).String()
|
||||
return chain(chainID, layers[1:])
|
||||
}
|
||||
|
||||
func getImageExclusionFunction(exclusions []string) func(string) bool {
|
||||
if len(exclusions) == 0 {
|
||||
return nil
|
||||
|
||||
@@ -2,8 +2,6 @@ package stereoscopesource
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"fmt"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
@@ -12,7 +10,6 @@ import (
|
||||
|
||||
"github.com/anchore/stereoscope"
|
||||
"github.com/anchore/stereoscope/pkg/imagetest"
|
||||
"github.com/anchore/syft/syft/artifact"
|
||||
"github.com/anchore/syft/syft/internal/testutil"
|
||||
"github.com/anchore/syft/syft/source"
|
||||
)
|
||||
@@ -112,146 +109,6 @@ func Test_StereoscopeImage_Exclusions(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func Test_StereoscopeImageSource_ID(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
alias source.Alias
|
||||
metadata source.ImageMetadata
|
||||
want artifact.ID
|
||||
}{
|
||||
{
|
||||
name: "use raw manifest over chain ID or user input",
|
||||
metadata: source.ImageMetadata{
|
||||
UserInput: "user-input",
|
||||
Layers: []source.LayerMetadata{
|
||||
{
|
||||
Digest: "a",
|
||||
},
|
||||
{
|
||||
Digest: "b",
|
||||
},
|
||||
{
|
||||
Digest: "c",
|
||||
},
|
||||
},
|
||||
RawManifest: []byte("raw-manifest"),
|
||||
},
|
||||
want: func() artifact.ID {
|
||||
hasher := sha256.New()
|
||||
hasher.Write([]byte("raw-manifest"))
|
||||
return artifact.ID(fmt.Sprintf("%x", hasher.Sum(nil)))
|
||||
}(),
|
||||
},
|
||||
{
|
||||
name: "use chain ID over user input",
|
||||
metadata: source.ImageMetadata{
|
||||
//UserInput: "user-input",
|
||||
Layers: []source.LayerMetadata{
|
||||
{
|
||||
Digest: "a",
|
||||
},
|
||||
{
|
||||
Digest: "b",
|
||||
},
|
||||
{
|
||||
Digest: "c",
|
||||
},
|
||||
},
|
||||
},
|
||||
want: func() artifact.ID {
|
||||
metadata := []source.LayerMetadata{
|
||||
{
|
||||
Digest: "a",
|
||||
},
|
||||
{
|
||||
Digest: "b",
|
||||
},
|
||||
{
|
||||
Digest: "c",
|
||||
},
|
||||
}
|
||||
return artifact.ID(strings.TrimPrefix(calculateChainID(metadata), "sha256:"))
|
||||
}(),
|
||||
},
|
||||
{
|
||||
name: "use user input last",
|
||||
metadata: source.ImageMetadata{
|
||||
UserInput: "user-input",
|
||||
},
|
||||
want: func() artifact.ID {
|
||||
hasher := sha256.New()
|
||||
hasher.Write([]byte("user-input"))
|
||||
return artifact.ID(fmt.Sprintf("%x", hasher.Sum(nil)))
|
||||
}(),
|
||||
},
|
||||
{
|
||||
name: "without alias (first)",
|
||||
metadata: source.ImageMetadata{
|
||||
UserInput: "user-input",
|
||||
Layers: []source.LayerMetadata{
|
||||
{
|
||||
Digest: "a",
|
||||
},
|
||||
{
|
||||
Digest: "b",
|
||||
},
|
||||
{
|
||||
Digest: "c",
|
||||
},
|
||||
},
|
||||
RawManifest: []byte("raw-manifest"),
|
||||
},
|
||||
want: "85298926ecd92ed57688f13039017160cd728f04dd0d2d10a10629007106f107",
|
||||
},
|
||||
{
|
||||
name: "always consider alias (first)",
|
||||
alias: source.Alias{
|
||||
Name: "alias",
|
||||
Version: "version",
|
||||
},
|
||||
metadata: source.ImageMetadata{
|
||||
UserInput: "user-input",
|
||||
Layers: []source.LayerMetadata{
|
||||
{
|
||||
Digest: "a",
|
||||
},
|
||||
{
|
||||
Digest: "b",
|
||||
},
|
||||
{
|
||||
Digest: "c",
|
||||
},
|
||||
},
|
||||
RawManifest: []byte("raw-manifest"),
|
||||
},
|
||||
want: "a8717e42449960c1dd4963f2f22bd69c7c105e7e82445be0a65aa1825d62ff0d",
|
||||
},
|
||||
{
|
||||
name: "without alias (last)",
|
||||
metadata: source.ImageMetadata{
|
||||
UserInput: "user-input",
|
||||
},
|
||||
want: "ab0dff627d80b9753193d7280bec8f45e8ec6b4cb0912c6fffcf7cd782d9739e",
|
||||
},
|
||||
{
|
||||
name: "always consider alias (last)",
|
||||
alias: source.Alias{
|
||||
Name: "alias",
|
||||
Version: "version",
|
||||
},
|
||||
metadata: source.ImageMetadata{
|
||||
UserInput: "user-input",
|
||||
},
|
||||
want: "fe86c0eecd5654d3c0c0b2176aa394aef6440347c241aa8d9b628dfdde4287cf",
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
assert.Equal(t, tt.want, deriveIDFromStereoscopeImage(tt.alias, tt.metadata))
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func Test_Describe(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
|
||||
Reference in New Issue
Block a user