mirror of
https://github.com/anchore/syft.git
synced 2026-10-30 05:52:25 +01:00
Compare commits
1
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
 Christopher Phillips
|
83dfbcc24a |
+2
-2
@@ -16,7 +16,7 @@ tools:
|
||||
# used to upload test fixture cache
|
||||
- name: oras
|
||||
version:
|
||||
want: v1.3.4
|
||||
want: v1.3.3
|
||||
method: github-release
|
||||
with:
|
||||
repo: oras-project/oras
|
||||
@@ -24,7 +24,7 @@ tools:
|
||||
# used to parse JSON/YAML annotations on the fixture cache image
|
||||
- name: yq
|
||||
version:
|
||||
want: v4.54.1
|
||||
want: v4.53.3
|
||||
method: github-release
|
||||
with:
|
||||
repo: mikefarah/yq
|
||||
|
||||
@@ -1,7 +0,0 @@
|
||||
# check out text files with lf on every platform (regardless of core.autocrlf), letting git detect binary files
|
||||
* text=auto eol=lf
|
||||
|
||||
# fixtures that must keep their crlf line endings byte for byte
|
||||
syft/linux/testdata/unprintable -text
|
||||
syft/pkg/cataloger/bitnami/testdata/no-rel/opt/bitnami/redis/.spdx-redis.spdx -text
|
||||
syft/pkg/cataloger/java/testdata/jar-metadata/**/META-INF/MANIFEST.MF -text
|
||||
@@ -30,7 +30,7 @@ runs:
|
||||
using: "composite"
|
||||
steps:
|
||||
- name: Setup go + go-make tooling
|
||||
uses: anchore/go-make/.github/actions/setup@abf41688f36f2cbec13105747abbe8e60f5c56ca # v0.9.3
|
||||
uses: anchore/go-make/.github/actions/setup@430e2175bb166bfd138ef75466adbcc8ddab875f # v0.8.0
|
||||
with:
|
||||
go-version: ${{ inputs.go-version }}
|
||||
cache-key-prefix: ${{ inputs.cache-key-prefix }}
|
||||
@@ -47,17 +47,7 @@ runs:
|
||||
APT_PACKAGES: ${{ inputs.bootstrap-apt-packages }}
|
||||
run: |
|
||||
IFS=' ' read -ra packages <<< "$APT_PACKAGES"
|
||||
# the AMI already ships most of these, and `apt update` costs ~40MB of index (minutes,
|
||||
# when archive.ubuntu.com throttles us) to find that out. only touch apt if we must.
|
||||
missing=()
|
||||
for pkg in "${packages[@]}"; do
|
||||
dpkg -s "$pkg" >/dev/null 2>&1 || missing+=("$pkg")
|
||||
done
|
||||
if [ ${#missing[@]} -eq 0 ]; then
|
||||
echo "all apt packages already present: ${packages[*]}"
|
||||
exit 0
|
||||
fi
|
||||
DEBIAN_FRONTEND=noninteractive sudo apt update && sudo -E apt install -y "${missing[@]}"
|
||||
DEBIAN_FRONTEND=noninteractive sudo apt update && sudo -E apt install -y "${packages[@]}"
|
||||
|
||||
# ORAS cache: restore-only on non-default branches / forks
|
||||
- name: Restore ORAS cache from github actions
|
||||
|
||||
@@ -21,11 +21,6 @@ updates:
|
||||
- dependency-name: "github.com/aquasecurity/go-version"
|
||||
- dependency-name: "github.com/knqyf263/go-apk-version"
|
||||
- dependency-name: "github.com/knqyf263/go-deb-version"
|
||||
# v1 changed SlicesAsSets hashing, which changes every artifact ID syft produces
|
||||
- dependency-name: "github.com/gohugoio/hashstructure"
|
||||
update-types: ["version-update:semver-major"]
|
||||
# go-make module and action refs are bumped together by automation
|
||||
- dependency-name: "github.com/anchore/go-make"
|
||||
|
||||
- package-ecosystem: "github-actions"
|
||||
directories:
|
||||
@@ -39,6 +34,3 @@ updates:
|
||||
open-pull-requests-limit: 10
|
||||
labels:
|
||||
- "dependencies"
|
||||
ignore:
|
||||
# go-make module and action refs are bumped together by automation
|
||||
- dependency-name: "anchore/go-make/*"
|
||||
|
||||
@@ -46,7 +46,7 @@ jobs:
|
||||
# these are checks that should be run on pull-request and merges to main.
|
||||
# we do NOT want to kick off a release if these have not been verified on main.
|
||||
# Please see the validations.yaml workflow for the names that should be used here.
|
||||
checks: '["Acceptance tests (Linux)", "Acceptance tests (Mac)", "Build snapshot artifacts", "CLI tests (Linux)", "Containers-storage tests (Podman)", "Containers-storage tests (Buildah)", "Integration tests", "Static analysis", "Unit tests"]'
|
||||
checks: '["Acceptance tests (Linux)", "Acceptance tests (Mac)", "Build snapshot artifacts", "CLI tests (Linux)", "Integration tests", "Static analysis", "Unit tests"]'
|
||||
|
||||
release:
|
||||
needs: [check-gate, version-available]
|
||||
@@ -86,16 +86,16 @@ jobs:
|
||||
# dockers_v2 builds multi-platform manifests in a single buildx invocation, which requires a
|
||||
# docker-container driver builder (the default builder on the runner cannot build multi-platform).
|
||||
- name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@f87e5991a6d7451dcb8d9637bfbc97413f497069 # v4.4.1
|
||||
uses: docker/setup-buildx-action@bb05f3f5519dd87d3ba754cc423b652a5edd6d2c # v4.2.0
|
||||
|
||||
- name: Login to Docker Hub
|
||||
uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f #v4.6.0
|
||||
uses: docker/login-action@abd2ef45e78c5afb21d64d4ca52ee8550d9572c7 #v4.5.1
|
||||
with:
|
||||
username: ${{ secrets.ANCHOREOSSWRITE_DH_USERNAME }}
|
||||
password: ${{ secrets.ANCHOREOSSWRITE_DH_PAT }}
|
||||
|
||||
- name: Login to GitHub Container Registry
|
||||
uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f #v4.6.0
|
||||
uses: docker/login-action@abd2ef45e78c5afb21d64d4ca52ee8550d9572c7 #v4.5.1
|
||||
with:
|
||||
registry: ghcr.io
|
||||
username: ${{ github.actor }}
|
||||
@@ -118,7 +118,7 @@ jobs:
|
||||
# for updating brew formula in anchore/homebrew-syft
|
||||
GITHUB_BREW_TOKEN: ${{ secrets.ANCHOREOPS_GITHUB_OSS_WRITE_TOKEN }}
|
||||
|
||||
- uses: anchore/sbom-action@66cbf4bc1f1c0d2edc94016e65bc221b6bb0ad6c #v0.24.3
|
||||
- uses: anchore/sbom-action@e22c389904149dbc22b58101806040fa8d37a610 #v0.24.0
|
||||
continue-on-error: true
|
||||
with:
|
||||
file: go.mod
|
||||
|
||||
@@ -25,7 +25,7 @@ jobs:
|
||||
persist-credentials: false
|
||||
|
||||
- name: "Run zizmor"
|
||||
uses: zizmorcore/zizmor-action@cc914d7f3750a2d13d75c7f184a1060aa0e9d482 # v0.6.4
|
||||
uses: zizmorcore/zizmor-action@6fc4b006235f201fdab3722e17240ab420d580e5 # v0.6.1
|
||||
with:
|
||||
# there is a pass/fail gate as a repo ruleset (if there is no ruleset configured then the action will pass by default)
|
||||
advanced-security: true
|
||||
|
||||
@@ -21,13 +21,10 @@ jobs:
|
||||
name: "Static analysis"
|
||||
# runs-on.com: memory & general purpose instances for testing
|
||||
# spot enabled: ok to interrupt non-production workloads
|
||||
# tmpfs intentionally omitted -- it RAM-backs /home/runner, /tmp and /var/lib/docker at
|
||||
# 100% of instance memory, starving `make unit` of headroom to link with -race.
|
||||
# volume: the caches and docker images tmpfs used to hold. gp3 baseline throughput is
|
||||
# enough; a 4 vCPU instance can't pull much more than that anyway.
|
||||
# tmpfs: faster io-intensive workflows
|
||||
# note: s3-cache intentionally omitted -- PR runs are untrusted and must not write to the
|
||||
# shared cache backend that the trusted release workflow reads from (cache poisoning).
|
||||
runs-on: &test-runner "runs-on=${{ github.run_id }}/cpu=4+8/ram=16+32/family=m6+m7+m8+c6+c7+c8/spot=price-capacity-optimized/volume=100gb"
|
||||
runs-on: &test-runner "runs-on=${{ github.run_id }}/cpu=4+8/ram=32+128/family=r5+r6+r7+r8+m4+m5+m6+m7+m8/spot=price-capacity-optimized/extras=tmpfs"
|
||||
permissions:
|
||||
contents: read
|
||||
steps:
|
||||
@@ -49,24 +46,6 @@ jobs:
|
||||
Unit-Test:
|
||||
# Note: changing this job name requires making the same update in the .github/workflows/release.yaml pipeline
|
||||
name: "Unit tests"
|
||||
runs-on: ubuntu-latest
|
||||
needs: [Unit-Test-Linux, Unit-Test-Windows]
|
||||
# a skipped job satisfies a required check, so this job must always run and fail explicitly
|
||||
# unless every dependency succeeded (otherwise a failing PR can still be auto-merged)
|
||||
if: ${{ always() }}
|
||||
steps:
|
||||
- name: Require all unit test jobs to succeed
|
||||
env:
|
||||
NEEDS: ${{ toJSON(needs) }}
|
||||
run: |
|
||||
echo "$NEEDS" | jq -e 'length > 0 and all(.[]; .result == "success")' > /dev/null || {
|
||||
echo "$NEEDS" | jq -r 'to_entries[] | "\(.key): \(.value.result)"'
|
||||
exit 1
|
||||
}
|
||||
echo All unit tests passed!
|
||||
|
||||
Unit-Test-Linux:
|
||||
name: "Unit tests (Linux)"
|
||||
runs-on: *test-runner
|
||||
permissions:
|
||||
contents: read
|
||||
@@ -86,35 +65,6 @@ jobs:
|
||||
- name: Check for capability drift
|
||||
run: make check-capability-drift
|
||||
|
||||
Unit-Test-Windows:
|
||||
name: "Unit tests (Windows)"
|
||||
runs-on: windows-latest
|
||||
permissions:
|
||||
contents: read
|
||||
steps:
|
||||
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 #v7.0.1
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
# go-make's setup only (not ./.github/actions/bootstrap): binny ships no windows release, so binny tools
|
||||
# are unavailable here (`make unit` skips the fixture refresh on windows)
|
||||
- name: Setup go + go-make tooling
|
||||
uses: anchore/go-make/.github/actions/setup@abf41688f36f2cbec13105747abbe8e60f5c56ca # v0.9.3
|
||||
with:
|
||||
# keep in sync with the go-version default in ./.github/actions/bootstrap
|
||||
go-version: "1.26.2"
|
||||
|
||||
# the binny-free version of `make download-test-fixture-cache` (keep the oras version in sync with .binny.yaml)
|
||||
- name: Download test fixture cache
|
||||
shell: bash
|
||||
run: |
|
||||
curl -sSfL https://github.com/oras-project/oras/releases/download/v1.3.4/oras_1.3.4_windows_amd64.zip -o "$RUNNER_TEMP/oras.zip"
|
||||
unzip -q "$RUNNER_TEMP/oras.zip" oras.exe -d "$RUNNER_TEMP"
|
||||
"$RUNNER_TEMP/oras.exe" pull ghcr.io/anchore/oss-cache/syft-test-fixture-cache:v2
|
||||
|
||||
- name: Run unit tests
|
||||
run: make unit
|
||||
|
||||
Integration-Test:
|
||||
# Note: changing this job name requires making the same update in the .github/workflows/release.yaml pipeline
|
||||
name: "Integration tests"
|
||||
@@ -171,78 +121,12 @@ jobs:
|
||||
run: make snapshot-smoke-test
|
||||
|
||||
- name: Upload snapshot artifacts
|
||||
uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 #v7.0.2
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a #v7.0.1
|
||||
with:
|
||||
name: snapshot
|
||||
path: snapshot/
|
||||
retention-days: 30
|
||||
|
||||
Containers-Storage-Podman:
|
||||
name: "Containers-storage tests (Podman)"
|
||||
needs: [Build-Snapshot-Artifacts]
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
contents: read
|
||||
steps:
|
||||
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 #v7.0.1
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 #v8.0.2
|
||||
with:
|
||||
name: snapshot
|
||||
path: snapshot
|
||||
|
||||
# go-make's setup only (not ./.github/actions/bootstrap): the test only needs make + the snapshot binary
|
||||
- name: Setup go + go-make tooling
|
||||
uses: anchore/go-make/.github/actions/setup@abf41688f36f2cbec13105747abbe8e60f5c56ca # v0.9.3
|
||||
with:
|
||||
# keep in sync with the go-version default in ./.github/actions/bootstrap
|
||||
go-version: "1.26.2"
|
||||
|
||||
- name: Install Podman
|
||||
run: |
|
||||
sudo apt-get update
|
||||
sudo apt-get install --no-install-recommends -y jq podman uidmap
|
||||
|
||||
- name: Scan image from Podman's local store
|
||||
run: make containers-storage-test
|
||||
env:
|
||||
BUILDER: podman
|
||||
|
||||
Containers-Storage-Buildah:
|
||||
name: "Containers-storage tests (Buildah)"
|
||||
needs: [Build-Snapshot-Artifacts]
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
contents: read
|
||||
steps:
|
||||
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 #v7.0.1
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 #v8.0.2
|
||||
with:
|
||||
name: snapshot
|
||||
path: snapshot
|
||||
|
||||
# go-make's setup only (not ./.github/actions/bootstrap): the test only needs make + the snapshot binary
|
||||
- name: Setup go + go-make tooling
|
||||
uses: anchore/go-make/.github/actions/setup@abf41688f36f2cbec13105747abbe8e60f5c56ca # v0.9.3
|
||||
with:
|
||||
# keep in sync with the go-version default in ./.github/actions/bootstrap
|
||||
go-version: "1.26.2"
|
||||
|
||||
- name: Install Buildah
|
||||
run: |
|
||||
sudo apt-get update
|
||||
sudo apt-get install --no-install-recommends -y buildah jq uidmap
|
||||
|
||||
- name: Scan image from Buildah's local store
|
||||
run: make containers-storage-test
|
||||
env:
|
||||
BUILDER: buildah
|
||||
|
||||
Acceptance-Linux:
|
||||
# Note: changing this job name requires making the same update in the .github/workflows/release.yaml pipeline
|
||||
name: "Acceptance tests (Linux)"
|
||||
@@ -261,7 +145,7 @@ jobs:
|
||||
download-test-fixture-cache: true
|
||||
|
||||
- name: Download snapshot artifacts
|
||||
uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 #v8.0.2
|
||||
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c #v8.0.1
|
||||
with:
|
||||
name: snapshot
|
||||
path: snapshot
|
||||
@@ -307,7 +191,7 @@ jobs:
|
||||
download-test-fixture-cache: true
|
||||
|
||||
- name: Download snapshot artifacts
|
||||
uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 #v8.0.2
|
||||
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c #v8.0.1
|
||||
with:
|
||||
name: snapshot
|
||||
path: snapshot
|
||||
@@ -339,7 +223,7 @@ jobs:
|
||||
download-test-fixture-cache: true
|
||||
|
||||
- name: Download snapshot artifacts
|
||||
uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 #v8.0.2
|
||||
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c #v8.0.1
|
||||
with:
|
||||
name: snapshot
|
||||
path: snapshot
|
||||
|
||||
@@ -20,7 +20,6 @@ bin/
|
||||
/.tool
|
||||
/.task
|
||||
/generate
|
||||
/spec
|
||||
/specs
|
||||
mise.toml
|
||||
.make/.make
|
||||
|
||||
+1
-22
@@ -1,17 +1,12 @@
|
||||
version: "2"
|
||||
run:
|
||||
tests: false
|
||||
# keep in sync with buildTags in .make/main.go
|
||||
build-tags:
|
||||
- containers_image_openpgp
|
||||
- exclude_graphdriver_btrfs
|
||||
linters:
|
||||
default: none
|
||||
enable:
|
||||
- asciicheck
|
||||
- bodyclose
|
||||
- copyloopvar
|
||||
- depguard
|
||||
- dogsled
|
||||
- dupl
|
||||
- errcheck
|
||||
@@ -34,20 +29,9 @@ linters:
|
||||
- unused
|
||||
- whitespace
|
||||
settings:
|
||||
depguard:
|
||||
rules:
|
||||
yaml:
|
||||
deny:
|
||||
# gopkg.in/yaml.v3 is unmaintained and panics on some malformed input (see #5327)
|
||||
- pkg: gopkg.in/yaml.v3
|
||||
desc: use go.yaml.in/yaml/v3 instead
|
||||
funlen:
|
||||
lines: 70
|
||||
statements: 50
|
||||
goconst:
|
||||
# the default of 3 turns every repeated literal in a lookup table or a test fixture into a finding.
|
||||
# 5 keeps the ones worth hoisting without flagging data that reads better written out.
|
||||
min-occurrences: 5
|
||||
gocritic:
|
||||
enabled-checks:
|
||||
- deferInLoop
|
||||
@@ -70,12 +54,6 @@ linters:
|
||||
- linters:
|
||||
- revive
|
||||
text: "var-naming: avoid package names that conflict"
|
||||
# cpegenerate is almost entirely lookup tables mapping artifact names to vendors, products, and
|
||||
# maven group ids. The repeated strings are the data itself, so hoisting them to constants would
|
||||
# replace a readable table with indirection for no gain.
|
||||
- linters:
|
||||
- goconst
|
||||
path: syft/pkg/cataloger/internal/cpegenerate/
|
||||
paths:
|
||||
- third_party$
|
||||
- builtin$
|
||||
@@ -83,6 +61,7 @@ linters:
|
||||
|
||||
# do not enable...
|
||||
# - deadcode # The owner seems to have abandoned the linter. Replaced by "unused".
|
||||
# - depguard # We don't have a configuration for this yet
|
||||
# - goprintffuncname # does not catch all cases and there are exceptions
|
||||
# - nakedret # does not catch all cases and should not fail a build
|
||||
# - gochecknoglobals
|
||||
|
||||
+6
-4
@@ -15,7 +15,6 @@ builds:
|
||||
binary: syft
|
||||
goos: [linux]
|
||||
goarch: [amd64, arm64, ppc64le, riscv64, s390x]
|
||||
tags: [containers_image_openpgp, exclude_graphdriver_btrfs]
|
||||
mod_timestamp: &build-timestamp '{{ .CommitTimestamp }}'
|
||||
ldflags: &build-ldflags |
|
||||
-w
|
||||
@@ -168,13 +167,16 @@ sboms:
|
||||
- "--output"
|
||||
- "json=$document"
|
||||
|
||||
# the signature, certificate, and transparency log proof all land in a single sigstore bundle.
|
||||
signs:
|
||||
- cmd: .tool/cosign
|
||||
signature: "${artifact}.sigstore.json"
|
||||
signature: "${artifact}.sig"
|
||||
certificate: "${artifact}.pem"
|
||||
args:
|
||||
- "sign-blob"
|
||||
- "--bundle=${signature}"
|
||||
- "--use-signing-config=false"
|
||||
- "--oidc-issuer=https://token.actions.githubusercontent.com"
|
||||
- "--output-certificate=${certificate}"
|
||||
- "--output-signature=${signature}"
|
||||
- "${artifact}"
|
||||
- "--yes"
|
||||
artifacts: checksum
|
||||
|
||||
+5
-5
@@ -1,12 +1,12 @@
|
||||
module github.com/anchore/syft/.make
|
||||
|
||||
go 1.26.0
|
||||
go 1.25.8
|
||||
|
||||
require github.com/anchore/go-make v0.9.3
|
||||
require github.com/anchore/go-make v0.8.0
|
||||
|
||||
require (
|
||||
github.com/bmatcuk/doublestar/v4 v4.10.2 // indirect
|
||||
github.com/bmatcuk/doublestar/v4 v4.10.0 // indirect
|
||||
github.com/goccy/go-yaml v1.19.2 // indirect
|
||||
golang.org/x/mod v0.41.0 // indirect
|
||||
golang.org/x/sys v0.48.0 // indirect
|
||||
golang.org/x/mod v0.37.0 // indirect
|
||||
golang.org/x/sys v0.46.0 // indirect
|
||||
)
|
||||
|
||||
+8
-8
@@ -1,10 +1,10 @@
|
||||
github.com/anchore/go-make v0.9.3 h1:D+MkpvEJsOGwkOqKN0rd2zYi9jEzwKtZreNysH0DenI=
|
||||
github.com/anchore/go-make v0.9.3/go.mod h1:4HuELygGhxex2SaCEgfCbZ0geJ3xFkqtWpmt1NrYwPg=
|
||||
github.com/bmatcuk/doublestar/v4 v4.10.2 h1:eF7W7HWKg3z9NrWV9pTLnNeoXaqq3Tq9DNKXVMfoCnw=
|
||||
github.com/bmatcuk/doublestar/v4 v4.10.2/go.mod h1:xBQ8jztBU6kakFMg+8WGxn0c6z1fTSPVIjEY1Wr7jzc=
|
||||
github.com/anchore/go-make v0.8.0 h1:3XAD4WNHIbqY4kwFyCQJ7kKiYLzSzjB0UwzDVwRqlUY=
|
||||
github.com/anchore/go-make v0.8.0/go.mod h1:4M6TnArb5w693VyWsgr5dCWrk2BLNu/ed4JUcsrzS34=
|
||||
github.com/bmatcuk/doublestar/v4 v4.10.0 h1:zU9WiOla1YA122oLM6i4EXvGW62DvKZVxIe6TYWexEs=
|
||||
github.com/bmatcuk/doublestar/v4 v4.10.0/go.mod h1:xBQ8jztBU6kakFMg+8WGxn0c6z1fTSPVIjEY1Wr7jzc=
|
||||
github.com/goccy/go-yaml v1.19.2 h1:PmFC1S6h8ljIz6gMRBopkjP1TVT7xuwrButHID66PoM=
|
||||
github.com/goccy/go-yaml v1.19.2/go.mod h1:XBurs7gK8ATbW4ZPGKgcbrY1Br56PdM69F7LkFRi1kA=
|
||||
golang.org/x/mod v0.41.0 h1:qJmnOUb4YB+FsEuM3HcWucdZASCPGhsX6uljO6pog0c=
|
||||
golang.org/x/mod v0.41.0/go.mod h1:Ek9pY8RKWXwsWvd3rQiHYtMqkjSUV+s1Rj7j4H5Ur6o=
|
||||
golang.org/x/sys v0.48.0 h1:bbX/i/6MgT9BVLM9RT1thmxL04yeTAhbEz4SyadbXoo=
|
||||
golang.org/x/sys v0.48.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og=
|
||||
golang.org/x/mod v0.37.0 h1:vF1DjpVEshcIqoEaauuHebaLk1O1forxjxBaVn884JQ=
|
||||
golang.org/x/mod v0.37.0/go.mod h1:m8S8VeM9r4dzDwjrKO0a1sZP3YjeMamRRlD+fmR2Q/0=
|
||||
golang.org/x/sys v0.46.0 h1:noSf2Fq6F8DBgS+LysIkx7rIExoNHJsxOAtPp4rthXw=
|
||||
golang.org/x/sys v0.46.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
|
||||
|
||||
+3
-37
@@ -16,12 +16,6 @@ import (
|
||||
"github.com/anchore/go-make/tasks/gotest"
|
||||
)
|
||||
|
||||
// buildTags must match the linux build in .goreleaser.yaml so tests and lint compile the same code that ships
|
||||
// (containers_image_openpgp compiles in stereoscope's real containers-storage provider instead of its stub).
|
||||
// exclude_graphdriver_btrfs drops the cgo-only btrfs driver, which needs libbtrfs headers whenever cgo is on (e.g.
|
||||
// under -race); release builds are CGO_ENABLED=0 and never include it anyway.
|
||||
const buildTags = "containers_image_openpgp,exclude_graphdriver_btrfs"
|
||||
|
||||
func main() {
|
||||
Makefile(
|
||||
// shared anchore tasks
|
||||
@@ -35,7 +29,6 @@ func main() {
|
||||
gotest.Name("unit"),
|
||||
gotest.ExcludeGlob("**/test/**"),
|
||||
gotest.CoverageThreshold(62),
|
||||
gotest.Tags(buildTags),
|
||||
race(),
|
||||
),
|
||||
|
||||
@@ -59,7 +52,7 @@ func main() {
|
||||
raceFlag = " -race"
|
||||
}
|
||||
Run(
|
||||
"go test -count=1 -timeout=30m -tags="+buildTags+raceFlag+" ./cmd/syft/internal/test/integration/...",
|
||||
"go test -count=1 -timeout=30m -v"+raceFlag+" ./cmd/syft/internal/test/integration/...",
|
||||
run.Env("GODEBUG", "dontfreezetheworld=1"),
|
||||
)
|
||||
},
|
||||
@@ -73,7 +66,7 @@ func main() {
|
||||
Log("race detector disabled (RACE=false); skipping race smoke")
|
||||
return
|
||||
}
|
||||
Run("go run -race -tags="+buildTags+" cmd/syft/main.go anchore/test_images:grype-quality-dotnet-69f15d2")
|
||||
Run("go run -race cmd/syft/main.go anchore/test_images:grype-quality-dotnet-69f15d2")
|
||||
},
|
||||
},
|
||||
|
||||
@@ -103,24 +96,6 @@ func main() {
|
||||
},
|
||||
},
|
||||
|
||||
// containers-storage tests: build a fixture image with BUILDER (podman or buildah) into an isolated local store
|
||||
// and scan it with the snapshot binary. Linux only, and not hooked into "test" since it needs the builder installed.
|
||||
Task{
|
||||
Name: "containers-storage-test",
|
||||
Description: "Run containers-storage tests (BUILDER=podman|buildah)",
|
||||
Run: func() {
|
||||
bin := snapshotBinPath()
|
||||
if !file.Exists(bin) {
|
||||
Log("snapshot binary not found at %s; building single-target snapshot", bin)
|
||||
Run("make snapshot:single-target")
|
||||
}
|
||||
Run(
|
||||
"bash test/containers-storage/source-test.sh "+config.Env("BUILDER", "podman"),
|
||||
run.Env("SYFT_BINARY_LOCATION", bin),
|
||||
)
|
||||
},
|
||||
},
|
||||
|
||||
// default validation pipeline (replaces Taskfile `default`/`pr-validations`/`validations`).
|
||||
Task{
|
||||
Name: "default",
|
||||
@@ -164,7 +139,7 @@ func main() {
|
||||
Task{
|
||||
Name: "unit:syft",
|
||||
RunsOn: lang.List("unit"),
|
||||
Dependencies: unitFixtureDeps(),
|
||||
Dependencies: Deps("refresh-fixtures"),
|
||||
},
|
||||
Task{
|
||||
Name: "clean:syft",
|
||||
@@ -192,15 +167,6 @@ func raceEnabled() bool {
|
||||
return config.CI && !config.Windows
|
||||
}
|
||||
|
||||
// unitFixtureDeps skips the fixture cache refresh on windows: it needs binny-managed tools (binny ships
|
||||
// no windows release) and linux docker, so fixture-backed tests there have to skip themselves instead.
|
||||
func unitFixtureDeps() []string {
|
||||
if config.Windows {
|
||||
return nil
|
||||
}
|
||||
return Deps("refresh-fixtures")
|
||||
}
|
||||
|
||||
// race applies raceEnabled() to a gotest suite. gotest exposes no functional option
|
||||
// for the race detector, but gotest.Config.Race is exported.
|
||||
func race() gotest.Option {
|
||||
|
||||
+1
-1
@@ -1,6 +1,6 @@
|
||||
# Syft Contributors
|
||||
|
||||
This file credits notable contributions from external authors and organizations; see `git log` for everyone who has contributed to Syft.
|
||||
The following Syft components were contributed by external authors/organizations.
|
||||
|
||||
## GraalVM Native Image
|
||||
|
||||
|
||||
@@ -1,4 +0,0 @@
|
||||
Copyright (c) Anchore, Inc.
|
||||
|
||||
See LICENSE for the full license text.
|
||||
See CONTRIBUTORS.md and git log for a full list of contributors.
|
||||
@@ -8,6 +8,7 @@
|
||||
|
||||
<p align="center">
|
||||
<a href="https://github.com/anchore/syft/actions/workflows/validations.yaml" target="_blank"><img alt="Validations" src="https://github.com/anchore/syft/actions/workflows/validations.yaml/badge.svg"></a>
|
||||
<a href="https://goreportcard.com/report/github.com/anchore/syft" target="_blank"><img alt="Go Report Card" src="https://goreportcard.com/badge/github.com/anchore/syft"></a>
|
||||
<a href="https://github.com/anchore/syft/releases/latest" target="_blank"><img alt="GitHub release" src="https://img.shields.io/github/release/anchore/syft.svg"></a>
|
||||
<a href="https://github.com/anchore/syft" target="_blank"><img alt="GitHub go.mod Go version" src="https://img.shields.io/github/go-mod/go-version/anchore/syft.svg"></a>
|
||||
<a href="https://github.com/anchore/syft/blob/main/LICENSE" target="_blank"><img alt="License: Apache-2.0" src="https://img.shields.io/badge/License-Apache%202.0-blue.svg"></a>
|
||||
@@ -61,18 +62,6 @@ syft <image> -o cyclonedx-json
|
||||
syft <image> -o spdx-json=./spdx.json -o cyclonedx-json=./cdx.json
|
||||
```
|
||||
|
||||
### Local containers-storage images
|
||||
|
||||
On Linux, syft can scan images from a local containers-storage store (e.g. built with Buildah or Podman) when asked explicitly; plain image references never look there:
|
||||
|
||||
```bash
|
||||
syft --from containers-storage localhost/myimage:latest
|
||||
|
||||
# rootless stores must be read from inside the builder's user namespace
|
||||
podman unshare syft --from containers-storage localhost/myimage:latest
|
||||
```
|
||||
|
||||
This is included in the Linux release binaries. When building from source, add `-tags containers_image_openpgp,exclude_graphdriver_btrfs`.
|
||||
|
||||
> [!TIP]
|
||||
> **Check out the [Getting Started guide](https://oss.anchore.com/docs/guides/sbom/getting-started/)** to explore all of the capabilities and features.
|
||||
|
||||
@@ -1,25 +0,0 @@
|
||||
package cli
|
||||
|
||||
import (
|
||||
"io"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"github.com/anchore/clio"
|
||||
"github.com/anchore/syft/internal/redact"
|
||||
)
|
||||
|
||||
func TestCommandCanRunMultipleTimes(t *testing.T) {
|
||||
// https://github.com/anchore/syft/issues/2285
|
||||
t.Cleanup(redact.Reset)
|
||||
|
||||
for i := 0; i < 2; i++ {
|
||||
_, cmd := create(clio.Identification{Name: "syft"}, io.Discard)
|
||||
cmd.SetArgs([]string{"cataloger", "list", "-o", "json"})
|
||||
require.NoError(t, cmd.Execute())
|
||||
|
||||
// the store is released at the end of each run so the next one can set its own
|
||||
require.Nil(t, redact.Get())
|
||||
}
|
||||
}
|
||||
@@ -158,11 +158,6 @@ func (l *backgroundLineReader) read(reader io.Reader, stage *progress.Stage) {
|
||||
|
||||
l.lock.Unlock()
|
||||
}
|
||||
if err := s.Err(); err != nil {
|
||||
log.WithFields("error", err).Debug("unable to read attestation output")
|
||||
// keep draining so the writer never blocks on a full pipe
|
||||
_, _ = io.Copy(io.Discard, reader)
|
||||
}
|
||||
}
|
||||
|
||||
func (l backgroundLineReader) Lines() []string {
|
||||
|
||||
@@ -41,8 +41,6 @@ func AppClioSetupConfig(id clio.Identification, out io.Writer) *clio.SetupConfig
|
||||
stereoscope.SetBus(state.Bus)
|
||||
bus.Set(state.Bus)
|
||||
|
||||
// the redact store is process-global and only cleared when a run ends (see the post-run below), so
|
||||
// setting it while another run is still in flight panics rather than splitting secrets across stores.
|
||||
redact.Set(state.RedactStore)
|
||||
|
||||
log.Set(state.Logger)
|
||||
@@ -50,14 +48,8 @@ func AppClioSetupConfig(id clio.Identification, out io.Writer) *clio.SetupConfig
|
||||
return nil
|
||||
},
|
||||
).
|
||||
WithPostRuns(func(state *clio.State, _ error) {
|
||||
WithPostRuns(func(_ *clio.State, _ error) {
|
||||
stereoscope.Cleanup() //nolint:staticcheck // we don't have access to the image object here
|
||||
|
||||
// release the redact store now that nothing in this run can report anymore, so a later run in the same
|
||||
// process (e.g. syft embedded as a library) can set its own. Only release the store this run set.
|
||||
if redact.Get() == state.RedactStore {
|
||||
redact.Reset()
|
||||
}
|
||||
})
|
||||
return clioCfg
|
||||
}
|
||||
|
||||
@@ -1,32 +0,0 @@
|
||||
package internal
|
||||
|
||||
import (
|
||||
"io"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"github.com/anchore/clio"
|
||||
"github.com/anchore/go-logger/adapter/discard"
|
||||
gologgerredact "github.com/anchore/go-logger/adapter/redact"
|
||||
"github.com/anchore/syft/internal/redact"
|
||||
)
|
||||
|
||||
func TestAppClioSetupConfigInitializerPanicsWhileRunInFlight(t *testing.T) {
|
||||
t.Cleanup(redact.Reset)
|
||||
|
||||
cfg := AppClioSetupConfig(clio.Identification{Name: "syft"}, io.Discard)
|
||||
require.Len(t, cfg.Initializers, 1)
|
||||
|
||||
newState := func() *clio.State {
|
||||
return &clio.State{Logger: discard.New(), RedactStore: gologgerredact.NewStore()}
|
||||
}
|
||||
|
||||
require.NoError(t, cfg.Initializers[0](newState()))
|
||||
|
||||
// a second run starting before the first has finished must not silently replace the store, otherwise
|
||||
// secrets added to the first store would no longer be redacted
|
||||
require.PanicsWithValue(t, "replace existing redaction store (probably unintentional)", func() {
|
||||
_ = cfg.Initializers[0](newState())
|
||||
})
|
||||
}
|
||||
@@ -349,8 +349,8 @@ func renderCatalogerInfoTable(_ *capabilities.Document, catalogers []capabilitie
|
||||
})),
|
||||
tablewriter.WithConfig(tablewriter.Config{
|
||||
Row: tw.CellConfig{
|
||||
Merging: tw.CellMerging{Mode: tw.MergeHierarchical},
|
||||
Alignment: tw.CellAlignment{Global: tw.AlignLeft},
|
||||
Formatting: tw.CellFormatting{MergeMode: tw.MergeHierarchical},
|
||||
Alignment: tw.CellAlignment{Global: tw.AlignLeft},
|
||||
},
|
||||
}),
|
||||
)
|
||||
|
||||
@@ -1,34 +1,24 @@
|
||||
package commands
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
|
||||
"github.com/spf13/cobra"
|
||||
|
||||
"github.com/anchore/clio"
|
||||
"github.com/anchore/go-homedir"
|
||||
"github.com/anchore/syft/cmd/syft/internal/options"
|
||||
"github.com/anchore/syft/cmd/syft/internal/ui"
|
||||
"github.com/anchore/syft/internal"
|
||||
"github.com/anchore/syft/internal/bus"
|
||||
"github.com/anchore/syft/internal/log"
|
||||
"github.com/anchore/syft/internal/spillbuf"
|
||||
"github.com/anchore/syft/internal/tmpdir"
|
||||
"github.com/anchore/syft/syft/format"
|
||||
"github.com/anchore/syft/syft/format/syftjson"
|
||||
"github.com/anchore/syft/syft/sbom"
|
||||
)
|
||||
|
||||
const (
|
||||
convertExample = ` {{.appName}} {{.command}} img.syft.json -o spdx-json convert a syft SBOM to spdx-json, output goes to stdout
|
||||
{{.appName}} {{.command}} img.syft.json -o cyclonedx-json=img.cdx.json convert a syft SBOM to CycloneDX, output is written to the file "img.cdx.json"
|
||||
{{.appName}} {{.command}} - -o spdx-json convert an SBOM from STDIN to spdx-json
|
||||
SYFT_CONVERT_PASSTHROUGH_EXACT_FORMAT=true {{.appName}} {{.command}} img.syft.json -o syft-json=out.syft.json copy the input to "out.syft.json" unchanged if it is already syft-json at the current schema version
|
||||
`
|
||||
)
|
||||
|
||||
@@ -36,7 +26,6 @@ type ConvertOptions struct {
|
||||
options.Config `yaml:",inline" mapstructure:",squash"`
|
||||
options.Output `yaml:",inline" mapstructure:",squash"`
|
||||
options.UpdateCheck `yaml:",inline" mapstructure:",squash"`
|
||||
Convert options.Convert `yaml:"convert" json:"convert" mapstructure:"convert"`
|
||||
}
|
||||
|
||||
func Convert(app clio.Application) *cobra.Command {
|
||||
@@ -45,7 +34,6 @@ func Convert(app clio.Application) *cobra.Command {
|
||||
opts := &ConvertOptions{
|
||||
UpdateCheck: options.DefaultUpdateCheck(),
|
||||
Output: options.DefaultOutput(),
|
||||
Convert: options.DefaultConvert(),
|
||||
}
|
||||
|
||||
return app.SetupCommand(&cobra.Command{
|
||||
@@ -58,11 +46,11 @@ func Convert(app clio.Application) *cobra.Command {
|
||||
}),
|
||||
Args: validateConvertArgs,
|
||||
PreRunE: applicationUpdateCheck(id, &opts.UpdateCheck),
|
||||
RunE: func(cmd *cobra.Command, args []string) error {
|
||||
RunE: func(_ *cobra.Command, args []string) error {
|
||||
restoreStdout := ui.CaptureStdoutToTraceLog()
|
||||
defer restoreStdout()
|
||||
|
||||
return RunConvert(cmd.Context(), opts, args[0])
|
||||
return RunConvert(opts, args[0])
|
||||
},
|
||||
}, opts)
|
||||
}
|
||||
@@ -71,67 +59,30 @@ func validateConvertArgs(cmd *cobra.Command, args []string) error {
|
||||
return validateArgs(cmd, args, "an SBOM argument is required")
|
||||
}
|
||||
|
||||
func RunConvert(ctx context.Context, opts *ConvertOptions, userInput string) error {
|
||||
func RunConvert(opts *ConvertOptions, userInput string) error {
|
||||
log.Warn("convert is an experimental feature, run `syft convert -h` for help")
|
||||
|
||||
// a temp dir is only needed when STDIN is a pipe and has to be spilled to disk; it is created lazily on
|
||||
// first use (under the user's TMPDIR), so a file input never touches it. Mirrors syft.CreateSBOM, which
|
||||
// also honors a TempDir already attached to the context.
|
||||
td := tmpdir.FromContext(ctx)
|
||||
if td == nil {
|
||||
_, td = tmpdir.Root(ctx, "syft-convert")
|
||||
defer func() {
|
||||
_ = td.Cleanup()
|
||||
}()
|
||||
}
|
||||
|
||||
reader, inputInfo, err := openConvertInput(userInput, td)
|
||||
writer, err := opts.SBOMWriter()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer func() {
|
||||
_ = reader.Close()
|
||||
}()
|
||||
|
||||
outputs := opts.Outputs
|
||||
var unchanged []string
|
||||
if opts.Convert.PassthroughExactFormat {
|
||||
log.Warn("convert.passthrough-exact-format is enabled: syft-json input that already exactly matches a requested output format and schema version will be copied to that output unchanged (this does not apply to any other input format)")
|
||||
var reader io.ReadSeekCloser
|
||||
|
||||
unchanged, outputs, err = partitionOutputsBySourceFormat(opts.Output, reader)
|
||||
if userInput == "-" {
|
||||
// though os.Stdin is an os.File, it does not support seeking
|
||||
// you will get errors such as "seek /dev/stdin: illegal seek".
|
||||
// We need to buffer what we read.
|
||||
reader = internal.NewBufferedSeeker(os.Stdin)
|
||||
} else {
|
||||
f, err := os.Open(userInput)
|
||||
if err != nil {
|
||||
return err
|
||||
return fmt.Errorf("failed to open SBOM file: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
// resolve (and validate) every output that needs converting before writing anything, so that
|
||||
// a bad -o value fails the command without leaving partial results behind
|
||||
var writer sbom.Writer
|
||||
if len(outputs) > 0 {
|
||||
outputOpts := opts.Output
|
||||
outputOpts.Outputs = outputs
|
||||
writer, err = outputOpts.SBOMWriter()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
// output files are opened up front; release them if we bail out before writing
|
||||
if c, ok := writer.(io.Closer); ok {
|
||||
defer func() { _ = c.Close() }()
|
||||
}
|
||||
}
|
||||
|
||||
for _, output := range unchanged {
|
||||
if err := writeUnchanged(output, opts.LegacyFile, inputInfo, reader); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
if writer == nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
if _, err := reader.Seek(0, io.SeekStart); err != nil {
|
||||
return fmt.Errorf("unable to seek to start of SBOM: %w", err)
|
||||
defer func() {
|
||||
_ = f.Close()
|
||||
}()
|
||||
reader = f
|
||||
}
|
||||
|
||||
s, _, _, err := format.Decode(reader)
|
||||
@@ -149,171 +100,3 @@ func RunConvert(ctx context.Context, opts *ConvertOptions, userInput string) err
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// openConvertInput opens the SBOM document at the given path, or STDIN when "-". The returned reader is seekable so
|
||||
// the document can be identified, copied, and decoded in turn without being loaded into memory. The returned
|
||||
// os.FileInfo describes the file the document is being read from, so that writing back over it can be recognized;
|
||||
// it is nil when the input is not a file (a pipe or a terminal on STDIN).
|
||||
func openConvertInput(userInput string, td *tmpdir.TempDir) (io.ReadSeekCloser, os.FileInfo, error) {
|
||||
if userInput == "-" {
|
||||
return openStdin(td)
|
||||
}
|
||||
|
||||
f, err := os.Open(userInput)
|
||||
if err != nil {
|
||||
return nil, nil, fmt.Errorf("failed to open SBOM file: %w", err)
|
||||
}
|
||||
|
||||
info, err := f.Stat()
|
||||
if err != nil {
|
||||
// the document is readable, we just cannot recognize it as an output destination
|
||||
log.WithFields("error", err, "path", userInput).Debug("unable to stat SBOM file")
|
||||
}
|
||||
return f, info, nil
|
||||
}
|
||||
|
||||
// openStdin returns a seekable view of STDIN. When STDIN is a regular file (such as a shell redirect) it is read in
|
||||
// place. A pipe or terminal cannot seek (you will get errors such as "seek /dev/stdin: illegal seek"), so its
|
||||
// contents are drained into a spill buffer: a small head stays in memory and everything beyond it goes to a temp
|
||||
// file under td, so an arbitrarily large piped SBOM never has to be held in memory to be made seekable.
|
||||
func openStdin(td *tmpdir.TempDir) (io.ReadSeekCloser, os.FileInfo, error) {
|
||||
if info, err := os.Stdin.Stat(); err == nil && info.Mode().IsRegular() {
|
||||
if start, err := os.Stdin.Seek(0, io.SeekCurrent); err == nil {
|
||||
// a redirect reads the file in place, so it is just as much an input file as a path argument is
|
||||
return readSeekNopCloser{io.NewSectionReader(os.Stdin, start, info.Size()-start)}, info, nil
|
||||
}
|
||||
}
|
||||
|
||||
buf := spillbuf.New(td)
|
||||
if _, err := io.Copy(io.NewOffsetWriter(buf, 0), os.Stdin); err != nil {
|
||||
_ = buf.Close()
|
||||
return nil, nil, fmt.Errorf("failed to read SBOM from STDIN: %w", err)
|
||||
}
|
||||
|
||||
return &spilledReader{
|
||||
SectionReader: io.NewSectionReader(buf, 0, buf.Size()),
|
||||
buf: buf,
|
||||
}, nil, nil
|
||||
}
|
||||
|
||||
type readSeekNopCloser struct {
|
||||
io.ReadSeeker
|
||||
}
|
||||
|
||||
func (readSeekNopCloser) Close() error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// spilledReader is a seekable view over a spill buffer; closing it releases the buffer's memory and spill file.
|
||||
type spilledReader struct {
|
||||
*io.SectionReader
|
||||
buf *spillbuf.Buffer
|
||||
}
|
||||
|
||||
func (r *spilledReader) Close() error {
|
||||
return r.buf.Close()
|
||||
}
|
||||
|
||||
// partitionOutputsBySourceFormat splits the requested outputs (in "<format>[@<version>][=<path>]" form) into those
|
||||
// that exactly match the format and schema version of a syft-json input document (which can be written out
|
||||
// unchanged) and those that still require conversion. Input in any other format is always converted. Outputs that
|
||||
// cannot be resolved to an encoder are left for the writer to report on.
|
||||
func partitionOutputsBySourceFormat(output options.Output, reader io.ReadSeeker) (unchanged []string, toConvert []string, err error) {
|
||||
if _, err := reader.Seek(0, io.SeekStart); err != nil {
|
||||
return nil, nil, fmt.Errorf("unable to seek to start of SBOM: %w", err)
|
||||
}
|
||||
|
||||
id, version := format.Identify(reader)
|
||||
if id == "" || version == "" {
|
||||
// let decoding surface the appropriate error
|
||||
return nil, output.Outputs, nil
|
||||
}
|
||||
|
||||
if id != syftjson.ID {
|
||||
log.WithFields("format", id, "version", version).Debug("input is not syft-json, passthrough does not apply")
|
||||
return nil, output.Outputs, nil
|
||||
}
|
||||
|
||||
encoders, err := output.Encoders()
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
collection := format.NewEncoderCollection(encoders...)
|
||||
|
||||
for _, out := range output.Outputs {
|
||||
name, _, _ := strings.Cut(strings.TrimSpace(out), "=")
|
||||
enc := collection.GetByString(name)
|
||||
if enc != nil && enc.ID() == id && enc.Version() == version {
|
||||
log.WithFields("format", id, "version", version, "output", out).Warn("input already exactly matches the requested output format and schema version, copying it unchanged instead of converting it")
|
||||
unchanged = append(unchanged, out)
|
||||
continue
|
||||
}
|
||||
toConvert = append(toConvert, out)
|
||||
}
|
||||
|
||||
return unchanged, toConvert, nil
|
||||
}
|
||||
|
||||
// writeUnchanged copies the SBOM document as-is to the destination described by the given "<format>[=<path>]"
|
||||
// output value, falling back to the (deprecated) --file path, and finally to the report bus (STDOUT). When the
|
||||
// destination is the input file itself there is nothing to do.
|
||||
func writeUnchanged(output string, defaultFile string, inputInfo os.FileInfo, reader io.ReadSeeker) error {
|
||||
if _, err := reader.Seek(0, io.SeekStart); err != nil {
|
||||
return fmt.Errorf("unable to seek to start of SBOM: %w", err)
|
||||
}
|
||||
|
||||
_, path, _ := strings.Cut(strings.TrimSpace(output), "=")
|
||||
if path == "" {
|
||||
path = defaultFile
|
||||
}
|
||||
|
||||
if path == "" {
|
||||
// the report bus only carries strings, so STDOUT output must be materialized in memory
|
||||
content, err := io.ReadAll(reader) //nolint:gocritic // the whole SBOM is the report; there is no meaningful bound
|
||||
if err != nil {
|
||||
return fmt.Errorf("unable to read SBOM: %w", err)
|
||||
}
|
||||
bus.Report(string(content))
|
||||
return nil
|
||||
}
|
||||
|
||||
expandedPath, err := homedir.Expand(path)
|
||||
if err != nil {
|
||||
log.Warnf("could not expand given writer output path=%q: %w", path, err)
|
||||
expandedPath = path
|
||||
}
|
||||
|
||||
if isSameFile(inputInfo, expandedPath) {
|
||||
log.WithFields("path", expandedPath).Info("output is the input file, leaving it as-is")
|
||||
return nil
|
||||
}
|
||||
|
||||
if err := os.MkdirAll(filepath.Dir(expandedPath), 0755); err != nil {
|
||||
return fmt.Errorf("unable to create report directory: %w", err)
|
||||
}
|
||||
|
||||
f, err := os.OpenFile(expandedPath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0644)
|
||||
if err != nil {
|
||||
return fmt.Errorf("unable to create report file: %w", err)
|
||||
}
|
||||
|
||||
if _, err := io.Copy(f, reader); err != nil {
|
||||
_ = f.Close()
|
||||
return fmt.Errorf("unable to write report file: %w", err)
|
||||
}
|
||||
|
||||
return f.Close()
|
||||
}
|
||||
|
||||
// isSameFile reports whether the given path names the file the input is being read from. Comparing the input's
|
||||
// os.FileInfo rather than its path covers a shell redirect onto STDIN, which has no path to compare.
|
||||
func isSameFile(inputInfo os.FileInfo, path string) bool {
|
||||
if inputInfo == nil {
|
||||
return false
|
||||
}
|
||||
pathInfo, err := os.Stat(path)
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
return os.SameFile(inputInfo, pathInfo)
|
||||
}
|
||||
|
||||
@@ -1,448 +0,0 @@
|
||||
package commands
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
"github.com/wagoodman/go-partybus"
|
||||
|
||||
"github.com/anchore/syft/cmd/syft/internal/options"
|
||||
"github.com/anchore/syft/internal/bus"
|
||||
"github.com/anchore/syft/internal/spillbuf"
|
||||
"github.com/anchore/syft/syft/event"
|
||||
"github.com/anchore/syft/syft/file"
|
||||
"github.com/anchore/syft/syft/format"
|
||||
"github.com/anchore/syft/syft/format/cyclonedxjson"
|
||||
"github.com/anchore/syft/syft/format/spdxjson"
|
||||
"github.com/anchore/syft/syft/format/syftjson"
|
||||
"github.com/anchore/syft/syft/pkg"
|
||||
"github.com/anchore/syft/syft/sbom"
|
||||
"github.com/anchore/syft/syft/source"
|
||||
)
|
||||
|
||||
func convertTestSBOM() sbom.SBOM {
|
||||
catalog := pkg.NewCollection()
|
||||
catalog.Add(pkg.Package{
|
||||
Name: "musl-utils",
|
||||
Version: "1.2.3-r4",
|
||||
Type: pkg.ApkPkg,
|
||||
FoundBy: "apk-db-cataloger",
|
||||
Locations: file.NewLocationSet(
|
||||
file.NewLocation("/lib/apk/db/installed"),
|
||||
),
|
||||
PURL: "pkg:apk/alpine/musl-utils@1.2.3-r4",
|
||||
})
|
||||
|
||||
return sbom.SBOM{
|
||||
Artifacts: sbom.Artifacts{
|
||||
Packages: catalog,
|
||||
},
|
||||
Source: source.Description{
|
||||
ID: "some-id",
|
||||
Name: "some-dir",
|
||||
Version: "latest",
|
||||
Metadata: source.DirectoryMetadata{
|
||||
Path: "/some/path",
|
||||
},
|
||||
},
|
||||
Descriptor: sbom.Descriptor{
|
||||
Name: "syft",
|
||||
Version: "v0.0.0-test",
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func mustConvertEncoder(enc sbom.FormatEncoder, err error) sbom.FormatEncoder {
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
return enc
|
||||
}
|
||||
|
||||
func encodeConvertTestSBOM(t *testing.T, enc sbom.FormatEncoder) []byte {
|
||||
buf := &bytes.Buffer{}
|
||||
require.NoError(t, enc.Encode(buf, convertTestSBOM()))
|
||||
return buf.Bytes()
|
||||
}
|
||||
|
||||
// prettySyftJSON returns the test SBOM as pretty-printed syft-json. Pretty input makes a verbatim copy
|
||||
// distinguishable from a re-encode, since re-encoding always produces compact output here.
|
||||
func prettySyftJSON(t *testing.T) []byte {
|
||||
cfg := syftjson.DefaultEncoderConfig()
|
||||
cfg.Pretty = true
|
||||
return encodeConvertTestSBOM(t, mustConvertEncoder(syftjson.NewFormatEncoderWithConfig(cfg)))
|
||||
}
|
||||
|
||||
func Test_partitionOutputsBySourceFormat(t *testing.T) {
|
||||
syftEncoder := mustConvertEncoder(syftjson.NewFormatEncoderWithConfig(syftjson.DefaultEncoderConfig()))
|
||||
syftVersion := syftEncoder.Version()
|
||||
syftContent := encodeConvertTestSBOM(t, syftEncoder)
|
||||
|
||||
// the same document claiming an older schema version
|
||||
olderSyftContent := bytes.ReplaceAll(syftContent, []byte(`"version":"`+syftVersion+`"`), []byte(`"version":"16.0.0"`))
|
||||
require.NotEqual(t, syftContent, olderSyftContent)
|
||||
|
||||
cdxContent := encodeConvertTestSBOM(t, mustConvertEncoder(cyclonedxjson.NewFormatEncoderWithConfig(cyclonedxjson.DefaultEncoderConfig())))
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
content []byte
|
||||
outputs []string
|
||||
wantUnchanged []string
|
||||
wantToConvert []string
|
||||
}{
|
||||
{
|
||||
name: "syft-json matches the bare format name",
|
||||
content: syftContent,
|
||||
outputs: []string{"syft-json"},
|
||||
wantUnchanged: []string{"syft-json"},
|
||||
},
|
||||
{
|
||||
name: "syft-json matches its alias",
|
||||
content: syftContent,
|
||||
outputs: []string{"json"},
|
||||
wantUnchanged: []string{"json"},
|
||||
},
|
||||
{
|
||||
name: "syft-json matches an explicit version and file path",
|
||||
content: syftContent,
|
||||
outputs: []string{"syft-json@" + syftVersion + "=out/some.syft.json"},
|
||||
wantUnchanged: []string{"syft-json@" + syftVersion + "=out/some.syft.json"},
|
||||
},
|
||||
{
|
||||
name: "syft-json at an older schema version requires conversion",
|
||||
content: olderSyftContent,
|
||||
outputs: []string{"syft-json"},
|
||||
wantToConvert: []string{"syft-json"},
|
||||
},
|
||||
{
|
||||
name: "a different output format requires conversion",
|
||||
content: syftContent,
|
||||
outputs: []string{"spdx-json"},
|
||||
wantToConvert: []string{"spdx-json"},
|
||||
},
|
||||
{
|
||||
name: "mixed outputs are partitioned",
|
||||
content: syftContent,
|
||||
outputs: []string{"syft-json=a.json", "spdx-json=b.json", "cyclonedx-xml"},
|
||||
wantUnchanged: []string{"syft-json=a.json"},
|
||||
wantToConvert: []string{"spdx-json=b.json", "cyclonedx-xml"},
|
||||
},
|
||||
{
|
||||
name: "unknown format is left for the writer to report",
|
||||
content: syftContent,
|
||||
outputs: []string{"bogus"},
|
||||
wantToConvert: []string{"bogus"},
|
||||
},
|
||||
{
|
||||
name: "non syft-json input is always converted, even when it matches the output",
|
||||
content: cdxContent,
|
||||
outputs: []string{"cyclonedx-json"},
|
||||
wantToConvert: []string{"cyclonedx-json"},
|
||||
},
|
||||
{
|
||||
name: "unidentifiable content is left for the decoder to report",
|
||||
content: []byte("definitely not an sbom"),
|
||||
outputs: []string{"syft-json"},
|
||||
wantToConvert: []string{"syft-json"},
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
output := options.DefaultOutput()
|
||||
output.Outputs = tt.outputs
|
||||
|
||||
unchanged, toConvert, err := partitionOutputsBySourceFormat(output, bytes.NewReader(tt.content))
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, tt.wantUnchanged, unchanged)
|
||||
assert.Equal(t, tt.wantToConvert, toConvert)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func Test_RunConvert_passthroughExactFormat(t *testing.T) {
|
||||
syftContent := prettySyftJSON(t)
|
||||
|
||||
newOpts := func(passthrough bool, outputs ...string) *ConvertOptions {
|
||||
opts := &ConvertOptions{
|
||||
Output: options.DefaultOutput(),
|
||||
Convert: options.Convert{
|
||||
PassthroughExactFormat: passthrough,
|
||||
},
|
||||
}
|
||||
opts.Outputs = outputs
|
||||
return opts
|
||||
}
|
||||
|
||||
writeInput := func(t *testing.T, content []byte) string {
|
||||
input := filepath.Join(t.TempDir(), "input.json")
|
||||
require.NoError(t, os.WriteFile(input, content, 0600))
|
||||
return input
|
||||
}
|
||||
|
||||
t.Run("matching output is copied unchanged while others are converted", func(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
syftOut := filepath.Join(dir, "out.syft.json")
|
||||
spdxOut := filepath.Join(dir, "out.spdx.json")
|
||||
|
||||
opts := newOpts(true, "syft-json="+syftOut, "spdx-json="+spdxOut)
|
||||
require.NoError(t, RunConvert(context.Background(), opts, writeInput(t, syftContent)))
|
||||
|
||||
got, err := os.ReadFile(syftOut)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, syftContent, got)
|
||||
|
||||
spdx, err := os.ReadFile(spdxOut)
|
||||
require.NoError(t, err)
|
||||
id, _ := format.Identify(bytes.NewReader(spdx))
|
||||
assert.Equal(t, spdxjson.ID, id)
|
||||
})
|
||||
|
||||
t.Run("STDIN input is copied unchanged while others are converted", func(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
syftOut := filepath.Join(dir, "out.syft.json")
|
||||
spdxOut := filepath.Join(dir, "out.spdx.json")
|
||||
|
||||
stdin, err := os.Open(writeInput(t, syftContent))
|
||||
require.NoError(t, err)
|
||||
defer stdin.Close()
|
||||
|
||||
originalStdin := os.Stdin
|
||||
os.Stdin = stdin
|
||||
t.Cleanup(func() { os.Stdin = originalStdin })
|
||||
|
||||
opts := newOpts(true, "syft-json="+syftOut, "spdx-json="+spdxOut)
|
||||
require.NoError(t, RunConvert(context.Background(), opts, "-"))
|
||||
|
||||
got, err := os.ReadFile(syftOut)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, syftContent, got)
|
||||
|
||||
spdx, err := os.ReadFile(spdxOut)
|
||||
require.NoError(t, err)
|
||||
id, _ := format.Identify(bytes.NewReader(spdx))
|
||||
assert.Equal(t, spdxjson.ID, id)
|
||||
})
|
||||
|
||||
t.Run("piped STDIN input is copied unchanged while others are converted", func(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
syftOut := filepath.Join(dir, "out.syft.json")
|
||||
spdxOut := filepath.Join(dir, "out.spdx.json")
|
||||
|
||||
pipeReader, pipeWriter, err := os.Pipe()
|
||||
require.NoError(t, err)
|
||||
defer pipeReader.Close()
|
||||
go func() {
|
||||
defer pipeWriter.Close()
|
||||
_, _ = pipeWriter.Write(syftContent)
|
||||
}()
|
||||
|
||||
originalStdin := os.Stdin
|
||||
os.Stdin = pipeReader
|
||||
t.Cleanup(func() { os.Stdin = originalStdin })
|
||||
|
||||
opts := newOpts(true, "syft-json="+syftOut, "spdx-json="+spdxOut)
|
||||
require.NoError(t, RunConvert(context.Background(), opts, "-"))
|
||||
|
||||
got, err := os.ReadFile(syftOut)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, syftContent, got)
|
||||
|
||||
spdx, err := os.ReadFile(spdxOut)
|
||||
require.NoError(t, err)
|
||||
id, _ := format.Identify(bytes.NewReader(spdx))
|
||||
assert.Equal(t, spdxjson.ID, id)
|
||||
})
|
||||
|
||||
t.Run("without the option the matching output is re-encoded", func(t *testing.T) {
|
||||
syftOut := filepath.Join(t.TempDir(), "out.syft.json")
|
||||
|
||||
opts := newOpts(false, "syft-json="+syftOut)
|
||||
require.NoError(t, RunConvert(context.Background(), opts, writeInput(t, syftContent)))
|
||||
|
||||
got, err := os.ReadFile(syftOut)
|
||||
require.NoError(t, err)
|
||||
id, _ := format.Identify(bytes.NewReader(got))
|
||||
assert.Equal(t, syftjson.ID, id)
|
||||
// the input was pretty-printed and the encoder is not configured to be, so a re-encode cannot reproduce it
|
||||
assert.NotEqual(t, syftContent, got)
|
||||
})
|
||||
|
||||
t.Run("non syft-json input is re-encoded even when it matches the output", func(t *testing.T) {
|
||||
cdxContent := encodeConvertTestSBOM(t, mustConvertEncoder(cyclonedxjson.NewFormatEncoderWithConfig(cyclonedxjson.DefaultEncoderConfig())))
|
||||
cdxOut := filepath.Join(t.TempDir(), "out.cdx.json")
|
||||
|
||||
opts := newOpts(true, "cyclonedx-json="+cdxOut)
|
||||
require.NoError(t, RunConvert(context.Background(), opts, writeInput(t, cdxContent)))
|
||||
|
||||
got, err := os.ReadFile(cdxOut)
|
||||
require.NoError(t, err)
|
||||
id, _ := format.Identify(bytes.NewReader(got))
|
||||
assert.Equal(t, cyclonedxjson.ID, id)
|
||||
// a fresh serial number is minted on every encode, so a round trip never reproduces the input
|
||||
assert.NotEqual(t, cdxContent, got)
|
||||
})
|
||||
|
||||
t.Run("the deprecated --file path is honored", func(t *testing.T) {
|
||||
syftOut := filepath.Join(t.TempDir(), "out.syft.json")
|
||||
|
||||
opts := newOpts(true, "syft-json")
|
||||
opts.LegacyFile = syftOut
|
||||
require.NoError(t, RunConvert(context.Background(), opts, writeInput(t, syftContent)))
|
||||
|
||||
got, err := os.ReadFile(syftOut)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, syftContent, got)
|
||||
})
|
||||
|
||||
t.Run("writing over the input file leaves it intact", func(t *testing.T) {
|
||||
input := writeInput(t, syftContent)
|
||||
|
||||
opts := newOpts(true, "syft-json="+input)
|
||||
require.NoError(t, RunConvert(context.Background(), opts, input))
|
||||
|
||||
got, err := os.ReadFile(input)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, syftContent, got)
|
||||
})
|
||||
|
||||
t.Run("writing over a redirected STDIN input leaves it intact", func(t *testing.T) {
|
||||
input := writeInput(t, syftContent)
|
||||
|
||||
stdin, err := os.Open(input)
|
||||
require.NoError(t, err)
|
||||
defer stdin.Close()
|
||||
|
||||
originalStdin := os.Stdin
|
||||
os.Stdin = stdin
|
||||
t.Cleanup(func() { os.Stdin = originalStdin })
|
||||
|
||||
opts := newOpts(true, "syft-json="+input)
|
||||
require.NoError(t, RunConvert(context.Background(), opts, "-"))
|
||||
|
||||
got, err := os.ReadFile(input)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, syftContent, got)
|
||||
})
|
||||
|
||||
t.Run("an invalid output fails before anything is written", func(t *testing.T) {
|
||||
syftOut := filepath.Join(t.TempDir(), "out.syft.json")
|
||||
|
||||
opts := newOpts(true, "syft-json="+syftOut, "bogus")
|
||||
require.Error(t, RunConvert(context.Background(), opts, writeInput(t, syftContent)))
|
||||
|
||||
assert.NoFileExists(t, syftOut)
|
||||
})
|
||||
|
||||
t.Run("a missing input file is reported", func(t *testing.T) {
|
||||
syftOut := filepath.Join(t.TempDir(), "out.syft.json")
|
||||
|
||||
opts := newOpts(true, "syft-json="+syftOut)
|
||||
err := RunConvert(context.Background(), opts, filepath.Join(t.TempDir(), "does-not-exist.json"))
|
||||
require.ErrorContains(t, err, "failed to open SBOM file")
|
||||
|
||||
assert.NoFileExists(t, syftOut)
|
||||
})
|
||||
|
||||
t.Run("unrecognized input is left to the decoder to reject", func(t *testing.T) {
|
||||
syftOut := filepath.Join(t.TempDir(), "out.syft.json")
|
||||
|
||||
opts := newOpts(true, "syft-json="+syftOut)
|
||||
err := RunConvert(context.Background(), opts, writeInput(t, []byte("definitely not an sbom")))
|
||||
require.ErrorContains(t, err, "failed to decode SBOM")
|
||||
})
|
||||
}
|
||||
|
||||
// largeSyftJSON returns a pretty-printed syft-json document of at least minBytes, built from many small packages.
|
||||
func largeSyftJSON(t *testing.T, minBytes int64) []byte {
|
||||
catalog := pkg.NewCollection()
|
||||
for i := 0; i < int(minBytes/100); i++ {
|
||||
catalog.Add(pkg.Package{
|
||||
Name: fmt.Sprintf("pkg-%d", i),
|
||||
Version: "1.0.0",
|
||||
Type: pkg.ApkPkg,
|
||||
PURL: fmt.Sprintf("pkg:apk/alpine/pkg-%d@1.0.0", i),
|
||||
})
|
||||
}
|
||||
s := convertTestSBOM()
|
||||
s.Artifacts.Packages = catalog
|
||||
|
||||
cfg := syftjson.DefaultEncoderConfig()
|
||||
cfg.Pretty = true
|
||||
enc := mustConvertEncoder(syftjson.NewFormatEncoderWithConfig(cfg))
|
||||
|
||||
buf := &bytes.Buffer{}
|
||||
require.NoError(t, enc.Encode(buf, s))
|
||||
require.Greater(t, int64(buf.Len()), minBytes)
|
||||
return buf.Bytes()
|
||||
}
|
||||
|
||||
func Test_RunConvert_pipedInputLargerThanMemoryLimitSpillsToDisk(t *testing.T) {
|
||||
// a piped document bigger than the spill buffer's in-memory tier is written to a temp file under TMPDIR;
|
||||
// the conversion must identify, copy and clean up from there rather than holding the document in memory
|
||||
dir := t.TempDir()
|
||||
syftOut := filepath.Join(dir, "out.syft.json")
|
||||
spillRoot := filepath.Join(dir, "spill")
|
||||
require.NoError(t, os.Mkdir(spillRoot, 0o755))
|
||||
t.Setenv("TMPDIR", spillRoot)
|
||||
|
||||
content := largeSyftJSON(t, 2*spillbuf.DefaultMemLimit)
|
||||
|
||||
pipeReader, pipeWriter, err := os.Pipe()
|
||||
require.NoError(t, err)
|
||||
defer pipeReader.Close()
|
||||
go func() {
|
||||
defer pipeWriter.Close()
|
||||
_, _ = pipeWriter.Write(content)
|
||||
}()
|
||||
|
||||
originalStdin := os.Stdin
|
||||
os.Stdin = pipeReader
|
||||
t.Cleanup(func() { os.Stdin = originalStdin })
|
||||
|
||||
opts := &ConvertOptions{
|
||||
Output: options.DefaultOutput(),
|
||||
Convert: options.Convert{
|
||||
PassthroughExactFormat: true,
|
||||
},
|
||||
}
|
||||
opts.Outputs = []string{"syft-json=" + syftOut}
|
||||
require.NoError(t, RunConvert(context.Background(), opts, "-"))
|
||||
|
||||
got, err := os.ReadFile(syftOut)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, content, got)
|
||||
|
||||
entries, err := os.ReadDir(spillRoot)
|
||||
require.NoError(t, err)
|
||||
assert.Empty(t, entries, "the spill directory must be removed once the conversion finishes")
|
||||
}
|
||||
|
||||
func Test_writeUnchanged_stdout(t *testing.T) {
|
||||
// with no output path the document is published on the report bus, which is how the CLI writes to STDOUT
|
||||
b := partybus.NewBus()
|
||||
subscription := b.Subscribe()
|
||||
bus.Set(b)
|
||||
t.Cleanup(func() {
|
||||
bus.Set(nil)
|
||||
})
|
||||
|
||||
content := []byte(`{"some":"document"}`)
|
||||
require.NoError(t, writeUnchanged("syft-json", "", nil, bytes.NewReader(content)))
|
||||
|
||||
select {
|
||||
case e := <-subscription.Events():
|
||||
assert.Equal(t, event.CLIReport, e.Type)
|
||||
assert.Equal(t, string(content), e.Value)
|
||||
case <-time.After(time.Second):
|
||||
t.Fatal("no report was published")
|
||||
}
|
||||
}
|
||||
@@ -4,7 +4,6 @@ import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"reflect"
|
||||
"strings"
|
||||
@@ -182,10 +181,6 @@ func runScan(ctx context.Context, id clio.Identification, opts *scanOptions, use
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
// output files are opened up front; release them if we bail out before writing
|
||||
if c, ok := writer.(io.Closer); ok {
|
||||
defer func() { _ = c.Close() }()
|
||||
}
|
||||
|
||||
sources := opts.From
|
||||
if len(sources) == 0 {
|
||||
|
||||
@@ -4,7 +4,6 @@ import (
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"runtime"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
@@ -23,10 +22,9 @@ func Test_defaultDir(t *testing.T) {
|
||||
homeDir := filepath.Join(tmpDir, "fake-home")
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
env map[string]string
|
||||
expected string
|
||||
skipOnWindows bool
|
||||
name string
|
||||
env map[string]string
|
||||
expected string
|
||||
}{
|
||||
{
|
||||
name: "no-xdg",
|
||||
@@ -34,8 +32,6 @@ func Test_defaultDir(t *testing.T) {
|
||||
"HOME": homeDir,
|
||||
},
|
||||
expected: homeDir,
|
||||
// xdg falls back to LOCALAPPDATA on windows, not HOME
|
||||
skipOnWindows: true,
|
||||
},
|
||||
{
|
||||
name: "xdg-cache",
|
||||
@@ -63,9 +59,6 @@ func Test_defaultDir(t *testing.T) {
|
||||
|
||||
for _, test := range tests {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
if test.skipOnWindows && runtime.GOOS == "windows" {
|
||||
t.Skip("posix-only cache dir fallback")
|
||||
}
|
||||
restoreCache(t)
|
||||
|
||||
for k, v := range test.env {
|
||||
|
||||
@@ -102,7 +102,6 @@ func (cfg Catalog) ToSBOMConfig(id clio.Identification) *syft.CreateSBOMConfig {
|
||||
WithComplianceConfig(cfg.ToComplianceConfig()).
|
||||
WithUnknownsConfig(cfg.ToUnknownsConfig()).
|
||||
WithSearchConfig(cfg.ToSearchConfig()).
|
||||
WithArchiveConfig(cfg.ToArchiveConfig()).
|
||||
WithPackagesConfig(cfg.ToPackagesConfig()).
|
||||
WithLicenseConfig(cfg.ToLicenseConfig()).
|
||||
WithFilesConfig(cfg.ToFilesConfig()).
|
||||
@@ -119,15 +118,6 @@ func (cfg Catalog) ToSearchConfig() cataloging.SearchConfig {
|
||||
}
|
||||
}
|
||||
|
||||
func (cfg Catalog) ToArchiveConfig() cataloging.ArchiveSearchConfig {
|
||||
return cataloging.DefaultArchiveSearchConfig().
|
||||
WithIncludeIndexedArchives(cfg.Package.SearchIndexedArchives).
|
||||
WithIncludeUnindexedArchives(cfg.Package.SearchUnindexedArchives).
|
||||
WithMaxDepth(cfg.Package.NestedArchiveMaxDepth).
|
||||
WithMaxMemoryBytes(cfg.Package.NestedArchiveMaxMemoryBytes).
|
||||
WithMaxDiskBytes(cfg.Package.NestedArchiveMaxDiskBytes)
|
||||
}
|
||||
|
||||
func (cfg Catalog) ToRelationshipsConfig() cataloging.RelationshipsConfig {
|
||||
return cataloging.RelationshipsConfig{
|
||||
PackageFileOwnership: cfg.Relationships.PackageFileOwnership,
|
||||
@@ -146,7 +136,6 @@ func (cfg Catalog) ToComplianceConfig() cataloging.ComplianceConfig {
|
||||
|
||||
func (cfg Catalog) ToUnknownsConfig() cataloging.UnknownsConfig {
|
||||
return cataloging.UnknownsConfig{
|
||||
RemoveWhenPackagesDefined: cfg.Unknowns.RemoveWhenPackagesDefined,
|
||||
IncludeExecutablesWithoutPackages: cfg.Unknowns.ExecutablesWithoutPackages,
|
||||
IncludeUnexpandedArchives: cfg.Unknowns.UnexpandedArchives,
|
||||
}
|
||||
@@ -180,6 +169,10 @@ func (cfg Catalog) ToLicenseConfig() cataloging.LicenseConfig {
|
||||
}
|
||||
|
||||
func (cfg Catalog) ToPackagesConfig() pkgcataloging.Config {
|
||||
archiveSearch := cataloging.ArchiveSearchConfig{
|
||||
IncludeIndexedArchives: cfg.Package.SearchIndexedArchives,
|
||||
IncludeUnindexedArchives: cfg.Package.SearchUnindexedArchives,
|
||||
}
|
||||
return pkgcataloging.Config{
|
||||
Binary: binary.DefaultClassifierCatalogerConfig(),
|
||||
Cpp: cpp.DefaultCatalogerConfig().
|
||||
@@ -206,8 +199,7 @@ func (cfg Catalog) ToPackagesConfig() pkgcataloging.Config {
|
||||
WithFromLDFlags(cfg.Golang.MainModuleVersion.FromLDFlags),
|
||||
).
|
||||
WithUsePackagesLib(*multiLevelOption(true, enrichmentEnabled(cfg.Enrich, task.Go, task.Golang), cfg.Golang.UsePackagesLib)).
|
||||
WithCaptureSymbols(cfg.Golang.CaptureSymbols).
|
||||
WithCaptureSymbolsModules(cfg.Golang.CaptureSymbolsModules),
|
||||
WithCaptureSymbols(cfg.Golang.CaptureSymbols),
|
||||
JavaScript: javascript.DefaultCatalogerConfig().
|
||||
WithIncludeDevDependencies(*multiLevelOption(false, cfg.JavaScript.IncludeDevDependencies)).
|
||||
WithSearchRemoteLicenses(*multiLevelOption(false, enrichmentEnabled(cfg.Enrich, task.JavaScript, task.Node, task.NPM), cfg.JavaScript.SearchRemoteLicenses)).
|
||||
@@ -226,7 +218,7 @@ func (cfg Catalog) ToPackagesConfig() pkgcataloging.Config {
|
||||
WithMavenLocalRepositoryDir(cfg.Java.MavenLocalRepositoryDir).
|
||||
WithUseNetwork(*multiLevelOption(false, enrichmentEnabled(cfg.Enrich, task.Java, task.Maven), cfg.Java.UseNetwork)).
|
||||
WithMavenBaseURL(cfg.Java.MavenURL).
|
||||
WithMaxParentRecursiveDepth(cfg.Java.MaxParentRecursiveDepth).
|
||||
WithArchiveTraversal(archiveSearch, cfg.Java.MaxParentRecursiveDepth).
|
||||
WithResolveTransitiveDependencies(cfg.Java.ResolveTransitiveDependencies),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,27 +0,0 @@
|
||||
package options
|
||||
|
||||
import (
|
||||
"github.com/anchore/clio"
|
||||
)
|
||||
|
||||
var _ clio.FieldDescriber = (*Convert)(nil)
|
||||
|
||||
// Convert holds the options specific to the convert command. These are deliberately configuration-only (config file
|
||||
// or environment) rather than flags: they change how the command behaves for every invocation in a pipeline and are
|
||||
// not expected to vary from one call to the next.
|
||||
type Convert struct {
|
||||
PassthroughExactFormat bool `yaml:"passthrough-exact-format" json:"passthrough-exact-format" mapstructure:"passthrough-exact-format"`
|
||||
}
|
||||
|
||||
func DefaultConvert() Convert {
|
||||
return Convert{
|
||||
PassthroughExactFormat: false,
|
||||
}
|
||||
}
|
||||
|
||||
func (o *Convert) DescribeFields(descriptions clio.FieldDescriptionSet) {
|
||||
descriptions.Add(&o.PassthroughExactFormat, `when a syft-json input SBOM already exactly matches the requested output format and schema version, copy it to that output unchanged instead of decoding and re-encoding it.
|
||||
Note: this only applies to syft-json input, and only when the document's schema version is identical to the schema version this version of syft produces. Any other input is always converted.
|
||||
Note: output written this way is a byte-for-byte copy of the input, so format options (such as 'format.pretty' or 'format.json.legacy') are not applied to it.
|
||||
Note: to keep memory use minimal on large documents, give the input as a file path (or a shell redirect) and write the output with '-o <format>=<path>'. Piped input, or output to STDOUT, requires the whole document to be held in memory.`)
|
||||
}
|
||||
@@ -4,7 +4,6 @@ import (
|
||||
"strings"
|
||||
|
||||
"github.com/anchore/clio"
|
||||
"github.com/anchore/syft/internal/log"
|
||||
"github.com/anchore/syft/syft/cataloging"
|
||||
"github.com/anchore/syft/syft/pkg/cataloger/golang"
|
||||
)
|
||||
@@ -20,7 +19,6 @@ type golangConfig struct {
|
||||
MainModuleVersion golangMainModuleVersionConfig `json:"main-module-version" yaml:"main-module-version" mapstructure:"main-module-version"`
|
||||
UsePackagesLib *bool `json:"use-packages-lib" yaml:"use-packages-lib" mapstructure:"use-packages-lib"`
|
||||
CaptureSymbols cataloging.SymbolScope `json:"capture-symbols" yaml:"capture-symbols" mapstructure:"capture-symbols"`
|
||||
CaptureSymbolsModules []string `json:"capture-symbols-modules" yaml:"capture-symbols-modules" mapstructure:"capture-symbols-modules"`
|
||||
}
|
||||
|
||||
var _ interface {
|
||||
@@ -44,16 +42,8 @@ if unset this defaults to $GONOPROXY`)
|
||||
always show (devel) as the version. Use these options to control heuristics to guess
|
||||
a more accurate version from the binary.`)
|
||||
descriptions.Add(&o.UsePackagesLib, `use the golang.org/x/tools/go/packages library, which executes golang tooling found on the path in addition to potential network access to get the most accurate results`)
|
||||
// note: descriptions must be static string literals; the app config discovery that generates the
|
||||
// capability docs reads them straight out of the AST
|
||||
descriptions.Add(&o.CaptureSymbols, `capture function symbols from the binary symbol table (pclntab). valid values are:
|
||||
"none" (disabled), "stdlib" (only the synthetic stdlib package), "extended-stdlib" (stdlib plus every
|
||||
module under golang.org/x/), and "all" (all module packages plus stdlib)`)
|
||||
descriptions.Add(&o.CaptureSymbolsModules, `glob patterns matched against go module paths (e.g. github.com/klauspost/**) that should have symbols
|
||||
captured in addition to whatever capture-symbols selects. ** crosses path separators, * does not.
|
||||
a trailing major version suffix is ignored when matching, so github.com/foo/* covers github.com/foo/bar/v2;
|
||||
spelling a suffix out in the pattern selects only that major version.
|
||||
this can only widen the selection, never narrow it, and is inert when capture-symbols is none`)
|
||||
"none" (disabled), "stdlib" (only the synthetic stdlib package), and "all" (all module packages plus stdlib)`)
|
||||
descriptions.Add(&o.MainModuleVersion.FromLDFlags, `look for LD flags that appear to be setting a version (e.g. -X main.version=1.0.0)`)
|
||||
descriptions.Add(&o.MainModuleVersion.FromBuildSettings, `use the build settings (e.g. vcs.version & vcs.time) to craft a v0 pseudo version
|
||||
(e.g. v0.0.0-20220308212642-53e6d0aaf6fb) when a more accurate version cannot be found otherwise`)
|
||||
@@ -61,24 +51,7 @@ this can only widen the selection, never narrow it, and is inert when capture-sy
|
||||
}
|
||||
|
||||
func (o *golangConfig) PostLoad() error {
|
||||
raw := strings.TrimSpace(string(o.CaptureSymbols))
|
||||
o.CaptureSymbols = o.CaptureSymbols.Parse()
|
||||
|
||||
// an unrecognized value still resolves to "none", but say so rather than silently capturing nothing.
|
||||
// stay quiet for unset and an explicit "none", which is the default and would otherwise warn on nearly
|
||||
// every scan.
|
||||
if o.CaptureSymbols == cataloging.SymbolScopeNone && raw != "" && !strings.EqualFold(raw, string(cataloging.SymbolScopeNone)) {
|
||||
log.Warnf("unknown golang.capture-symbols value %q, defaulting to %q (valid values: none, stdlib, extended-stdlib, all)", raw, cataloging.SymbolScopeNone)
|
||||
}
|
||||
|
||||
// trim only, deliberately not Flatten: viper already splits a comma-separated scalar (env var or a bare
|
||||
// yaml string) into a slice before this point, and there is no CLI flag feeding this key. The one thing
|
||||
// Flatten would add is splitting commas *inside* a list entry, which silently breaks doublestar brace
|
||||
// alternation like github.com/{foo,bar}/**. Viper's split does not trim, so that part is still needed.
|
||||
for i, pattern := range o.CaptureSymbolsModules {
|
||||
o.CaptureSymbolsModules[i] = strings.TrimSpace(pattern)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -103,8 +76,7 @@ func defaultGolangConfig() golangConfig {
|
||||
FromContents: def.MainModuleVersion.FromContents,
|
||||
FromBuildSettings: def.MainModuleVersion.FromBuildSettings,
|
||||
},
|
||||
UsePackagesLib: nil, // this defaults to true, which is the API default
|
||||
CaptureSymbols: def.CaptureSymbols,
|
||||
CaptureSymbolsModules: def.CaptureSymbolsModules,
|
||||
UsePackagesLib: nil, // this defaults to true, which is the API default
|
||||
CaptureSymbols: def.CaptureSymbols,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -10,11 +10,10 @@ import (
|
||||
|
||||
func Test_golangConfig_PostLoad(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
cfg golangConfig
|
||||
expected cataloging.SymbolScope
|
||||
expectedModules []string
|
||||
wantErr assert.ErrorAssertionFunc
|
||||
name string
|
||||
cfg golangConfig
|
||||
expected cataloging.SymbolScope
|
||||
wantErr assert.ErrorAssertionFunc
|
||||
}{
|
||||
{
|
||||
name: "normalize all",
|
||||
@@ -26,32 +25,6 @@ func Test_golangConfig_PostLoad(t *testing.T) {
|
||||
cfg: golangConfig{CaptureSymbols: "stdlib"},
|
||||
expected: cataloging.SymbolScopeStdlib,
|
||||
},
|
||||
{
|
||||
name: "normalize extended-stdlib",
|
||||
cfg: golangConfig{CaptureSymbols: " Extended-Stdlib "},
|
||||
expected: cataloging.SymbolScopeExtendedStdlib,
|
||||
},
|
||||
{
|
||||
name: "module patterns keep embedded commas",
|
||||
cfg: golangConfig{
|
||||
CaptureSymbols: "stdlib",
|
||||
// brace alternation contains a comma; splitting on it would corrupt the pattern
|
||||
CaptureSymbolsModules: []string{"github.com/{foo,bar}/**", "golang.org/x/**"},
|
||||
},
|
||||
expected: cataloging.SymbolScopeStdlib,
|
||||
expectedModules: []string{"github.com/{foo,bar}/**", "golang.org/x/**"},
|
||||
},
|
||||
{
|
||||
// viper splits a comma-separated scalar (env var or bare yaml string) but does not trim,
|
||||
// so a leading space would otherwise survive into a pattern that silently matches nothing
|
||||
name: "module patterns are trimmed",
|
||||
cfg: golangConfig{
|
||||
CaptureSymbols: "stdlib",
|
||||
CaptureSymbolsModules: []string{"golang.org/x/**", " github.com/foo/** "},
|
||||
},
|
||||
expected: cataloging.SymbolScopeStdlib,
|
||||
expectedModules: []string{"golang.org/x/**", "github.com/foo/**"},
|
||||
},
|
||||
{
|
||||
name: "empty defaults to none",
|
||||
cfg: golangConfig{CaptureSymbols: ""},
|
||||
@@ -59,7 +32,7 @@ func Test_golangConfig_PostLoad(t *testing.T) {
|
||||
},
|
||||
{
|
||||
name: "invalid value defaults to none",
|
||||
cfg: golangConfig{CaptureSymbols: "stdlbi"},
|
||||
cfg: golangConfig{CaptureSymbols: "bogus"},
|
||||
expected: cataloging.SymbolScopeNone,
|
||||
},
|
||||
{
|
||||
@@ -67,16 +40,6 @@ func Test_golangConfig_PostLoad(t *testing.T) {
|
||||
cfg: golangConfig{CaptureSymbols: "true"},
|
||||
expected: cataloging.SymbolScopeNone,
|
||||
},
|
||||
{
|
||||
name: "explicit none resolves to none",
|
||||
cfg: golangConfig{CaptureSymbols: "none"},
|
||||
expected: cataloging.SymbolScopeNone,
|
||||
},
|
||||
{
|
||||
name: "explicit none is not case sensitive",
|
||||
cfg: golangConfig{CaptureSymbols: " NONE "},
|
||||
expected: cataloging.SymbolScopeNone,
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
@@ -89,7 +52,6 @@ func Test_golangConfig_PostLoad(t *testing.T) {
|
||||
return
|
||||
}
|
||||
assert.Equal(t, tt.expected, tt.cfg.CaptureSymbols)
|
||||
assert.Equal(t, tt.expectedModules, tt.cfg.CaptureSymbolsModules)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
@@ -6,12 +6,9 @@ import (
|
||||
)
|
||||
|
||||
type packageConfig struct {
|
||||
SearchUnindexedArchives bool `yaml:"search-unindexed-archives" json:"search-unindexed-archives" mapstructure:"search-unindexed-archives"`
|
||||
SearchIndexedArchives bool `yaml:"search-indexed-archives" json:"search-indexed-archives" mapstructure:"search-indexed-archives"`
|
||||
ExcludeBinaryOverlapByOwnership bool `yaml:"exclude-binary-overlap-by-ownership" json:"exclude-binary-overlap-by-ownership" mapstructure:"exclude-binary-overlap-by-ownership"` // exclude synthetic binary packages owned by os package files
|
||||
NestedArchiveMaxDepth int `yaml:"nested-archive-max-depth" json:"nested-archive-max-depth" mapstructure:"nested-archive-max-depth"`
|
||||
NestedArchiveMaxMemoryBytes int64 `yaml:"nested-archive-max-memory-bytes" json:"nested-archive-max-memory-bytes" mapstructure:"nested-archive-max-memory-bytes"`
|
||||
NestedArchiveMaxDiskBytes int64 `yaml:"nested-archive-max-disk-bytes" json:"nested-archive-max-disk-bytes" mapstructure:"nested-archive-max-disk-bytes"`
|
||||
SearchUnindexedArchives bool `yaml:"search-unindexed-archives" json:"search-unindexed-archives" mapstructure:"search-unindexed-archives"`
|
||||
SearchIndexedArchives bool `yaml:"search-indexed-archives" json:"search-indexed-archives" mapstructure:"search-indexed-archives"`
|
||||
ExcludeBinaryOverlapByOwnership bool `yaml:"exclude-binary-overlap-by-ownership" json:"exclude-binary-overlap-by-ownership" mapstructure:"exclude-binary-overlap-by-ownership"` // exclude synthetic binary packages owned by os package files
|
||||
}
|
||||
|
||||
var _ interface {
|
||||
@@ -20,17 +17,12 @@ var _ interface {
|
||||
|
||||
func (o *packageConfig) DescribeFields(descriptions clio.FieldDescriptionSet) {
|
||||
descriptions.Add(&o.SearchIndexedArchives, `search within archives that do contain a file index to search against (zip)
|
||||
note: this only applies to the java package cataloger, and is ignored when nested-archive-max-depth is not 0 (every archive is searched)`)
|
||||
note: for now this only applies to the java package cataloger`)
|
||||
descriptions.Add(&o.SearchUnindexedArchives, `search within archives that do not contain a file index to search against (tar, tar.gz, tar.bz2, etc)
|
||||
note: enabling this may result in a performance impact since all discovered compressed tars will be decompressed
|
||||
note: this only applies to the java package cataloger, and is ignored when nested-archive-max-depth is not 0 (every archive is searched)`)
|
||||
note: for now this only applies to the java package cataloger`)
|
||||
descriptions.Add(&o.ExcludeBinaryOverlapByOwnership, `allows users to exclude synthetic binary packages from the sbom
|
||||
these packages are removed if an overlap with a non-synthetic package is found`)
|
||||
descriptions.Add(&o.NestedArchiveMaxDepth, `maximum depth to recursively catalog nested archives (0 = disabled, -1 = as deep as 16 levels, the most allowed)
|
||||
note: when enabled this replaces the java cataloger's own unbounded jar-in-jar recursion, so archives deeper than this are reported as unknowns rather than cataloged`)
|
||||
descriptions.Add(&o.NestedArchiveMaxMemoryBytes, `maximum bytes of nested archive content held in memory at once (0 = none, -1 = unbounded)`)
|
||||
descriptions.Add(&o.NestedArchiveMaxDiskBytes, `maximum bytes of nested archive content on disk at once (0 = none, -1 = unbounded)
|
||||
note: this and nested-archive-max-memory-bytes cannot both be 0 when nested-archive-max-depth is not 0`)
|
||||
}
|
||||
|
||||
func defaultPackageConfig() packageConfig {
|
||||
@@ -39,8 +31,5 @@ func defaultPackageConfig() packageConfig {
|
||||
SearchIndexedArchives: c.IncludeIndexedArchives,
|
||||
SearchUnindexedArchives: c.IncludeUnindexedArchives,
|
||||
ExcludeBinaryOverlapByOwnership: true,
|
||||
NestedArchiveMaxDepth: c.MaxDepth,
|
||||
NestedArchiveMaxMemoryBytes: c.MaxMemoryBytes,
|
||||
NestedArchiveMaxDiskBytes: c.MaxDiskBytes,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,53 +0,0 @@
|
||||
package options
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"github.com/anchore/syft/syft/cataloging"
|
||||
)
|
||||
|
||||
func Test_packageConfigDefaultsMatchArchiveSearchConfig(t *testing.T) {
|
||||
want := cataloging.DefaultArchiveSearchConfig()
|
||||
got := defaultPackageConfig()
|
||||
|
||||
assert.Equal(t, want.MaxDepth, got.NestedArchiveMaxDepth)
|
||||
assert.Equal(t, want.MaxMemoryBytes, got.NestedArchiveMaxMemoryBytes)
|
||||
assert.Equal(t, want.MaxDiskBytes, got.NestedArchiveMaxDiskBytes)
|
||||
|
||||
require.Zero(t, got.NestedArchiveMaxDepth, "nested archive cataloging must be off by default")
|
||||
}
|
||||
|
||||
func Test_ToArchiveConfig(t *testing.T) {
|
||||
defaults := cataloging.DefaultArchiveSearchConfig()
|
||||
|
||||
t.Run("defaults round trip unchanged", func(t *testing.T) {
|
||||
cfg := Catalog{Package: defaultPackageConfig()}
|
||||
got := cfg.ToArchiveConfig()
|
||||
|
||||
assert.Equal(t, defaults.MaxDepth, got.MaxDepth)
|
||||
assert.Equal(t, defaults.MaxMemoryBytes, got.MaxMemoryBytes)
|
||||
assert.Equal(t, defaults.MaxDiskBytes, got.MaxDiskBytes)
|
||||
})
|
||||
|
||||
t.Run("every setting is threaded through", func(t *testing.T) {
|
||||
// a key that parses but is not carried into the archive config is invisible in a config
|
||||
// dump, so assert each one arrives
|
||||
cfg := Catalog{Package: packageConfig{
|
||||
NestedArchiveMaxDepth: 3,
|
||||
NestedArchiveMaxMemoryBytes: 111,
|
||||
NestedArchiveMaxDiskBytes: 3333,
|
||||
SearchIndexedArchives: true,
|
||||
SearchUnindexedArchives: true,
|
||||
}}
|
||||
got := cfg.ToArchiveConfig()
|
||||
|
||||
assert.Equal(t, 3, got.MaxDepth)
|
||||
assert.Equal(t, int64(111), got.MaxMemoryBytes)
|
||||
assert.Equal(t, int64(3333), got.MaxDiskBytes)
|
||||
assert.True(t, got.IncludeIndexedArchives)
|
||||
assert.True(t, got.IncludeUnindexedArchives)
|
||||
})
|
||||
}
|
||||
@@ -36,7 +36,7 @@ var _ clio.PostLoader = (*imageSource)(nil)
|
||||
func (o *sourceConfig) DescribeFields(descriptions clio.FieldDescriptionSet) {
|
||||
descriptions.Add(&o.File.Digests, `the file digest algorithms to use on the scanned file (options: "md5", "sha1", "sha224", "sha256", "sha384", "sha512")`)
|
||||
descriptions.Add(&o.Image.DefaultPullSource, `allows users to specify which image source should be used to generate the sbom
|
||||
valid values are: registry, docker, podman, containers-storage`)
|
||||
valid values are: registry, docker, podman`)
|
||||
}
|
||||
|
||||
type imageSource struct {
|
||||
@@ -74,7 +74,7 @@ func (c *imageSource) PostLoad() error {
|
||||
return checkDefaultSourceValues(c.DefaultPullSource)
|
||||
}
|
||||
|
||||
var validDefaultSourceValues = []string{"registry", "docker", "podman", "containers-storage", ""}
|
||||
var validDefaultSourceValues = []string{"registry", "docker", "podman", ""}
|
||||
|
||||
func checkDefaultSourceValues(source string) error {
|
||||
validValues := strset.New(validDefaultSourceValues...)
|
||||
|
||||
@@ -2,11 +2,10 @@ package options
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"path"
|
||||
"sort"
|
||||
"strings"
|
||||
|
||||
@@ -21,10 +20,7 @@ import (
|
||||
"github.com/anchore/syft/syft/sbom"
|
||||
)
|
||||
|
||||
var _ interface {
|
||||
io.Closer
|
||||
sbom.Writer
|
||||
} = (*sbomMultiWriter)(nil)
|
||||
var _ sbom.Writer = (*sbomMultiWriter)(nil)
|
||||
|
||||
var _ interface {
|
||||
io.Closer
|
||||
@@ -168,12 +164,6 @@ func newSBOMMultiWriter(options ...sbomWriterDescription) (_ *sbomMultiWriter, e
|
||||
}
|
||||
|
||||
out := &sbomMultiWriter{}
|
||||
defer func() {
|
||||
// don't leak the files already opened when a later output fails
|
||||
if err != nil {
|
||||
_ = out.Close()
|
||||
}
|
||||
}()
|
||||
|
||||
for _, option := range options {
|
||||
switch len(option.Path) {
|
||||
@@ -182,8 +172,8 @@ func newSBOMMultiWriter(options ...sbomWriterDescription) (_ *sbomMultiWriter, e
|
||||
format: option.Format,
|
||||
})
|
||||
default:
|
||||
// create any missing subdirectories (option.Path is a host path, so this must be filepath)
|
||||
dir := filepath.Dir(option.Path)
|
||||
// create any missing subdirectories
|
||||
dir := path.Dir(option.Path)
|
||||
if dir != "" {
|
||||
s, err := os.Stat(dir)
|
||||
if err != nil {
|
||||
@@ -220,20 +210,6 @@ func (m *sbomMultiWriter) Write(s sbom.SBOM) (errs error) {
|
||||
return errs
|
||||
}
|
||||
|
||||
// Close releases any files held by the writers. Write already closes each file it writes to, so this matters
|
||||
// when the SBOM is never written (e.g. an error after the writer was created).
|
||||
func (m *sbomMultiWriter) Close() error {
|
||||
var errs error
|
||||
for _, w := range m.writers {
|
||||
if c, ok := w.(io.Closer); ok {
|
||||
if err := c.Close(); err != nil && !errors.Is(err, os.ErrClosed) {
|
||||
errs = multierror.Append(errs, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
return errs
|
||||
}
|
||||
|
||||
// sbomStreamWriter implements sbom.Writer for a given format and io.Writer, also providing a close function for cleanup
|
||||
type sbomStreamWriter struct {
|
||||
format sbom.FormatEncoder
|
||||
|
||||
@@ -202,9 +202,9 @@ func Test_newSBOMMultiWriter(t *testing.T) {
|
||||
if test.err {
|
||||
assert.Error(t, err)
|
||||
return
|
||||
} else {
|
||||
assert.NoError(t, err)
|
||||
}
|
||||
require.NoError(t, err)
|
||||
t.Cleanup(func() { _ = mw.Close() })
|
||||
|
||||
assert.Len(t, mw.writers, len(test.expected))
|
||||
|
||||
|
||||
@@ -524,12 +524,4 @@ var commonTestCases = []testCase{
|
||||
"kong": "3.7.0-0",
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "find cpan distribution",
|
||||
pkgType: pkg.CpanPkg,
|
||||
pkgLanguage: pkg.Perl,
|
||||
pkgInfo: map[string]string{
|
||||
"URI": "5.35",
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
@@ -16,21 +16,19 @@ import (
|
||||
"github.com/anchore/syft/syft/source"
|
||||
)
|
||||
|
||||
// multiplatformImage is busybox:1.38.0 pinned by index digest, since docker hub periodically rebuilds and
|
||||
// re-pushes official image tags, which changes the per-platform digests asserted below.
|
||||
const multiplatformImage = "docker.io/library/busybox@sha256:fd7dc98638c8e305f4dc34e979f1c0fdfdcaeb0fbf8fcff77ae834b6da3d7e6e"
|
||||
|
||||
// TestMultiPlatformOCIImageSelection verifies that syft can load a single platform out of a
|
||||
// multi-platform OCI image on disk (both --from oci-dir and --from oci-archive) by selecting the
|
||||
// image that matches the requested platform.
|
||||
func TestMultiPlatformOCIImageSelection(t *testing.T) {
|
||||
remoteImage := "docker.io/library/busybox:1.38.0"
|
||||
|
||||
// Per-platform image config digests within the multi-platform index, obtained from the OCI layout
|
||||
// (see anchore/stereoscope integration tests: TestPlatformSelectionWithOciLocalSources).
|
||||
expectedDigest := map[string]string{
|
||||
"arm64": "sha256:d2482869a6b838d3b4c9cad0c8085c06277909482771eb525d298fc3a7d07927",
|
||||
"s390x": "sha256:e55a0fa55032cceac2d983f4b4a25b28e42303b46a2f1915fc0fa45051e6ca70",
|
||||
"amd64": "sha256:aaef90e065235eb0b2d9938be85d1b81add2902cf89affe2cdfa19be33476ac0",
|
||||
"ppc64le": "sha256:8158829b015d1c5285d63d54d37e95998ef0af4177e3f416678decbcaff3170c",
|
||||
"arm64": "sha256:e0e8b3cbfed68a90084781e2962f9c0deead51c5a3f11a488eef0283a4284bc2",
|
||||
"s390x": "sha256:0cf160e720a8e4f20883b72276a6eaded83b79539f3f1d39e35a3336154b9960",
|
||||
"amd64": "sha256:c6348fa86ba0fb2108c9334f5fe913ddc6d853313e655891f133a0127c30099f",
|
||||
"ppc64le": "sha256:b144fc0e06537d07956cde340b878a81a717e394edb736d3110858a12a6635cb",
|
||||
}
|
||||
|
||||
// syft --from source tag -> stereoscope OCI source used to prepare the local fixture
|
||||
@@ -41,7 +39,7 @@ func TestMultiPlatformOCIImageSelection(t *testing.T) {
|
||||
|
||||
for from, imageSource := range sources {
|
||||
t.Run(from, func(t *testing.T) {
|
||||
localPath := imagetest.PrepareMultiplatformFixtureImage(t, imageSource, multiplatformImage)
|
||||
localPath := imagetest.PrepareMultiplatformFixtureImage(t, imageSource, remoteImage)
|
||||
for _, arch := range []string{"amd64", "arm64", "s390x", "ppc64le"} {
|
||||
t.Run(fmt.Sprintf("linux/%s", arch), func(t *testing.T) {
|
||||
platform, err := image.NewPlatform("linux/" + arch)
|
||||
@@ -62,8 +60,6 @@ func TestMultiPlatformOCIImageSelection(t *testing.T) {
|
||||
|
||||
// The raw config of the selected image must match the requested platform...
|
||||
assertConfigPlatform(t, meta.RawConfig, "linux", arch)
|
||||
assert.Equal(t, "linux", meta.OS)
|
||||
assert.Equal(t, arch, meta.Architecture)
|
||||
// ...and it must be the exact per-platform image from the multi-platform index.
|
||||
assert.Equal(t, expectedDigest[arch], meta.ID)
|
||||
})
|
||||
@@ -75,6 +71,8 @@ func TestMultiPlatformOCIImageSelection(t *testing.T) {
|
||||
// TestMultiPlatformOCIImageSelection_UnavailablePlatform verifies that requesting a platform not present
|
||||
// in the multi-platform OCI image results in an error rather than silently selecting the wrong image.
|
||||
func TestMultiPlatformOCIImageSelection_UnavailablePlatform(t *testing.T) {
|
||||
remoteImage := "docker.io/library/busybox:1.38.0"
|
||||
|
||||
// windows/amd64 is not present in this linux-only multi-platform image
|
||||
platform, err := image.NewPlatform("windows/amd64")
|
||||
require.NoError(t, err)
|
||||
@@ -86,7 +84,7 @@ func TestMultiPlatformOCIImageSelection_UnavailablePlatform(t *testing.T) {
|
||||
|
||||
for from, imageSource := range sources {
|
||||
t.Run(from, func(t *testing.T) {
|
||||
localPath := imagetest.PrepareMultiplatformFixtureImage(t, imageSource, multiplatformImage)
|
||||
localPath := imagetest.PrepareMultiplatformFixtureImage(t, imageSource, remoteImage)
|
||||
|
||||
_, err := syft.GetSource(
|
||||
context.Background(),
|
||||
@@ -101,6 +99,8 @@ func TestMultiPlatformOCIImageSelection_UnavailablePlatform(t *testing.T) {
|
||||
// TestMultiPlatformOCIImageSelection_DefaultPlatform verifies that not specifying a platform results
|
||||
// in the current platform being selected.
|
||||
func TestMultiPlatformOCIImageSelection_DefaultPlatform(t *testing.T) {
|
||||
remoteImage := "docker.io/library/busybox:1.38.0"
|
||||
|
||||
sources := map[string]image.Source{
|
||||
"oci-dir": image.OciDirectorySource,
|
||||
"oci-archive": image.OciTarballSource,
|
||||
@@ -108,7 +108,7 @@ func TestMultiPlatformOCIImageSelection_DefaultPlatform(t *testing.T) {
|
||||
|
||||
for from, imageSource := range sources {
|
||||
t.Run(from, func(t *testing.T) {
|
||||
localPath := imagetest.PrepareMultiplatformFixtureImage(t, imageSource, multiplatformImage)
|
||||
localPath := imagetest.PrepareMultiplatformFixtureImage(t, imageSource, remoteImage)
|
||||
|
||||
src, err := syft.GetSource(
|
||||
context.Background(),
|
||||
@@ -125,8 +125,6 @@ func TestMultiPlatformOCIImageSelection_DefaultPlatform(t *testing.T) {
|
||||
|
||||
// The raw config of the selected image must match the current platform
|
||||
assertConfigPlatform(t, meta.RawConfig, "linux", runtime.GOARCH)
|
||||
assert.Equal(t, "linux", meta.OS)
|
||||
assert.Equal(t, runtime.GOARCH, meta.Architecture)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,64 +0,0 @@
|
||||
package integration
|
||||
|
||||
import (
|
||||
"context"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"github.com/anchore/stereoscope/pkg/image"
|
||||
"github.com/anchore/stereoscope/pkg/imagetest"
|
||||
"github.com/anchore/syft/syft"
|
||||
"github.com/anchore/syft/syft/source"
|
||||
)
|
||||
|
||||
// TestSinglePlatformOCIImage covers the single-image (non-index) OCI layouts, where syft used to report
|
||||
// empty os/architecture metadata and silently accept any --platform. Both now come from the image config.
|
||||
func TestSinglePlatformOCIImage(t *testing.T) {
|
||||
// a platform this fixture is never built for, so the mismatch is not host dependent
|
||||
unavailablePlatform, err := image.NewPlatform("linux/ppc64le")
|
||||
require.NoError(t, err)
|
||||
|
||||
sources := map[string]image.Source{
|
||||
"oci-dir": image.OciDirectorySource,
|
||||
"oci-archive": image.OciTarballSource,
|
||||
}
|
||||
|
||||
for from, imageSource := range sources {
|
||||
t.Run(from, func(t *testing.T) {
|
||||
localPath := strings.TrimPrefix(imagetest.PrepareFixtureImage(t, imageSource, "image-distro-id"), string(imageSource)+":")
|
||||
|
||||
t.Run("reports the platform from the image config", func(t *testing.T) {
|
||||
src, err := syft.GetSource(
|
||||
context.Background(),
|
||||
localPath,
|
||||
syft.DefaultGetSourceConfig().WithSources(from),
|
||||
)
|
||||
require.NoError(t, err)
|
||||
t.Cleanup(func() {
|
||||
require.NoError(t, src.Close())
|
||||
})
|
||||
|
||||
meta, ok := src.Describe().Metadata.(source.ImageMetadata)
|
||||
require.True(t, ok, "expected image metadata, got %T", src.Describe().Metadata)
|
||||
|
||||
// the fixture is built for whatever the local docker daemon runs, so assert against the
|
||||
// config rather than a fixed platform
|
||||
assert.NotEmpty(t, meta.OS)
|
||||
assert.NotEmpty(t, meta.Architecture)
|
||||
assertConfigPlatform(t, meta.RawConfig, meta.OS, meta.Architecture)
|
||||
})
|
||||
|
||||
t.Run("rejects a platform the image does not match", func(t *testing.T) {
|
||||
_, err := syft.GetSource(
|
||||
context.Background(),
|
||||
localPath,
|
||||
syft.DefaultGetSourceConfig().WithSources(from).WithPlatform(unavailablePlatform),
|
||||
)
|
||||
require.ErrorContains(t, err, "linux/ppc64le")
|
||||
})
|
||||
})
|
||||
}
|
||||
}
|
||||
+3
-2
@@ -8,6 +8,8 @@ RUN mkdir tmp
|
||||
|
||||
WORKDIR /usr/share/jenkins/tmp
|
||||
|
||||
RUN apt-get update 2>&1 > /dev/null && apt-get install -y less zip 2>&1 > /dev/null
|
||||
|
||||
RUN unzip ../jenkins.war 2>&1 > /dev/null
|
||||
|
||||
RUN rm -rf ./META-INF/MANIFEST.MF ./WEB-INF ./jsbundles ./scripts ./css
|
||||
@@ -16,8 +18,7 @@ WORKDIR /usr/share/jenkins
|
||||
|
||||
RUN rm -rf jenkins.war
|
||||
|
||||
# use jar (from the JDK in the base image) to rezip -- M keeps it manifest-less, which is the point of this fixture
|
||||
RUN cd ./tmp && jar cfM ../jenkins.war . && cd ..
|
||||
RUN cd ./tmp && zip -r ../jenkins.war . && cd ..
|
||||
|
||||
RUN rm -rf ./tmp
|
||||
|
||||
|
||||
@@ -16,7 +16,7 @@ RUN unxz /lib/modules/6.0.7-301.fc37.x86_64/kernel/drivers/tty/ttynull.ko.xz
|
||||
# dotnet pkg coverage
|
||||
# https://nuget.info/packages/DocuSign.eSign.dll/6.8.0
|
||||
# https://github.com/docusign/docusign-esign-csharp-client/blob/master/LICENSE
|
||||
RUN curl -fLO https://www.nuget.org/api/v2/package/DocuSign.eSign.dll/6.8.0
|
||||
RUN curl -LO https://www.nuget.org/api/v2/package/DocuSign.eSign.dll/6.8.0
|
||||
RUN unzip 6.8.0
|
||||
RUN chmod 600 lib/net462/DocuSign.eSign.dll
|
||||
RUN rm 6.8.0
|
||||
|
||||
-9
@@ -1,9 +0,0 @@
|
||||
{
|
||||
"abstract": "Uniform Resource Identifiers (absolute and relative)",
|
||||
"license": ["perl_5"],
|
||||
"meta-spec": {"version": 2},
|
||||
"name": "URI",
|
||||
"prereqs": {"runtime": {"requires": {"Carp": "0", "perl": "5.008001"}}},
|
||||
"release_status": "stable",
|
||||
"version": "5.35"
|
||||
}
|
||||
-11
@@ -1,11 +0,0 @@
|
||||
{
|
||||
"dist": "URI-5.35",
|
||||
"name": "URI",
|
||||
"pathname": "O/OA/OALDERS/URI-5.35.tar.gz",
|
||||
"provides": {
|
||||
"URI": {"file": "lib/URI.pm", "version": "5.35"},
|
||||
"URI::Escape": {"file": "lib/URI/Escape.pm", "version": "5.35"}
|
||||
},
|
||||
"target": "URI",
|
||||
"version": "5.35"
|
||||
}
|
||||
@@ -1,16 +1,13 @@
|
||||
package ui
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
tea "github.com/charmbracelet/bubbletea"
|
||||
"github.com/charmbracelet/lipgloss"
|
||||
"github.com/wagoodman/go-partybus"
|
||||
|
||||
"github.com/anchore/bubbly"
|
||||
@@ -55,7 +52,7 @@ func New(out io.Writer, quiet bool, handlers ...bubbly.EventHandler) *UI {
|
||||
func (m *UI) Setup(subscription partybus.Unsubscribable) error {
|
||||
// we still want to collect log messages, however, we also the logger shouldn't write to the screen directly
|
||||
if logWrapper, ok := log.Get().(logger.Controller); ok {
|
||||
logWrapper.SetOutput(&logFooter{w: m.frame.(*frame.Frame).Footer()})
|
||||
logWrapper.SetOutput(m.frame.(*frame.Frame).Footer())
|
||||
}
|
||||
|
||||
m.subscription = subscription
|
||||
@@ -187,56 +184,9 @@ func (m *UI) Update(msg tea.Msg) (tea.Model, tea.Cmd) {
|
||||
}
|
||||
|
||||
func (m UI) View() string {
|
||||
v := m.frame.View()
|
||||
// bubbletea erases the line the cursor rests on at exit, which would drop the frame's last line
|
||||
if !strings.HasSuffix(v, "\n") {
|
||||
return v + "\n"
|
||||
}
|
||||
return v
|
||||
return m.frame.View()
|
||||
}
|
||||
|
||||
var (
|
||||
logTitleStyle = lipgloss.NewStyle().Foreground(lipgloss.Color("8")) // dark grey
|
||||
// the first log line gets a branch pointing at it from the title, the rest are indented to match
|
||||
logBranch = logTitleStyle.Render("└─▶") + " "
|
||||
logIndent = " "
|
||||
)
|
||||
|
||||
// logFooter decorates log entries drawn below the TUI with a one-time title (written when the first entry arrives)
|
||||
// and a branch from the title to the first log line. Writes are serialized by the logger.
|
||||
type logFooter struct {
|
||||
w io.Writer
|
||||
started bool
|
||||
}
|
||||
|
||||
func (l *logFooter) Write(p []byte) (int, error) {
|
||||
var buf bytes.Buffer
|
||||
first := !l.started
|
||||
if first {
|
||||
buf.WriteString(logTitleStyle.Render("Logs (non-fatal, see -v for more):") + "\n")
|
||||
l.started = true
|
||||
}
|
||||
for i, line := range bytes.SplitAfter(p, []byte("\n")) {
|
||||
if len(line) == 0 {
|
||||
continue
|
||||
}
|
||||
if first && i == 0 {
|
||||
buf.WriteString(logBranch)
|
||||
} else {
|
||||
buf.WriteString(logIndent)
|
||||
}
|
||||
buf.Write(line)
|
||||
}
|
||||
if _, err := l.w.Write(buf.Bytes()); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return len(p), nil
|
||||
}
|
||||
|
||||
// IsTerminal marks the footer as terminal-bound (it is drawn by the TUI) so the logger keeps colors. NO_COLOR is
|
||||
// still honored by the formatter.
|
||||
func (*logFooter) IsTerminal() bool { return true }
|
||||
|
||||
func runWithTimeout(timeout time.Duration, fn func() error) (err error) {
|
||||
c := make(chan struct{}, 1)
|
||||
go func() {
|
||||
|
||||
@@ -1,31 +0,0 @@
|
||||
package ui
|
||||
|
||||
import (
|
||||
"io"
|
||||
"testing"
|
||||
|
||||
tea "github.com/charmbracelet/bubbletea"
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"github.com/anchore/bubbly/bubbles/frame"
|
||||
)
|
||||
|
||||
var _ tea.Model = (*staticModel)(nil)
|
||||
|
||||
type staticModel string
|
||||
|
||||
func Test_UI_viewEndsOnAnEmptyLine(t *testing.T) {
|
||||
// bubbletea erases the line the cursor rests on when the program stops, so a view whose last line
|
||||
// carries content loses it. Unless a log line happened to be in the footer, that dropped the last row
|
||||
// of the cataloging tree on any source that logs nothing.
|
||||
subject := New(io.Discard, false)
|
||||
subject.frame.(*frame.Frame).AppendModel(staticModel("first row\nlast row"))
|
||||
|
||||
require.Equal(t, "first row\nlast row\n", subject.View())
|
||||
}
|
||||
|
||||
func (m staticModel) Init() tea.Cmd { return nil }
|
||||
|
||||
func (m staticModel) Update(tea.Msg) (tea.Model, tea.Cmd) { return m, nil }
|
||||
|
||||
func (m staticModel) View() string { return string(m) }
|
||||
@@ -1,33 +1,33 @@
|
||||
module github.com/anchore/syft
|
||||
|
||||
go 1.26.8
|
||||
go 1.26.3
|
||||
|
||||
require (
|
||||
github.com/BurntSushi/toml v1.6.0
|
||||
github.com/CycloneDX/cyclonedx-go v0.12.0
|
||||
github.com/CycloneDX/cyclonedx-go v0.11.0
|
||||
github.com/Masterminds/semver/v3 v3.5.0
|
||||
github.com/Masterminds/sprig/v3 v3.3.0
|
||||
github.com/OneOfOne/xxhash v1.2.8
|
||||
github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d
|
||||
github.com/acobaugh/osrelease v0.1.0
|
||||
github.com/adrg/xdg v0.5.3
|
||||
github.com/anchore/bubbly v0.2.2
|
||||
github.com/anchore/bubbly v0.2.1
|
||||
github.com/anchore/clio v0.1.1
|
||||
github.com/anchore/fangs v0.1.1
|
||||
github.com/anchore/go-collections v0.1.1
|
||||
github.com/anchore/go-homedir v0.1.1
|
||||
github.com/anchore/go-logger v0.2.0
|
||||
github.com/anchore/go-logger v0.1.1
|
||||
github.com/anchore/go-macholibre v0.1.1
|
||||
github.com/anchore/go-rpmdb v0.2.2
|
||||
github.com/anchore/go-sync v0.1.2
|
||||
github.com/anchore/go-rpmdb v0.2.0
|
||||
github.com/anchore/go-sync v0.1.1
|
||||
github.com/anchore/go-version v1.2.2-0.20200701162849-18adb9c92b9b
|
||||
github.com/anchore/packageurl-go v0.2.0
|
||||
github.com/anchore/stereoscope v0.3.4
|
||||
github.com/anchore/stereoscope v0.3.0
|
||||
github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be
|
||||
github.com/aquasecurity/go-pep440-version v0.0.1
|
||||
github.com/bitnami/go-version v0.0.0-20250131085805-b1f57a8634ef
|
||||
github.com/blakesmith/ar v0.0.0-20190502131153-809d4375e1fb
|
||||
github.com/bmatcuk/doublestar/v4 v4.10.2
|
||||
github.com/bmatcuk/doublestar/v4 v4.10.0
|
||||
github.com/cespare/xxhash/v2 v2.3.0
|
||||
github.com/charmbracelet/bubbles v1.0.0
|
||||
github.com/charmbracelet/bubbletea v1.3.10
|
||||
@@ -36,38 +36,38 @@ require (
|
||||
github.com/deitch/magic v0.0.0-20230404182410-1ff89d7342da
|
||||
github.com/diskfs/go-diskfs v1.9.4
|
||||
github.com/distribution/reference v0.6.0
|
||||
github.com/dustin/go-humanize v1.1.0
|
||||
github.com/dustin/go-humanize v1.0.1
|
||||
github.com/elliotchance/phpserialize v1.4.0
|
||||
github.com/facebookincubator/nvdtools v0.1.5
|
||||
github.com/github/go-spdx/v2 v2.7.1
|
||||
github.com/github/go-spdx/v2 v2.7.0
|
||||
github.com/gkampitakis/go-snaps v0.5.23
|
||||
github.com/go-git/go-billy/v5 v5.9.2
|
||||
github.com/go-git/go-git/v5 v5.19.3
|
||||
github.com/go-git/go-billy/v5 v5.9.1
|
||||
github.com/go-git/go-git/v5 v5.19.1
|
||||
github.com/go-test/deep v1.1.1
|
||||
github.com/go-viper/mapstructure/v2 v2.5.0
|
||||
github.com/goccy/go-yaml v1.19.2
|
||||
github.com/gohugoio/hashstructure v0.6.0
|
||||
github.com/google/go-cmp v0.7.0
|
||||
github.com/google/go-containerregistry v0.22.1
|
||||
github.com/google/go-containerregistry v0.21.7
|
||||
github.com/google/licensecheck v0.3.1
|
||||
github.com/google/uuid v1.6.0
|
||||
github.com/gookit/color v1.6.2
|
||||
github.com/gpustack/gguf-parser-go v0.26.4
|
||||
github.com/gookit/color v1.6.1
|
||||
github.com/gpustack/gguf-parser-go v0.25.0
|
||||
github.com/hashicorp/go-cleanhttp v0.5.2
|
||||
github.com/hashicorp/go-getter v1.8.10
|
||||
github.com/hashicorp/go-getter v1.8.6
|
||||
github.com/hashicorp/go-multierror v1.1.1
|
||||
github.com/hashicorp/hcl/v2 v2.25.0
|
||||
github.com/hashicorp/hcl/v2 v2.24.0
|
||||
github.com/iancoleman/strcase v0.3.0
|
||||
github.com/invopop/jsonschema v0.14.0
|
||||
github.com/jedib0t/go-pretty/v6 v6.8.3
|
||||
github.com/jinzhu/copier v0.4.0
|
||||
github.com/kastenhq/goversion v0.0.0-20230811215019-93b2f8823953
|
||||
github.com/klauspost/compress v1.20.1
|
||||
github.com/magiconair/properties v1.18.12
|
||||
github.com/klauspost/compress v1.19.1
|
||||
github.com/magiconair/properties v1.18.11
|
||||
github.com/mholt/archives v0.1.5
|
||||
github.com/moby/sys/mountinfo v0.7.2
|
||||
github.com/nix-community/go-nix v0.0.0-20250101154619-4bdde671e0a1
|
||||
github.com/olekukonko/tablewriter v1.1.5
|
||||
github.com/olekukonko/tablewriter v1.1.4
|
||||
github.com/opencontainers/go-digest v1.0.0
|
||||
github.com/pelletier/go-toml v1.9.5
|
||||
github.com/quasilyte/go-ruleguard/dsl v0.3.23
|
||||
@@ -82,8 +82,8 @@ require (
|
||||
github.com/spdx/tools-golang v0.6.0-rc4
|
||||
github.com/spf13/afero v1.15.0
|
||||
github.com/spf13/cobra v1.10.2
|
||||
github.com/stretchr/testify v1.12.1
|
||||
github.com/ulikunitz/xz v0.5.17
|
||||
github.com/stretchr/testify v1.11.1
|
||||
github.com/ulikunitz/xz v0.5.16
|
||||
github.com/vbatts/go-mtree v0.7.0
|
||||
github.com/vifraa/gopom v1.0.0
|
||||
github.com/wagoodman/go-partybus v0.0.0-20230516145632-8ccac152c651
|
||||
@@ -91,69 +91,67 @@ require (
|
||||
github.com/xeipuuv/gojsonschema v1.2.0
|
||||
github.com/zyedidia/generic v1.2.2-0.20230320175451-4410d2372cb1
|
||||
go.uber.org/goleak v1.3.0
|
||||
go.yaml.in/yaml/v3 v3.0.5
|
||||
go.yaml.in/yaml/v3 v3.0.4
|
||||
golang.org/x/exp v0.0.0-20260410095643-746e56fc9e2f
|
||||
golang.org/x/mod v0.41.0
|
||||
golang.org/x/net v0.61.0
|
||||
golang.org/x/time v0.16.0
|
||||
golang.org/x/tools v0.51.0
|
||||
modernc.org/sqlite v1.60.1
|
||||
golang.org/x/mod v0.38.0
|
||||
golang.org/x/net v0.57.0
|
||||
golang.org/x/time v0.15.0
|
||||
golang.org/x/tools v0.48.0
|
||||
gopkg.in/yaml.v3 v3.0.1
|
||||
modernc.org/sqlite v1.55.0
|
||||
)
|
||||
|
||||
require (
|
||||
github.com/pb33f/ordered-map/v2 v2.3.2
|
||||
github.com/pb33f/ordered-map/v2 v2.3.1
|
||||
github.com/tailscale/hujson v0.0.0-20260302212456-ecc657c15afd
|
||||
golang.org/x/sys v0.49.0
|
||||
howett.net/plist v1.0.1
|
||||
)
|
||||
|
||||
require (
|
||||
cel.dev/expr v0.25.2 // indirect
|
||||
cel.dev/expr v0.25.1 // indirect
|
||||
cloud.google.com/go v0.123.0 // indirect
|
||||
cloud.google.com/go/auth v0.23.3 // indirect
|
||||
cloud.google.com/go/auth v0.18.2 // indirect
|
||||
cloud.google.com/go/auth/oauth2adapt v0.2.8 // indirect
|
||||
cloud.google.com/go/compute/metadata v0.9.1 // indirect
|
||||
cloud.google.com/go/iam v1.12.0 // indirect
|
||||
cloud.google.com/go/monitoring v1.30.0 // indirect
|
||||
cloud.google.com/go/storage v1.68.0 // indirect
|
||||
cyphar.com/go-pathrs v0.2.5 // indirect
|
||||
cloud.google.com/go/compute/metadata v0.9.0 // indirect
|
||||
cloud.google.com/go/iam v1.5.3 // indirect
|
||||
cloud.google.com/go/monitoring v1.24.3 // indirect
|
||||
cloud.google.com/go/storage v1.61.3 // indirect
|
||||
dario.cat/mergo v1.0.2 // indirect
|
||||
github.com/DataDog/zstd v1.5.5 // indirect
|
||||
github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.34.0 // indirect
|
||||
github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.57.0 // indirect
|
||||
github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.57.0 // indirect
|
||||
github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.32.0 // indirect
|
||||
github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.55.0 // indirect
|
||||
github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.55.0 // indirect
|
||||
github.com/Masterminds/goutils v1.1.1 // indirect
|
||||
github.com/Microsoft/go-winio v0.6.3-0.20251027160822-ad3df93bed29 // indirect
|
||||
github.com/Microsoft/hcsshim v0.15.0-rc.4 // indirect
|
||||
github.com/Microsoft/hcsshim v0.15.0-rc.1 // indirect
|
||||
github.com/ProtonMail/go-crypto v1.4.1 // indirect
|
||||
github.com/STARRY-S/zip v0.2.3 // indirect
|
||||
github.com/VividCortex/ewma v1.2.0 // indirect
|
||||
github.com/agext/levenshtein v1.2.1 // indirect
|
||||
github.com/anchore/go-lzo v0.1.1 // indirect
|
||||
github.com/anchore/go-struct-converter v0.2.0-rc2 // indirect
|
||||
github.com/andybalholm/brotli v1.2.0 // indirect
|
||||
github.com/apparentlymart/go-textseg/v15 v15.0.0 // indirect
|
||||
github.com/apparentlymart/go-textseg/v17 v17.0.1 // indirect
|
||||
github.com/aquasecurity/go-version v0.0.1 // indirect
|
||||
github.com/atotto/clipboard v0.1.4 // indirect
|
||||
github.com/aws/aws-sdk-go-v2 v1.47.1 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.20 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/config v1.33.6 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/credentials v1.20.6 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.20.1 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/configsources v1.5.4 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.8.4 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/v4a v1.5.4 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.19 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.11.5 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.14.4 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.20.4 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/s3 v1.114.0 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/signin v1.10.1 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/sso v1.38.1 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.43.1 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/sts v1.51.1 // indirect
|
||||
github.com/aws/smithy-go v1.28.1 // indirect
|
||||
github.com/aws/aws-sdk-go-v2 v1.41.5 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.8 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/config v1.32.12 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/credentials v1.19.12 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.20 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.21 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.21 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/ini v1.8.6 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.22 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.7 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.13 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.21 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.21 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/s3 v1.97.3 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/signin v1.0.8 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/sso v1.30.13 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.17 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/sts v1.41.9 // indirect
|
||||
github.com/aws/smithy-go v1.24.2 // indirect
|
||||
github.com/aymanbagabas/go-osc52/v2 v2.0.1 // indirect
|
||||
github.com/bahlo/generic-list-go v0.2.0 // indirect
|
||||
github.com/becheran/wildmatch-go v1.0.0 // indirect
|
||||
@@ -172,52 +170,48 @@ require (
|
||||
github.com/cloudflare/circl v1.6.3 // indirect
|
||||
github.com/cncf/xds/go v0.0.0-20260202195803-dba9d589def2 // indirect
|
||||
github.com/containerd/cgroups/v3 v3.1.3 // indirect
|
||||
github.com/containerd/containerd/api v1.12.0 // indirect
|
||||
github.com/containerd/containerd/v2 v2.4.1 // indirect
|
||||
github.com/containerd/containerd/api v1.11.1 // indirect
|
||||
github.com/containerd/containerd/v2 v2.3.3 // indirect
|
||||
github.com/containerd/continuity v0.5.0 // indirect
|
||||
github.com/containerd/errdefs v1.0.0 // indirect
|
||||
github.com/containerd/errdefs/pkg v0.3.0 // indirect
|
||||
github.com/containerd/fifo v1.1.0 // indirect
|
||||
github.com/containerd/log v0.2.0 // indirect
|
||||
github.com/containerd/log/otel v0.1.0 // indirect
|
||||
github.com/containerd/platforms v1.0.0-rc.5 // indirect
|
||||
github.com/containerd/log v0.1.0 // indirect
|
||||
github.com/containerd/platforms v1.0.0-rc.4 // indirect
|
||||
github.com/containerd/plugin v1.1.0 // indirect
|
||||
github.com/containerd/stargz-snapshotter/estargz v0.18.2 // indirect
|
||||
github.com/containerd/ttrpc v1.2.9 // indirect
|
||||
github.com/containerd/typeurl/v2 v2.3.0 // indirect
|
||||
github.com/containers/libtrust v0.0.0-20230121012942-c1716e8a8d01 // indirect
|
||||
github.com/containers/ocicrypt v1.3.2 // indirect
|
||||
github.com/cyberphone/json-canonicalization v0.0.0-20241213102144-19d51d7fe467 // indirect
|
||||
github.com/cyphar/filepath-securejoin v0.7.0 // indirect
|
||||
github.com/docker/cli v29.8.2+incompatible // indirect
|
||||
github.com/docker/docker-credential-helpers v0.9.8 // indirect
|
||||
github.com/docker/go-connections v0.8.1 // indirect
|
||||
github.com/containerd/ttrpc v1.2.8 // indirect
|
||||
github.com/containerd/typeurl/v2 v2.2.3 // indirect
|
||||
github.com/cyphar/filepath-securejoin v0.6.1 // indirect
|
||||
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
|
||||
github.com/docker/cli v29.6.1+incompatible // indirect
|
||||
github.com/docker/docker-credential-helpers v0.9.5 // indirect
|
||||
github.com/docker/go-connections v0.7.0 // indirect
|
||||
github.com/docker/go-units v0.5.0 // indirect
|
||||
github.com/dsnet/compress v0.0.2-0.20230904184137-39efe44ab707 // indirect
|
||||
github.com/emirpasic/gods v1.18.1 // indirect
|
||||
github.com/envoyproxy/go-control-plane/envoy v1.37.0 // indirect
|
||||
github.com/envoyproxy/protoc-gen-validate v1.3.3 // indirect
|
||||
github.com/erikgeiser/coninput v0.0.0-20211004153227-1c3628e74d0f // indirect
|
||||
github.com/fatih/color v1.19.0 // indirect
|
||||
github.com/fatih/color v1.18.0 // indirect
|
||||
github.com/felixge/fgprof v0.9.5 // indirect
|
||||
github.com/felixge/httpsnoop v1.1.0 // indirect
|
||||
github.com/fsnotify/fsnotify v1.10.1 // indirect
|
||||
github.com/gabriel-vasile/mimetype v1.4.15 // indirect
|
||||
github.com/felixge/httpsnoop v1.0.4 // indirect
|
||||
github.com/fsnotify/fsnotify v1.9.0 // indirect
|
||||
github.com/gabriel-vasile/mimetype v1.4.13 // indirect
|
||||
github.com/gkampitakis/ciinfo v0.3.4 // indirect
|
||||
github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 // indirect
|
||||
github.com/go-jose/go-jose/v4 v4.1.5 // indirect
|
||||
github.com/go-logr/logr v1.4.4 // indirect
|
||||
github.com/go-jose/go-jose/v4 v4.1.4 // indirect
|
||||
github.com/go-logr/logr v1.4.3 // indirect
|
||||
github.com/go-logr/stdr v1.2.2 // indirect
|
||||
github.com/go-restruct/restruct v1.2.0-alpha // indirect
|
||||
github.com/gogo/protobuf v1.3.2 // indirect
|
||||
github.com/golang/groupcache v0.0.0-20241129210726-2c02b8208cf8 // indirect
|
||||
github.com/google/go-intervals v0.0.2 // indirect
|
||||
github.com/google/pprof v0.0.0-20260802141513-ef3492d7dac3 // indirect
|
||||
github.com/google/s2a-go v0.1.10 // indirect
|
||||
github.com/googleapis/enterprise-certificate-proxy v0.3.22 // indirect
|
||||
github.com/googleapis/gax-go/v2 v2.24.1 // indirect
|
||||
github.com/hashicorp/aws-sdk-go-base/v2 v2.0.0-beta.74 // indirect
|
||||
github.com/google/pprof v0.0.0-20250317173921-a4b03ec1a45e // indirect
|
||||
github.com/google/s2a-go v0.1.9 // indirect
|
||||
github.com/googleapis/enterprise-certificate-proxy v0.3.14 // indirect
|
||||
github.com/googleapis/gax-go/v2 v2.17.0 // indirect
|
||||
github.com/hashicorp/aws-sdk-go-base/v2 v2.0.0-beta.72 // indirect
|
||||
github.com/hashicorp/errwrap v1.1.0 // indirect
|
||||
github.com/hashicorp/go-version v1.9.0 // indirect
|
||||
github.com/hashicorp/go-version v1.8.0 // indirect
|
||||
github.com/hashicorp/golang-lru/v2 v2.0.7 // indirect
|
||||
github.com/henvic/httpretty v0.1.4 // indirect
|
||||
github.com/huandu/xstrings v1.5.0 // indirect
|
||||
@@ -225,128 +219,114 @@ require (
|
||||
github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 // indirect
|
||||
github.com/json-iterator/go v1.1.12 // indirect
|
||||
github.com/kevinburke/ssh_config v1.2.0 // indirect
|
||||
github.com/klauspost/cpuid/v2 v2.3.0 // indirect
|
||||
github.com/klauspost/pgzip v1.2.6 // indirect
|
||||
github.com/kr/pretty v0.3.1 // indirect
|
||||
github.com/kr/text v0.2.0 // indirect
|
||||
github.com/lucasb-eyer/go-colorful v1.3.0 // indirect
|
||||
github.com/maruel/natural v1.3.0 // indirect
|
||||
github.com/mattn/go-colorable v0.1.14 // indirect
|
||||
github.com/mattn/go-isatty v0.0.24 // indirect
|
||||
github.com/mattn/go-isatty v0.0.20 // indirect
|
||||
github.com/mattn/go-localereader v0.0.2-0.20220822084749-2491eb6c1c75 // indirect
|
||||
github.com/mattn/go-runewidth v0.0.27 // indirect
|
||||
github.com/mattn/go-sqlite3 v1.14.48 // indirect
|
||||
github.com/mattn/go-runewidth v0.0.21 // indirect
|
||||
github.com/mgutz/ansi v0.0.0-20200706080929-d51e80ef957d // indirect
|
||||
github.com/miekg/pkcs11 v1.1.2 // indirect
|
||||
github.com/mikelolasagasti/xz v1.0.1 // indirect
|
||||
github.com/minio/minlz v1.0.1 // indirect
|
||||
github.com/mistifyio/go-zfs/v4 v4.0.0 // indirect
|
||||
github.com/mitchellh/copystructure v1.2.0 // indirect
|
||||
github.com/mitchellh/go-homedir v1.1.0 // indirect
|
||||
github.com/mitchellh/go-wordwrap v1.0.1 // indirect
|
||||
github.com/mitchellh/reflectwalk v1.0.2 // indirect
|
||||
github.com/moby/docker-image-spec v1.3.1 // indirect
|
||||
github.com/moby/locker v1.0.1 // indirect
|
||||
github.com/moby/moby/api v1.56.0 // indirect
|
||||
github.com/moby/moby/client v0.6.0 // indirect
|
||||
github.com/moby/sys/capability v0.4.0 // indirect
|
||||
github.com/moby/moby/api v1.55.0 // indirect
|
||||
github.com/moby/moby/client v0.5.0 // indirect
|
||||
github.com/moby/sys/sequential v0.6.0 // indirect
|
||||
github.com/moby/sys/signal v0.7.1 // indirect
|
||||
github.com/moby/sys/user v0.4.1 // indirect
|
||||
github.com/moby/sys/userns v0.2.1 // indirect
|
||||
github.com/moby/sys/user v0.4.0 // indirect
|
||||
github.com/moby/sys/userns v0.1.0 // indirect
|
||||
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
|
||||
github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee // indirect
|
||||
github.com/muesli/ansi v0.0.0-20230316100256-276c6243b2f6 // indirect
|
||||
github.com/muesli/cancelreader v0.2.2 // indirect
|
||||
github.com/muesli/termenv v0.16.0 // indirect
|
||||
github.com/ncruces/go-strftime v1.0.0 // indirect
|
||||
github.com/nwaples/rardecode/v2 v2.4.1 // indirect
|
||||
github.com/nwaples/rardecode/v2 v2.2.0 // indirect
|
||||
github.com/olekukonko/cat v0.0.0-20250911104152-50322a0618f6 // indirect
|
||||
github.com/olekukonko/errors v1.2.0 // indirect
|
||||
github.com/olekukonko/ll v0.1.6 // indirect
|
||||
github.com/opencontainers/image-spec v1.1.1 // indirect
|
||||
github.com/opencontainers/runtime-spec v1.3.0 // indirect
|
||||
github.com/opencontainers/selinux v1.15.1 // indirect
|
||||
github.com/pb33f/go-yaml v0.1.0 // indirect
|
||||
github.com/pborman/indent v1.2.1 // indirect
|
||||
github.com/pelletier/go-toml/v2 v2.4.3 // indirect
|
||||
github.com/pierrec/lz4/v4 v4.1.26 // indirect
|
||||
github.com/piprate/json-gold v0.7.0 // indirect
|
||||
github.com/pjbgf/sha1cd v0.7.0 // indirect
|
||||
github.com/pjbgf/sha1cd v0.6.0 // indirect
|
||||
github.com/pkg/errors v0.9.1 // indirect
|
||||
github.com/pkg/profile v1.7.0 // indirect
|
||||
github.com/pkg/xattr v0.4.12 // indirect
|
||||
github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 // indirect
|
||||
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
|
||||
github.com/pquerna/cachecontrol v0.0.0-20180517163645-1555304b9b35 // indirect
|
||||
github.com/proglottis/gpgme v0.1.6 // indirect
|
||||
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect
|
||||
github.com/rivo/uniseg v0.4.7 // indirect
|
||||
github.com/rogpeppe/go-internal v1.14.1 // indirect
|
||||
github.com/sagikazarmark/locafero v0.11.0 // indirect
|
||||
github.com/sahilm/fuzzy v0.1.1 // indirect
|
||||
github.com/secure-systems-lab/go-securesystemslib v0.11.1 // indirect
|
||||
github.com/shopspring/decimal v1.4.0 // indirect
|
||||
github.com/sigstore/fulcio v1.8.7 // indirect
|
||||
github.com/sigstore/protobuf-specs v0.5.2 // indirect
|
||||
github.com/sigstore/sigstore v1.10.11 // indirect
|
||||
github.com/sirupsen/logrus v1.10.2 // indirect
|
||||
github.com/sirupsen/logrus v1.9.4 // indirect
|
||||
github.com/skeema/knownhosts v1.3.1 // indirect
|
||||
github.com/smallnest/ringbuffer v0.0.0-20241116012123-461381446e3d // indirect
|
||||
github.com/smallstep/pkcs7 v0.2.1 // indirect
|
||||
github.com/sorairolake/lzip-go v0.3.8 // indirect
|
||||
github.com/sourcegraph/conc v0.3.1-0.20240121214520-5f936abd7ae8 // indirect
|
||||
github.com/spf13/cast v1.10.0 // indirect
|
||||
github.com/spf13/pflag v1.0.10 // indirect
|
||||
github.com/spf13/viper v1.21.0 // indirect
|
||||
github.com/spiffe/go-spiffe/v2 v2.8.1 // indirect
|
||||
github.com/stefanberger/go-pkcs11uri v0.0.0-20230803200340-78284954bff6 // indirect
|
||||
github.com/stretchr/objx v0.5.3 // indirect
|
||||
github.com/spiffe/go-spiffe/v2 v2.6.0 // indirect
|
||||
github.com/stretchr/objx v0.5.2 // indirect
|
||||
github.com/subosito/gotenv v1.6.0 // indirect
|
||||
github.com/sylabs/sif/v2 v2.24.2 // indirect
|
||||
github.com/sylabs/sif/v2 v2.24.1 // indirect
|
||||
github.com/sylabs/squashfs v1.0.6 // indirect
|
||||
github.com/tchap/go-patricia/v2 v2.3.3 // indirect
|
||||
github.com/therootcompany/xz v1.0.1 // indirect
|
||||
github.com/tidwall/gjson v1.19.0 // indirect
|
||||
github.com/tidwall/match v1.1.1 // indirect
|
||||
github.com/tidwall/pretty v1.2.1 // indirect
|
||||
github.com/tidwall/sjson v1.2.5 // indirect
|
||||
github.com/vbatts/tar-split v0.12.3 // indirect
|
||||
github.com/vbauerster/cupwriter v0.0.4 // indirect
|
||||
github.com/vbauerster/mpb/v8 v8.14.0 // indirect
|
||||
github.com/xanzy/ssh-agent v0.3.3 // indirect
|
||||
github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb // indirect
|
||||
github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 // indirect
|
||||
github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8 // indirect
|
||||
github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e // indirect
|
||||
github.com/youmark/pkcs8 v0.0.0-20240726163527-a2c0da244d78 // indirect
|
||||
github.com/zclconf/go-cty v1.19.0 // indirect
|
||||
github.com/zclconf/go-cty v1.16.3 // indirect
|
||||
go.opencensus.io v0.24.0 // indirect
|
||||
go.opentelemetry.io/auto/sdk v1.2.1 // indirect
|
||||
go.opentelemetry.io/contrib/detectors/gcp v1.44.0 // indirect
|
||||
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.71.0 // indirect
|
||||
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.71.0 // indirect
|
||||
go.opentelemetry.io/otel v1.46.0 // indirect
|
||||
go.opentelemetry.io/otel/metric v1.46.0 // indirect
|
||||
go.opentelemetry.io/otel/sdk v1.46.0 // indirect
|
||||
go.opentelemetry.io/otel/sdk/metric v1.46.0 // indirect
|
||||
go.opentelemetry.io/otel/trace v1.46.0 // indirect
|
||||
go.podman.io/image/v5 v5.41.2 // indirect
|
||||
go.podman.io/storage v1.64.1 // indirect
|
||||
go.opentelemetry.io/contrib/detectors/gcp v1.43.0 // indirect
|
||||
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.68.0 // indirect
|
||||
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.68.0 // indirect
|
||||
go.opentelemetry.io/otel v1.43.0 // indirect
|
||||
go.opentelemetry.io/otel/metric v1.43.0 // indirect
|
||||
go.opentelemetry.io/otel/sdk v1.43.0 // indirect
|
||||
go.opentelemetry.io/otel/sdk/metric v1.43.0 // indirect
|
||||
go.opentelemetry.io/otel/trace v1.43.0 // indirect
|
||||
go.yaml.in/yaml/v4 v4.0.0-rc.2 // indirect
|
||||
go4.org v0.0.0-20230225012048-214862532bf5 // indirect
|
||||
golang.org/x/crypto v0.58.0 // indirect
|
||||
golang.org/x/oauth2 v0.37.0 // indirect
|
||||
golang.org/x/sync v0.24.0 // indirect
|
||||
golang.org/x/term v0.47.0 // indirect
|
||||
golang.org/x/text v0.43.0 // indirect
|
||||
golang.org/x/crypto v0.54.0 // indirect
|
||||
golang.org/x/oauth2 v0.36.0 // indirect
|
||||
golang.org/x/sync v0.22.0 // indirect
|
||||
golang.org/x/sys v0.47.0 // indirect
|
||||
golang.org/x/term v0.45.0 // indirect
|
||||
golang.org/x/text v0.40.0 // indirect
|
||||
golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 // indirect
|
||||
gonum.org/v1/gonum v0.17.0 // indirect
|
||||
google.golang.org/api v0.299.0 // indirect
|
||||
google.golang.org/genproto v0.0.0-20260715232425-e75dac1f907d // indirect
|
||||
google.golang.org/genproto/googleapis/api v0.0.0-20260825221802-da73d73af1c5 // indirect
|
||||
google.golang.org/genproto/googleapis/rpc v0.0.0-20260921155816-b14227669459 // indirect
|
||||
google.golang.org/grpc v1.84.0 // indirect
|
||||
google.golang.org/protobuf v1.36.12 // indirect
|
||||
google.golang.org/api v0.271.0 // indirect
|
||||
google.golang.org/genproto v0.0.0-20260128011058-8636f8732409 // indirect
|
||||
google.golang.org/genproto/googleapis/api v0.0.0-20260414002931-afd174a4e478 // indirect
|
||||
google.golang.org/genproto/googleapis/rpc v0.0.0-20260414002931-afd174a4e478 // indirect
|
||||
google.golang.org/grpc v1.82.1 // indirect
|
||||
google.golang.org/protobuf v1.36.12-0.20260120151049-f2248ac996af // indirect
|
||||
gopkg.in/warnings.v0 v0.1.2 // indirect
|
||||
modernc.org/libc v1.77.1 // indirect
|
||||
modernc.org/libc v1.74.1 // indirect
|
||||
modernc.org/mathutil v1.7.1 // indirect
|
||||
modernc.org/memory v1.12.1 // indirect
|
||||
modernc.org/memory v1.11.0 // indirect
|
||||
)
|
||||
|
||||
retract (
|
||||
|
||||
+20
-49
@@ -8,6 +8,7 @@ REPO="${PROJECT_NAME}"
|
||||
GITHUB_DOWNLOAD_PREFIX=https://github.com/${OWNER}/${REPO}/releases/download
|
||||
INSTALL_SH_BASE_URL=https://get.anchore.io/${PROJECT_NAME}
|
||||
LEGACY_INSTALL_SH_BASE_URL=https://raw.githubusercontent.com/${OWNER}/${PROJECT_NAME}
|
||||
PROGRAM_ARGS=$@
|
||||
|
||||
# signature verification options
|
||||
|
||||
@@ -18,8 +19,6 @@ VERIFY_SIGN=false
|
||||
VERIFY_SIGN_SUPPORTED_VERSION=v0.104.0
|
||||
# this is the earliest tag in the repo where the -v flag was introduced to this install.sh script
|
||||
VERIFY_SIGN_FLAG_VERSION=v1.6.0
|
||||
# releases at or after this version are signed with a sigstore bundle (.sigstore.json); earlier releases ship a separate .sig and .pem
|
||||
VERIFY_SIGN_BUNDLE_VERSION=v1.53.0
|
||||
|
||||
# do not change the name of this parameter (this must always be backwards compatible)
|
||||
DOWNLOAD_TAG_INSTALL_SCRIPT=${DOWNLOAD_TAG_INSTALL_SCRIPT:-true}
|
||||
@@ -387,8 +386,8 @@ download_github_release_checksums_files() (
|
||||
complete_url=$(github_release_asset_url "${download_url}" "${name}" "${version}" "${filename}")
|
||||
output_path="${output_dir}/${complete_filename}"
|
||||
|
||||
http_download "${output_path}" "${complete_url}" "" || return 1
|
||||
asset_file_exists "${output_path}" || return 1
|
||||
http_download "${output_path}" "${complete_url}" ""
|
||||
asset_file_exists "${output_path}"
|
||||
|
||||
log_trace "download_github_release_checksums_files() returned '${output_path}' for file '${complete_filename}'"
|
||||
|
||||
@@ -586,31 +585,28 @@ download_and_install_asset() (
|
||||
install_asset "${asset_filepath}" "${install_path}" "${binary}"
|
||||
)
|
||||
|
||||
# verify_sign [checksums-file-path] [verification-material-flags...]
|
||||
# verify_sign [checksums-file-path] [certificate-reference] [signature-reference] [version]
|
||||
#
|
||||
# attempts verify the signature of the checksums file from the release workflow in Github Actions run against the main branch.
|
||||
# the verification material is either "--bundle <path>" or (for older releases) "--certificate <ref> --signature <ref>".
|
||||
#
|
||||
verify_sign() {
|
||||
checksums_file=$1
|
||||
shift
|
||||
cert_reference=$2
|
||||
sig_reference=$3
|
||||
|
||||
log_trace "verifying artifact ${checksums_file}"
|
||||
log_trace "verifying artifact $1"
|
||||
|
||||
log_file=$(mktemp)
|
||||
|
||||
${COSIGN_BINARY} \
|
||||
verify-blob "$checksums_file" \
|
||||
"$@" \
|
||||
--certificate "$cert_reference" \
|
||||
--signature "$sig_reference" \
|
||||
--certificate-identity "https://github.com/${OWNER}/${REPO}/.github/workflows/release.yaml@refs/heads/main" \
|
||||
--certificate-oidc-issuer "https://token.actions.githubusercontent.com" > "${log_file}" 2>&1
|
||||
|
||||
if [ $? -ne 0 ]; then
|
||||
log_err "$(cat "${log_file}")"
|
||||
# cosign older than v2.4.2 can't read the bundle's verification material and reports a missing cert
|
||||
if [ "$1" = "--bundle" ] && grep -q "bundle does not contain cert" "${log_file}"; then
|
||||
log_err "note: releases >= ${VERIFY_SIGN_BUNDLE_VERSION} are verified with a sigstore bundle, which requires cosign v2.5.0 or newer (check '${COSIGN_BINARY} version')"
|
||||
fi
|
||||
rm -f "${log_file}"
|
||||
return 1
|
||||
fi
|
||||
@@ -634,10 +630,7 @@ download_asset() (
|
||||
|
||||
log_trace "download_asset(url=${download_url}, destination=${destination}, name=${name}, os=${os}, arch=${arch}, version=${version}, format=${format})"
|
||||
|
||||
if ! checksums_filepath=$(download_github_release_checksums "${download_url}" "${name}" "${version}" "${destination}"); then
|
||||
log_err "unable to download checksums file"
|
||||
return 1
|
||||
fi
|
||||
checksums_filepath=$(download_github_release_checksums "${download_url}" "${name}" "${version}" "${destination}")
|
||||
|
||||
log_trace "checksums content:\n$(cat ${checksums_filepath})"
|
||||
|
||||
@@ -649,25 +642,13 @@ download_asset() (
|
||||
fi
|
||||
|
||||
if [ "$VERIFY_SIGN" = true ]; then
|
||||
if compare_semver "${version}" "${VERIFY_SIGN_BUNDLE_VERSION}"; then
|
||||
if ! checksums_bundle_filepath=$(download_github_release_checksums_files "${download_url}" "${name}" "${version}" "${destination}" "checksums.txt.sigstore.json"); then
|
||||
log_err "unable to download checksums signature bundle"
|
||||
return 1
|
||||
fi
|
||||
log_trace "checksums bundle: ${checksums_bundle_filepath}"
|
||||
checksum_sig_file_url=$(github_release_checksums_sig_url "${download_url}" "${name}" "${version}")
|
||||
log_trace "checksums signature url: ${checksum_sig_file_url}"
|
||||
|
||||
set -- --bundle "${checksums_bundle_filepath}"
|
||||
else
|
||||
checksum_sig_file_url=$(github_release_checksums_sig_url "${download_url}" "${name}" "${version}")
|
||||
log_trace "checksums signature url: ${checksum_sig_file_url}"
|
||||
checksums_cert_file_url=$(github_release_checksums_cert_url "${download_url}" "${name}" "${version}")
|
||||
log_trace "checksums certificate url: ${checksums_cert_file_url}"
|
||||
|
||||
checksums_cert_file_url=$(github_release_checksums_cert_url "${download_url}" "${name}" "${version}")
|
||||
log_trace "checksums certificate url: ${checksums_cert_file_url}"
|
||||
|
||||
set -- --certificate "${checksums_cert_file_url}" --signature "${checksum_sig_file_url}"
|
||||
fi
|
||||
|
||||
if ! verify_sign "${checksums_filepath}" "$@"; then
|
||||
if ! verify_sign "${checksums_filepath}" "${checksums_cert_file_url}" "${checksum_sig_file_url}"; then
|
||||
log_err "signature verification failed"
|
||||
return 1
|
||||
fi
|
||||
@@ -676,15 +657,9 @@ download_asset() (
|
||||
|
||||
asset_url="${download_url}/${asset_filename}"
|
||||
asset_filepath="${destination}/${asset_filename}"
|
||||
if ! http_download "${asset_filepath}" "${asset_url}" ""; then
|
||||
log_err "unable to download asset '${asset_url}'"
|
||||
return 1
|
||||
fi
|
||||
http_download "${asset_filepath}" "${asset_url}" ""
|
||||
|
||||
# this is what ties the asset to the (optionally signature verified) checksums file, so a mismatch must stop the install
|
||||
if ! hash_sha256_verify "${asset_filepath}" "${checksums_filepath}"; then
|
||||
return 1
|
||||
fi
|
||||
hash_sha256_verify "${asset_filepath}" "${checksums_filepath}"
|
||||
|
||||
log_trace "download_asset_by_checksums_file() returned '${asset_filepath}'"
|
||||
|
||||
@@ -790,7 +765,7 @@ prep_signature_verification() {
|
||||
fi
|
||||
}
|
||||
|
||||
parse_install_args() {
|
||||
main() (
|
||||
# parse arguments
|
||||
|
||||
# note: never change default install directory (this must always be backwards compatible)
|
||||
@@ -830,11 +805,6 @@ EOF
|
||||
|
||||
set +u
|
||||
tag=$1
|
||||
set -u
|
||||
}
|
||||
|
||||
main() (
|
||||
parse_install_args "$@"
|
||||
|
||||
if [ -z "${tag}" ]; then
|
||||
log_info "checking github for the current release tag"
|
||||
@@ -842,6 +812,7 @@ main() (
|
||||
else
|
||||
log_info "checking github for release tag='${tag}'"
|
||||
fi
|
||||
set -u
|
||||
|
||||
if ! tag=$(get_release_tag "${OWNER}" "${REPO}" "${tag}"); then
|
||||
log_err "unable to find tag='${tag}'"
|
||||
@@ -872,7 +843,7 @@ main() (
|
||||
log_warn "failed to fetch from ${INSTALL_SH_BASE_URL}, trying fallback URL"
|
||||
install_script=$(http_copy "${LEGACY_INSTALL_SH_BASE_URL}/${tag}/install.sh" "")
|
||||
fi
|
||||
echo "${install_script}" | sh -s -- "$@"
|
||||
echo "${install_script}" | sh -s -- ${PROGRAM_ARGS}
|
||||
exit $?
|
||||
fi
|
||||
|
||||
|
||||
@@ -1,106 +0,0 @@
|
||||
# internal/archive
|
||||
|
||||
Opens one archive as its own filesystem so the ordinary catalogers can run against its contents.
|
||||
The archive cataloger task (`internal/task/archive_tasks.go`) drives the recursion; this package
|
||||
does everything for a single archive.
|
||||
|
||||
## Flow
|
||||
|
||||
```
|
||||
Extract(ctx, reader, archiveCoordinates, limiter, exclusions) -> *Resolver
|
||||
acquire read in place if the reader is already seekable, else hold in memory / spill to disk
|
||||
identifyFormat internal/file.IdentifyArchive first, end-of-central-directory fallback for stub-prefixed zips
|
||||
Resolver.extract every entry -> entryHeader -> Resolver.add (excluded entries are never stored)
|
||||
digestsOf SHA-1 of the archive itself
|
||||
```
|
||||
|
||||
`Extract` returns `nil` for content that is not an archive (a bare compression format such as a
|
||||
gzipped file that is not a tar counts as not an archive), `ErrDiskLimitReached` when the archive's
|
||||
own bytes cannot be placed, and otherwise a `Resolver` whose `Cleanup` releases everything.
|
||||
|
||||
## Resolver
|
||||
|
||||
One type does the work: `Resolver` is fed entries by `Extract`, holds their content, indexes them,
|
||||
and implements `file.Resolver` over the result. Each entry is a `node` carrying its header and a
|
||||
`blob`; the archive's own bytes, when they have to be held at all, are one more blob.
|
||||
|
||||
Every location it returns extends the archive's coordinates: `RealPath` and `FileSystemID` stay
|
||||
those of the outermost archive on the scanned filesystem, and `FullPath` is the `!/`-delimited
|
||||
chain down to the entry (`/app.war!/WEB-INF/lib/dep.jar!/META-INF/MANIFEST.MF`). `LocalPath` is the
|
||||
entry's path within this archive.
|
||||
|
||||
## Where bytes live
|
||||
|
||||
Blobs are held in memory while the memory limit admits them and in one spill file per archive
|
||||
(`archive-spill-*`) once it does not; when a chunk is refused, everything held in memory moves to
|
||||
the file first. The file is created on first use, so most archives never touch the filesystem, and
|
||||
blobs are located in it by offset with no framing of their own. It goes under the scan's temp root
|
||||
when the context carries one (`internal/tmpdir`), otherwise the system temp directory.
|
||||
|
||||
Anything already random-access (an `*os.File`, or a nested archive read out of its parent's
|
||||
resolver) is used in place and costs nothing, so only a top-level stream is ever copied. Its memory
|
||||
is refunded once it is extracted and digested; bytes it spilled wait for `Cleanup` with the entries.
|
||||
|
||||
## Limits
|
||||
|
||||
`Limits` bound what one scan holds *at once*; each falls as archives are released. For both limits,
|
||||
positive is the bound, zero forbids the resource, negative is unbounded.
|
||||
|
||||
- `Limiter` is scan-wide; a `charge` is one archive's draw on it. Bytes are charged as they land,
|
||||
never from declared sizes.
|
||||
- Memory refused → content moves to disk. Disk refused → the archive's own bytes cannot be placed
|
||||
(`ErrDiskLimitReached`, archive skipped) or an entry cannot be stored (`Resolver.Truncated`, the
|
||||
entries stored so far are still cataloged).
|
||||
- Index bookkeeping is approximated as `approxIndexBytesPerEntry` (2 KiB) per node, listed or
|
||||
implied by an entry's path, plus name length. It lives in memory, so it is charged to the memory
|
||||
limit, spilling held content to make room; only a zero memory limit lets it fall back to the disk
|
||||
budget so that configuration still indexes. An entry whose index cost is refused is dropped like
|
||||
one whose content is, and marks the resolver `Truncated`.
|
||||
- `Budget` bounds the bytes decompressed under one top-level archive, summed over everything nested
|
||||
in it. Unlike `Limits` it never refills, so a recursive bomb whose every level fits the limits
|
||||
still ends (`errBudgetSpent`, the resolver is `Truncated`).
|
||||
- `Limiter.Peak` reports the most held at once over the scan, for reporting.
|
||||
|
||||
## Hostile archives
|
||||
|
||||
Nothing an archive declares is trusted. Content is charged chunk by chunk as it decompresses, so a
|
||||
deflate or gzip bomb stops at the disk limit with the entry dropped whole. Every node is charged
|
||||
against the memory limit, including directories implied by deep paths, so an archive of millions of
|
||||
tiny entries or of a few entries with 4 KiB names stops at the memory limit. A recursive bomb stops
|
||||
at the `Budget`. Names longer than PATH_MAX are refused, and nesting stops at the task's `MaxDepth`.
|
||||
See the bomb tests in `extract_test.go`.
|
||||
|
||||
## Lookup
|
||||
|
||||
Entries are indexed in a stereoscope `filetree.FileTree` + `filetree.Index` and searched with
|
||||
`filetree.SearchContext`, the same index the directory and image resolvers use. A query may name an
|
||||
entry by its full path or by its `LocalPath` (leading slash optional); a full path into another
|
||||
archive in the scan names nothing. Directories implied by entry paths exist for `HasPath` but are
|
||||
never files. The last entry at a path wins over earlier
|
||||
duplicates, as it does on extraction; a file and a directory at one path keep the first.
|
||||
|
||||
Links resolve inside the archive only, as the file tree resolves them: a hard link names its target
|
||||
from the archive root. A lookup matching a link and its target returns one location, with
|
||||
`AccessPath` the full path matched and `FullPath` that of the file read. Dangling and cyclic links are not
|
||||
files. `AllLocations` enumerates every non-directory path, links included. Answers are sorted by path.
|
||||
|
||||
## Other pieces
|
||||
|
||||
- `Exclusions`: the scan's exclusion patterns, of which only the `**/` shape means the same thing
|
||||
re-rooted at an archive. Matched against each entry path and its parent directories at extraction
|
||||
time. `syft.CreateSBOM` passes the source's patterns to the task, which builds one `Exclusions`
|
||||
for the scan.
|
||||
- `Traversal`: put on the context by the task so a cataloger (java) knows which archive it is inside,
|
||||
what its digests are, and which packages describe its parent. `VirtualPath(location)` converts a
|
||||
location's `FullPath` to the java cataloger's colon-delimited `app.war:WEB-INF/lib/inner.jar`
|
||||
form, escaping `%` and `:` in entry names. `Container` and `EntryCoordinates` split and build the
|
||||
coordinates of files within archives (the SPDX and CycloneDX formats use them too); `OwnerContainers` and
|
||||
`OwnedCoordinates` map package-owned file paths to coordinates in the right archive.
|
||||
- `WithNestedCataloging` / `NestedCatalogingEnabled`: `syft.CreateSBOM` marks the context when the
|
||||
archive task is running, so the java cataloger leaves nested archives to the task instead of
|
||||
opening them itself.
|
||||
- `entryHeader`: keeps regular files, directories and links only (no devices or fifos), cleans entry names to
|
||||
archive-relative paths, refuses names longer than PATH_MAX or containing control characters, line
|
||||
separators or the `!/` archive path separator (marking the resolver truncated), and drops header
|
||||
fields nothing reads (PAX records, user and group names, sub-second and access times) since the
|
||||
header is held for the life of the archive.
|
||||
@@ -1,38 +0,0 @@
|
||||
package archive
|
||||
|
||||
import (
|
||||
"strings"
|
||||
|
||||
"github.com/bmatcuk/doublestar/v4"
|
||||
)
|
||||
|
||||
// Exclusions are the scan's exclusion patterns that apply inside archives.
|
||||
//
|
||||
// A scan accepts three pattern shapes: `./x` anchored at the scan root, `*/x` exactly one level below
|
||||
// it, and `**/x` at any depth. Only the last means the same thing when re-rooted at an archive, so
|
||||
// only it reaches inside one.
|
||||
type Exclusions []string
|
||||
|
||||
func NewExclusions(scanPatterns []string) Exclusions {
|
||||
var out Exclusions
|
||||
for _, pattern := range scanPatterns {
|
||||
if strings.HasPrefix(pattern, "**/") {
|
||||
// a trailing slash reads as "a directory" but makes doublestar match nothing (issue #4839)
|
||||
out = append(out, strings.TrimSuffix(pattern, "/"))
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// Excludes reports whether an archive-relative path, or any directory above it, matches a pattern.
|
||||
// Directories are checked because an archive need not list a directory before the files in it.
|
||||
func (e Exclusions) Excludes(entryPath string) bool {
|
||||
for _, pattern := range e {
|
||||
for end := len(entryPath); end > 0; end = strings.LastIndexByte(entryPath[:end], '/') {
|
||||
if matched, _ := doublestar.Match(pattern, entryPath[:end]); matched {
|
||||
return true
|
||||
}
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
@@ -1,56 +0,0 @@
|
||||
package archive
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
)
|
||||
|
||||
func TestNewExclusions(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
given []string
|
||||
want Exclusions
|
||||
}{
|
||||
{name: "nothing configured"},
|
||||
{name: "root-anchored does not reach inside", given: []string{"./nested/marker.txt"}},
|
||||
{name: "one level down is anchored too", given: []string{"*/vendor"}},
|
||||
{name: "any depth reaches inside", given: []string{"**/*.rpm"}, want: Exclusions{"**/*.rpm"}},
|
||||
{name: "mixed keeps only the any-depth patterns", given: []string{"./a", "**/b", "*/c", "**/d"}, want: Exclusions{"**/b", "**/d"}},
|
||||
// a trailing slash makes doublestar match nothing (issue #4839)
|
||||
{name: "a trailing slash is trimmed", given: []string{"**/vendor/"}, want: Exclusions{"**/vendor"}},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
assert.Equal(t, tt.want, NewExclusions(tt.given))
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestExclusions_Excludes(t *testing.T) {
|
||||
exclusions := NewExclusions([]string{"**/vendor", "**/*.rpm"})
|
||||
|
||||
tests := []struct {
|
||||
path string
|
||||
want bool
|
||||
}{
|
||||
{"vendor", true},
|
||||
{"a/vendor", true},
|
||||
{"vendor/x/y.go", true},
|
||||
{"a/vendor/y.go", true},
|
||||
{"pkg.rpm", true},
|
||||
{"deep/down/pkg.rpm", true},
|
||||
{"keep/vendored.txt", false},
|
||||
{"vendor-ish", false},
|
||||
{"keep.txt", false},
|
||||
{"PKG.RPM", false},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.path, func(t *testing.T) {
|
||||
assert.Equal(t, tt.want, exclusions.Excludes(tt.path))
|
||||
})
|
||||
}
|
||||
|
||||
assert.False(t, NewExclusions(nil).Excludes("anything"))
|
||||
assert.False(t, Exclusions{"["}.Excludes("keep.txt"), "a malformed pattern excludes nothing")
|
||||
}
|
||||
@@ -1,263 +0,0 @@
|
||||
package archive
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"path"
|
||||
|
||||
"github.com/mholt/archives"
|
||||
|
||||
"github.com/anchore/syft/internal"
|
||||
intFile "github.com/anchore/syft/internal/file"
|
||||
"github.com/anchore/syft/internal/log"
|
||||
"github.com/anchore/syft/syft/file"
|
||||
)
|
||||
|
||||
// ErrUnsupportedFormat is returned for an archive in a format that is not extracted (see supportedExtractor).
|
||||
var ErrUnsupportedFormat = errors.New("archive format is not supported")
|
||||
|
||||
// Extract opens the archive read from r as its own filesystem, holding its bytes and entries within
|
||||
// the limiter's bounds. It returns nil when the content is not an archive, ErrUnsupportedFormat when it
|
||||
// is one that is not extracted, and ErrDiskLimitReached when the archive's own bytes cannot be placed
|
||||
// within the disk limit. The caller must Cleanup the result.
|
||||
//
|
||||
// archiveCoordinates are those of the archive file itself; every location the resolver returns
|
||||
// extends them with its entry path (see file.Coordinates).
|
||||
func Extract(ctx context.Context, r io.Reader, archiveCoordinates file.Coordinates, limiter *Limiter, exclusions Exclusions) (*Resolver, error) {
|
||||
charge := limiter.charge()
|
||||
resolver := newResolver(ctx, archiveCoordinates, charge)
|
||||
archivePath := archiveCoordinates.FullNestedPath()
|
||||
|
||||
// deferred so that a panic in a decoder still gives back the charge and removes any spill file
|
||||
succeeded := false
|
||||
defer func() {
|
||||
if !succeeded {
|
||||
resolver.Cleanup()
|
||||
}
|
||||
}()
|
||||
|
||||
content, release, err := resolver.acquire(r)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
// the archive's own bytes are needed only until they are extracted and digested
|
||||
defer release()
|
||||
|
||||
format, err := identifyFormat(ctx, path.Base(archiveCoordinates.LocalPath()), content)
|
||||
if format == nil || err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if err := resolver.extract(ctx, format, content, exclusions); err != nil {
|
||||
return nil, fmt.Errorf("unable to extract archive %q: %w", archivePath, err)
|
||||
}
|
||||
resolver.Digests = digestsOf(ctx, content, archivePath)
|
||||
succeeded = true
|
||||
return resolver, nil
|
||||
}
|
||||
|
||||
// acquire makes the archive's own bytes random-access, which identifying, extracting and digesting
|
||||
// them needs. Bytes that already are, such as a file on disk or an entry of a parent archive, are
|
||||
// read in place and cost nothing. Anything else is held within the limits like an entry, and the
|
||||
// returned release gives back what memory it takes; bytes spilled to disk stay until Cleanup.
|
||||
// Reaching the disk limit yields ErrDiskLimitReached.
|
||||
func (r *Resolver) acquire(content io.Reader) (ReaderAtSeeker, func(), error) {
|
||||
if ras, ok := content.(ReaderAtSeeker); ok {
|
||||
return ras, func() {}, nil
|
||||
}
|
||||
var b blob
|
||||
if err := r.put(&b, content); err != nil {
|
||||
if errors.Is(err, ErrDiskLimitReached) {
|
||||
return nil, nil, err
|
||||
}
|
||||
return nil, nil, fmt.Errorf("unable to read archive content: %w", err)
|
||||
}
|
||||
return r.open(&b), func() { r.discard(&b) }, nil
|
||||
}
|
||||
|
||||
// identifyFormat returns the format able to extract this content, or nil when it is not an archive.
|
||||
//
|
||||
// Identification from the head of the stream wins: a tar holding one jar ends in that jar's central
|
||||
// directory, and reading the tar as a zip would lose the rest. The end of the stream is consulted
|
||||
// only when the head identifies nothing supported, which is how a zip appended to a launcher script (a
|
||||
// Spring Boot executable jar) is found, and how a java resource adapter (a zip named .rar) is read.
|
||||
func identifyFormat(ctx context.Context, name string, content ReaderAtSeeker) (archives.Extractor, error) {
|
||||
if _, err := content.Seek(0, io.SeekStart); err != nil {
|
||||
return nil, nil
|
||||
}
|
||||
format, _, err := intFile.IdentifyArchive(ctx, name, content)
|
||||
if err == nil {
|
||||
if extractor := supportedExtractor(format); extractor != nil {
|
||||
return extractor, nil
|
||||
}
|
||||
}
|
||||
if HasZipEndOfCentralDirectory(content) {
|
||||
return archives.Zip{}, nil
|
||||
}
|
||||
if err == nil && holdsFiles(format) {
|
||||
return nil, fmt.Errorf("%w: %s", ErrUnsupportedFormat, format.MediaType())
|
||||
}
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
// holdsFiles reports whether the format is an archive of files rather than a bare compression format,
|
||||
// such as a gzipped man page, which holds one stream and is not cataloged as an archive.
|
||||
func holdsFiles(format archives.Format) bool {
|
||||
switch f := format.(type) {
|
||||
case archives.CompressedArchive:
|
||||
return f.Extraction != nil
|
||||
case archives.Extractor:
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// supportedExtractor admits only the zip and tar families, the tar family optionally compressed. Other
|
||||
// formats mholt identifies, such as 7z and rar, pull in decoders that have not been exercised against
|
||||
// untrusted input here, so they are not opened until they are deliberately supported. A bare
|
||||
// compression format, such as a gzipped file that is not a tar, is not extractable either.
|
||||
func supportedExtractor(format archives.Format) archives.Extractor {
|
||||
switch f := format.(type) {
|
||||
case archives.Zip:
|
||||
return f
|
||||
case archives.Tar:
|
||||
return f
|
||||
case archives.CompressedArchive:
|
||||
if _, ok := f.Extraction.(archives.Tar); ok {
|
||||
return f
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// extract adds every entry of the archive. Reaching the disk limit stops the walk and marks the
|
||||
// resolver truncated rather than failing.
|
||||
func (r *Resolver) extract(ctx context.Context, format archives.Extractor, content ReaderAtSeeker, exclusions Exclusions) error {
|
||||
if _, err := content.Seek(0, io.SeekStart); err != nil {
|
||||
return err
|
||||
}
|
||||
var source io.Reader = content
|
||||
if ca, ok := format.(archives.CompressedArchive); ok && ca.Compression != nil {
|
||||
// a compressed tar decompresses everything to reach each header, including entries skipped whole
|
||||
// and whatever an entry leaves unread, so the budget is charged for every byte the decoder
|
||||
// produces rather than for what is kept. A zip or a plain tar seeks past what it skips
|
||||
rc, err := ca.OpenReader(content)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer internal.CloseAndLogError(rc, "decompressor")
|
||||
source, format, r.metered = &meteredReader{ctx: ctx, reader: rc, budget: r.budget}, ca.Extraction, true
|
||||
}
|
||||
err := format.Extract(ctx, source, func(_ context.Context, f archives.FileInfo) error {
|
||||
hdr, ok := r.entryHeader(f)
|
||||
if !ok || exclusions.Excludes(hdr.Name) {
|
||||
return nil
|
||||
}
|
||||
if !f.Mode().IsRegular() {
|
||||
return r.add(hdr, nil)
|
||||
}
|
||||
entry, err := f.Open()
|
||||
if err != nil {
|
||||
log.WithFields("entry", hdr.Name, "error", err).Trace("unable to open archive entry, skipping it")
|
||||
return nil
|
||||
}
|
||||
defer func() {
|
||||
if err := entry.Close(); err != nil {
|
||||
log.WithFields("entry", hdr.Name, "error", err).Trace("unable to close archive entry")
|
||||
}
|
||||
}()
|
||||
return r.add(hdr, entry)
|
||||
})
|
||||
r.finish()
|
||||
if errors.Is(err, ErrDiskLimitReached) {
|
||||
r.truncate(fmt.Errorf("extraction stopped: %w", err))
|
||||
return nil
|
||||
}
|
||||
if errors.Is(err, errBudgetSpent) {
|
||||
r.truncate(fmt.Errorf("extraction stopped, possible decompression bomb: %w", err))
|
||||
return nil
|
||||
}
|
||||
if err != nil && ctx.Err() == nil && r.entries > 0 {
|
||||
// the stream broke partway (a truncated .tar.gz); what was read before that is still good
|
||||
r.truncate(fmt.Errorf("extraction stopped early: %w", err))
|
||||
return nil
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
// archiveDigestHashes are the digests taken of an archive as a whole: SHA-1, matching what the java
|
||||
// cataloger reports for a jar.
|
||||
var archiveDigestHashes = []crypto.Hash{crypto.SHA1}
|
||||
|
||||
// digestsOf digests the archive's own bytes while they are in hand. A failure is logged and returns
|
||||
// nothing rather than discarding an archive that extracted fine.
|
||||
func digestsOf(ctx context.Context, content ReaderAtSeeker, archivePath string) []file.Digest {
|
||||
if _, err := content.Seek(0, io.SeekStart); err != nil {
|
||||
log.WithFields("archive", archivePath, "error", err).Trace("unable to rewind archive content for digests")
|
||||
return nil
|
||||
}
|
||||
digests, err := intFile.NewDigestsFromFile(ctx, io.NopCloser(content), archiveDigestHashes)
|
||||
if err != nil {
|
||||
log.WithFields("archive", archivePath, "error", err).Trace("unable to digest archive")
|
||||
return nil
|
||||
}
|
||||
return digests
|
||||
}
|
||||
|
||||
var (
|
||||
// zipLocalFileHeaderSignature starts every entry in a zip.
|
||||
zipLocalFileHeaderSignature = []byte{'P', 'K', 0x03, 0x04}
|
||||
|
||||
// zipEOCDSignature starts the end-of-central-directory record that closes every zip. Readers
|
||||
// locate the central directory from it, which is why a zip with data prepended still opens.
|
||||
zipEOCDSignature = []byte{'P', 'K', 0x05, 0x06}
|
||||
)
|
||||
|
||||
const (
|
||||
// maxAppendedArchiveStubBytes is how far into a file an appended zip's first entry is looked for. A
|
||||
// Spring Boot launcher stub is about 9 KB.
|
||||
maxAppendedArchiveStubBytes = 64 * 1024
|
||||
|
||||
// maxZipEOCDSearch is how far from the end the end-of-central-directory record can start: the 22
|
||||
// byte record plus a comment of up to 65535 bytes.
|
||||
maxZipEOCDSearch = 22 + 65535
|
||||
)
|
||||
|
||||
// MayHideAnAppendedArchive reports whether a zip entry signature appears in the head of the stream,
|
||||
// which a zip appended to a launcher stub does and a file that merely has an archive's extension
|
||||
// almost never does. The returned reader replays what was consumed.
|
||||
func MayHideAnAppendedArchive(r io.Reader) (io.Reader, bool) {
|
||||
head := make([]byte, maxAppendedArchiveStubBytes)
|
||||
n, err := io.ReadFull(r, head)
|
||||
if err != nil && !errors.Is(err, io.EOF) && !errors.Is(err, io.ErrUnexpectedEOF) {
|
||||
return r, false
|
||||
}
|
||||
head = head[:n]
|
||||
found := bytes.Contains(head, zipLocalFileHeaderSignature)
|
||||
|
||||
if seeker, ok := r.(io.Seeker); ok {
|
||||
if _, err := seeker.Seek(0, io.SeekStart); err == nil {
|
||||
return r, found
|
||||
}
|
||||
}
|
||||
return io.MultiReader(bytes.NewReader(head), r), found
|
||||
}
|
||||
|
||||
// HasZipEndOfCentralDirectory reports whether the content ends in a zip end-of-central-directory
|
||||
// record, the one part of a zip whose position is known without parsing it.
|
||||
func HasZipEndOfCentralDirectory(r ReaderAtSeeker) bool {
|
||||
size, err := r.Seek(0, io.SeekEnd)
|
||||
if err != nil || size < int64(len(zipEOCDSignature)) {
|
||||
return false
|
||||
}
|
||||
window := min(size, int64(maxZipEOCDSearch))
|
||||
buf := make([]byte, window)
|
||||
if _, err := r.ReadAt(buf, size-window); err != nil && !errors.Is(err, io.EOF) {
|
||||
return false
|
||||
}
|
||||
return bytes.Contains(buf, zipEOCDSignature)
|
||||
}
|
||||
@@ -1,706 +0,0 @@
|
||||
package archive
|
||||
|
||||
import (
|
||||
"archive/tar"
|
||||
"archive/zip"
|
||||
"bytes"
|
||||
"compress/gzip"
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"io/fs"
|
||||
"math/rand"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/mholt/archives"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
stereoscopeFile "github.com/anchore/stereoscope/pkg/file"
|
||||
"github.com/anchore/syft/syft/file"
|
||||
)
|
||||
|
||||
var sampleFiles = map[string]string{
|
||||
"file1.txt": "content1",
|
||||
"dir/file2.txt": "content2",
|
||||
}
|
||||
|
||||
func Test_identifyFormat(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
launcher := append([]byte("#!/bin/bash\nexec java -jar \"$0\" \"$@\"\nexit 0\n"), zipBytes(t, sampleFiles)...)
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
data []byte
|
||||
want archives.Extractor
|
||||
unsupported bool
|
||||
}{
|
||||
{"test.zip", zipBytes(t, sampleFiles), archives.Zip{}, false},
|
||||
{"test.tar.gz", tarGzBytes(t, sampleFiles), archives.CompressedArchive{}, false},
|
||||
{"plain.txt", []byte("just text"), nil, false},
|
||||
// identified by mholt, but not opened until their decoders are deliberately supported
|
||||
{"test.7z", []byte("7z\xbc\xaf\x27\x1c\x00\x04"), nil, true},
|
||||
{"test.rar", []byte("Rar!\x1a\x07\x01\x00"), nil, true},
|
||||
// a bare compression format holds one stream, not files
|
||||
{"page.1.gz", gzBytes(t, "a man page"), nil, false},
|
||||
// a java resource adapter archive is a zip that shares rar's extension
|
||||
{"adapter.rar", zipBytes(t, sampleFiles), archives.Zip{}, false},
|
||||
// the name can be claimed by identification, so the probe alone must not admit prose
|
||||
{"app.jar", []byte("plain text, no central directory anywhere in it"), nil, false},
|
||||
{"bundle", []byte("plain text, no central directory anywhere in it"), nil, false},
|
||||
// content sniffing types this as a shell script; only its tail says zip
|
||||
{"app.jar", launcher, archives.Zip{}, false},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(fmt.Sprintf("%s/%d bytes", tt.name, len(tt.data)), func(t *testing.T) {
|
||||
got, err := identifyFormat(ctx, tt.name, bytes.NewReader(tt.data))
|
||||
if tt.unsupported {
|
||||
assert.ErrorIs(t, err, ErrUnsupportedFormat)
|
||||
} else {
|
||||
assert.NoError(t, err)
|
||||
}
|
||||
if tt.want == nil {
|
||||
assert.Nil(t, got)
|
||||
return
|
||||
}
|
||||
assert.IsType(t, tt.want, got)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestResolver_extract_storesEveryEntryInMemoryWhenUnbounded(t *testing.T) {
|
||||
for name, data := range map[string][]byte{
|
||||
"test.zip": zipBytes(t, sampleFiles),
|
||||
"test.tar.gz": tarGzBytes(t, sampleFiles),
|
||||
} {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
r, root := extractBytes(t, data, name, nil)
|
||||
assert.NoError(t, r.Truncated)
|
||||
assert.Zero(t, r.written)
|
||||
assert.Empty(t, filesIn(t, root))
|
||||
|
||||
entries := readStore(t, r)
|
||||
assert.Equal(t, "content1", entries["file1.txt"].body)
|
||||
assert.Equal(t, "content2", entries["dir/file2.txt"].body)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestResolver_extract_diskLimitTruncatesAtTheEntryThatDoesNotFit(t *testing.T) {
|
||||
// each entry costs its index estimate plus its content on disk
|
||||
record := approxIndexBytes(tar.Header{Name: "file1.txt"})
|
||||
data := zipBytes(t, map[string]string{"file1.txt": "a", "file2.txt": "b", "file3.txt": "c"})
|
||||
|
||||
r, _ := extractBytes(t, data, "test.zip", diskCharge(2*(record+1)+record))
|
||||
|
||||
assert.Error(t, r.Truncated)
|
||||
assert.Equal(t, []string{"file1.txt", "file2.txt"}, storedNames(t, r), "the refused entry is not stored")
|
||||
entries := readStore(t, r)
|
||||
assert.Equal(t, "a", entries["file1.txt"].body)
|
||||
assert.Equal(t, "b", entries["file2.txt"].body)
|
||||
}
|
||||
|
||||
func TestResolver_extract_diskLimitBitesMidEntry(t *testing.T) {
|
||||
// a tar is walked entry by entry, so the limit can bite part way through an entry
|
||||
data := tarGzBytes(t, map[string]string{
|
||||
"a/first.txt": strings.Repeat("a", 40*1024),
|
||||
"b/second.txt": strings.Repeat("b", 40*1024),
|
||||
})
|
||||
const budget = 80 * 1024
|
||||
charge := diskCharge(budget)
|
||||
|
||||
r, root := extractBytes(t, data, "test.tar.gz", charge)
|
||||
|
||||
assert.Error(t, r.Truncated)
|
||||
assert.Equal(t, []string{"a", "a/first.txt", "b"}, storedNames(t, r), "the directory a refused entry implied stays")
|
||||
assert.Greater(t, r.written, int64(40*1024), "the first entry landed whole and the second part way")
|
||||
_, disk := charge.held()
|
||||
assert.LessOrEqual(t, disk, int64(budget))
|
||||
assert.Greater(t, disk, r.written, "the index estimate is charged to disk too")
|
||||
|
||||
info, err := os.Stat(filepath.Join(root, spillFile(t, root)))
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, r.written, info.Size())
|
||||
}
|
||||
|
||||
func TestResolver_extract_directoryOnlyArchivesAreBoundedByTheIndexEstimate(t *testing.T) {
|
||||
var entries []testEntry
|
||||
for i := range 50 {
|
||||
entries = append(entries, testEntry{name: fmt.Sprintf("d%04d/", i), dir: true})
|
||||
}
|
||||
room := approxIndexBytes(tar.Header{Name: "d0000"}) * 10
|
||||
|
||||
for name, data := range map[string][]byte{
|
||||
"dirs.zip": zipFromEntries(t, entries),
|
||||
"dirs.tar.gz": tarGzFromEntries(t, entries),
|
||||
} {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
r, _ := extractBytes(t, data, name, diskCharge(room))
|
||||
assert.Error(t, r.Truncated)
|
||||
assert.Len(t, storedNames(t, r), 10)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestResolver_extract_entryNamesAreCleanedAndNothingIsWrittenUnderThem(t *testing.T) {
|
||||
entries := []testEntry{
|
||||
{name: "../../etc/passwd", body: "evil"},
|
||||
{name: "/etc/hosts", body: "pwned"},
|
||||
{name: "META-INF/", dir: true, mode: 0o555},
|
||||
{name: "META-INF/MANIFEST.MF", body: "Manifest-Version: 1.0"},
|
||||
{name: "passwd", link: "../../../../etc/passwd"},
|
||||
{name: "shadow", link: "/etc/shadow"},
|
||||
{name: "bin/link.txt", link: "../lib/real.txt"},
|
||||
}
|
||||
|
||||
for name, data := range map[string][]byte{
|
||||
"evil.zip": zipFromEntries(t, entries),
|
||||
"evil.tar.gz": tarGzFromEntries(t, entries),
|
||||
} {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
r, root := extractBytes(t, data, name, nil)
|
||||
require.NoError(t, r.Truncated)
|
||||
|
||||
assert.Equal(t, []string{"META-INF", "META-INF/MANIFEST.MF", "bin", "bin/link.txt", "etc", "etc/hosts", "etc/passwd", "passwd", "shadow"},
|
||||
storedNames(t, r))
|
||||
assert.Empty(t, filesIn(t, root), "entries live in the store, never under their own names")
|
||||
assert.NoFileExists(t, filepath.Join(t.TempDir(), "..", "etc", "passwd"))
|
||||
|
||||
got := readStore(t, r)
|
||||
assert.Equal(t, "evil", got["etc/passwd"].body)
|
||||
assert.Equal(t, "pwned", got["etc/hosts"].body)
|
||||
assert.Equal(t, stereoscopeFile.TypeDirectory, got["META-INF"].kind)
|
||||
assert.Equal(t, fs.FileMode(0o555), got["META-INF"].mode.Perm())
|
||||
assert.Equal(t, "Manifest-Version: 1.0", got["META-INF/MANIFEST.MF"].body)
|
||||
|
||||
// links are headers carrying their target verbatim; the index resolves them inside the archive
|
||||
for _, link := range []string{"passwd", "shadow", "bin/link.txt"} {
|
||||
assert.Equal(t, stereoscopeFile.TypeSymLink, got[link].kind)
|
||||
assert.Empty(t, got[link].body)
|
||||
}
|
||||
assert.Equal(t, "../../../../etc/passwd", got["passwd"].linkTarget)
|
||||
assert.Equal(t, "/etc/shadow", got["shadow"].linkTarget)
|
||||
assert.Equal(t, "../lib/real.txt", got["bin/link.txt"].linkTarget)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestResolver_extract_dataEdgeCases(t *testing.T) {
|
||||
t.Run("an empty archive is not an error", func(t *testing.T) {
|
||||
r, _ := extractBytes(t, zipBytes(t, map[string]string{}), "test.zip", nil)
|
||||
assert.Zero(t, r.entries)
|
||||
assert.NoError(t, r.Truncated)
|
||||
})
|
||||
|
||||
t.Run("a zero-byte entry costs no disk", func(t *testing.T) {
|
||||
r, _ := extractBytes(t, zipBytes(t, map[string]string{"empty.txt": ""}), "test.zip", memCharge(0))
|
||||
assert.Equal(t, []string{"empty.txt"}, storedNames(t, r))
|
||||
assert.Zero(t, r.written)
|
||||
assert.Empty(t, readStore(t, r)["empty.txt"].body)
|
||||
})
|
||||
|
||||
t.Run("unicode entry names survive", func(t *testing.T) {
|
||||
names := map[string]string{
|
||||
"café/naïve.txt": "accents",
|
||||
"日本語/ファイル.txt": "japanese",
|
||||
"emoji-🎉.txt": "emoji",
|
||||
}
|
||||
r, _ := extractBytes(t, zipBytes(t, names), "test.zip", nil)
|
||||
entries := readStore(t, r)
|
||||
for name, want := range names {
|
||||
assert.Equal(t, want, entries[name].body, name)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("an entry name with too many path components is skipped and reported", func(t *testing.T) {
|
||||
atCap := strings.Repeat("a/", maxEntryNameDepth-1) + "f"
|
||||
data := zipBytes(t, map[string]string{
|
||||
atCap: "at the cap",
|
||||
strings.Repeat("b/", maxEntryNameDepth) + "f": "over the cap",
|
||||
})
|
||||
r, _ := extractBytes(t, data, "test.zip", nil)
|
||||
names := storedNames(t, r)
|
||||
assert.Contains(t, names, atCap)
|
||||
assert.Len(t, names, maxEntryNameDepth, "only the entry at the cap and its implied directories")
|
||||
assert.ErrorContains(t, r.Truncated, "path components")
|
||||
})
|
||||
|
||||
t.Run("an entry whose chain is longer than a path is skipped and reported", func(t *testing.T) {
|
||||
// the chain is the archive's own path, the separator, then the name
|
||||
fits := strings.Repeat("a", MaxPathBytes-len("test-archive!/"))
|
||||
data := zipBytes(t, map[string]string{
|
||||
fits: "at the cap",
|
||||
fits + "b": "over the cap",
|
||||
})
|
||||
r, _ := extractBytes(t, data, "test.zip", nil)
|
||||
assert.Equal(t, []string{fits}, storedNames(t, r))
|
||||
assert.ErrorContains(t, r.Truncated, "archive chain")
|
||||
})
|
||||
|
||||
t.Run("an over-long entry name is skipped", func(t *testing.T) {
|
||||
data := zipBytes(t, map[string]string{
|
||||
strings.Repeat("a/", maxEntryNameBytes) + "f": "deep",
|
||||
"ok.txt": "fine",
|
||||
})
|
||||
r, _ := extractBytes(t, data, "test.zip", nil)
|
||||
assert.Equal(t, []string{"ok.txt"}, storedNames(t, r))
|
||||
})
|
||||
}
|
||||
|
||||
func TestResolver_extract_unsafeEntryNamesAreRefusedAndReported(t *testing.T) {
|
||||
jar := string(zipBytes(t, map[string]string{"META-INF/MANIFEST.MF": "real"}))
|
||||
tests := []struct {
|
||||
name string
|
||||
forged string
|
||||
}{
|
||||
{name: "archive path separator", forged: "a.jar!/META-INF/MANIFEST.MF"},
|
||||
{name: "line break", forged: "x\nlayerID: sha256:forged"},
|
||||
{name: "unicode line separator", forged: "x\u2028y"},
|
||||
{name: "C1 control", forged: "x\u0085y"},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
data := zipBytes(t, map[string]string{"a.jar": jar, tt.forged: "forged"})
|
||||
r, _ := extractBytes(t, data, "outer.zip", nil)
|
||||
assert.Equal(t, []string{"a.jar"}, storedNames(t, r))
|
||||
require.Error(t, r.Truncated)
|
||||
assert.Contains(t, r.Truncated.Error(), fmt.Sprintf("%q", tt.forged))
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestResolver_extract_excludedEntriesAreNeverStored(t *testing.T) {
|
||||
data := zipBytes(t, map[string]string{
|
||||
"keep.txt": "kept",
|
||||
"vendor/lib.go": "excluded by directory",
|
||||
"deep/down/pkg.rpm": "excluded by name",
|
||||
})
|
||||
limiter := NewLimiter(Limits{MaxMemoryBytes: 1 << 20, MaxDiskBytes: -1})
|
||||
r, _ := resolverIn(t, limiter.charge())
|
||||
content := bytes.NewReader(data)
|
||||
|
||||
format, err := identifyFormat(context.Background(), "test.zip", content)
|
||||
require.NoError(t, err)
|
||||
err = r.extract(context.Background(), format, content,
|
||||
NewExclusions([]string{"**/vendor", "**/*.rpm"}))
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, r.Truncated)
|
||||
|
||||
assert.Equal(t, []string{"keep.txt"}, storedNames(t, r))
|
||||
mem, _ := limiter.InUse()
|
||||
assert.Equal(t, approxIndexBytes(tar.Header{Name: "keep.txt"})+4, mem, "an excluded entry costs nothing")
|
||||
}
|
||||
|
||||
// a compressed tar reader decompresses a skipped entry to reach the next header, so skipping must still
|
||||
// spend the budget or an excluded or oversized entry would decompress without bound
|
||||
func TestResolver_extract_skippedCompressedTarBytesSpendTheBudget(t *testing.T) {
|
||||
big := strings.Repeat("\x00", 4<<20)
|
||||
tests := []struct {
|
||||
name string
|
||||
exclusions []string
|
||||
entryCap int64
|
||||
}{
|
||||
{name: "excluded entry", exclusions: []string{"**/*.bin"}, entryCap: maxEntryBytes},
|
||||
{name: "entry over the size cap", entryCap: 1 << 10},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
defer func(prev int64) { maxEntryBytes = prev }(maxEntryBytes)
|
||||
maxEntryBytes = tt.entryCap
|
||||
|
||||
data := tarGzFromEntries(t, []testEntry{{name: "bomb.bin", body: big}, {name: "after.txt", body: "after"}})
|
||||
r, _ := resolverIn(t, memCharge(1<<30))
|
||||
r.budget = NewBudget(1 << 20)
|
||||
content := bytes.NewReader(data)
|
||||
format, err := identifyFormat(context.Background(), "test.tar.gz", content)
|
||||
require.NoError(t, err)
|
||||
|
||||
require.NoError(t, r.extract(context.Background(), format, content, NewExclusions(tt.exclusions)))
|
||||
require.Error(t, r.Truncated)
|
||||
assert.Empty(t, storedNames(t, r), "the walk stopped at the entry that spent the budget")
|
||||
assert.Zero(t, r.budget.remaining)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// a body behind a header that doesn't read as a regular file (here a socket per its mode bits) is never
|
||||
// opened, but the tar reader still decompresses it to reach the next header
|
||||
func TestResolver_extract_bodiesOfNonRegularTarEntriesSpendTheBudget(t *testing.T) {
|
||||
var buf bytes.Buffer
|
||||
gw := gzip.NewWriter(&buf)
|
||||
tw := tar.NewWriter(gw)
|
||||
body := strings.Repeat("\x00", 4<<20)
|
||||
require.NoError(t, tw.WriteHeader(&tar.Header{Name: "sock", Typeflag: tar.TypeReg, Mode: 0o140644, Size: int64(len(body))}))
|
||||
_, err := tw.Write([]byte(body))
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, tw.WriteHeader(&tar.Header{Name: "after.txt", Typeflag: tar.TypeReg, Mode: 0o644, Size: 5}))
|
||||
_, err = tw.Write([]byte("after"))
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, tw.Close())
|
||||
require.NoError(t, gw.Close())
|
||||
|
||||
r, _ := resolverIn(t, memCharge(1<<30))
|
||||
r.budget = NewBudget(1 << 20)
|
||||
content := bytes.NewReader(buf.Bytes())
|
||||
format, err := identifyFormat(context.Background(), "test.tar.gz", content)
|
||||
require.NoError(t, err)
|
||||
|
||||
require.NoError(t, r.extract(context.Background(), format, content, nil))
|
||||
assert.ErrorIs(t, r.Truncated, errBudgetSpent)
|
||||
assert.Empty(t, storedNames(t, r))
|
||||
assert.Zero(t, r.budget.remaining)
|
||||
}
|
||||
|
||||
// a zip or a plain tar seeks past what it skips, so an excluded entry there costs nothing
|
||||
func TestResolver_extract_skippedSeekableBytesAreFree(t *testing.T) {
|
||||
entries := []testEntry{{name: "bomb.bin", body: strings.Repeat("\x00", 4<<20)}, {name: "after.txt", body: "after"}}
|
||||
var plainTar bytes.Buffer
|
||||
tw := tar.NewWriter(&plainTar)
|
||||
for _, e := range entries {
|
||||
hdr := e.header()
|
||||
require.NoError(t, tw.WriteHeader(&hdr))
|
||||
_, err := tw.Write([]byte(e.body))
|
||||
require.NoError(t, err)
|
||||
}
|
||||
require.NoError(t, tw.Close())
|
||||
|
||||
for name, data := range map[string][]byte{"test.zip": zipFromEntries(t, entries), "test.tar": plainTar.Bytes()} {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
r, _ := resolverIn(t, memCharge(1<<30))
|
||||
r.budget = NewBudget(1 << 20)
|
||||
content := bytes.NewReader(data)
|
||||
format, err := identifyFormat(context.Background(), name, content)
|
||||
require.NoError(t, err)
|
||||
|
||||
require.NoError(t, r.extract(context.Background(), format, content, NewExclusions([]string{"**/*.bin"})))
|
||||
require.NoError(t, r.Truncated)
|
||||
assert.Equal(t, []string{"after.txt"}, storedNames(t, r))
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestExtract_readsEntriesAndDigestsTheArchive(t *testing.T) {
|
||||
data := zipBytes(t, map[string]string{"dir/hello.txt": "hello world"})
|
||||
|
||||
extracted, err := Extract(context.Background(), bytes.NewReader(data), file.Coordinates{RealPath: "app.war", FileSystemID: "parentFS", FullPath: "app.war!/some/path/app.zip"}, nil, nil)
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, extracted)
|
||||
t.Cleanup(extracted.Cleanup)
|
||||
|
||||
assert.NoError(t, extracted.Truncated)
|
||||
require.Len(t, extracted.Digests, 1)
|
||||
assert.Equal(t, "sha1", extracted.Digests[0].Algorithm)
|
||||
|
||||
locations, err := extracted.FilesByGlob("**/*.txt")
|
||||
require.NoError(t, err)
|
||||
require.Len(t, locations, 1)
|
||||
assert.Equal(t, file.Coordinates{RealPath: "app.war", FileSystemID: "parentFS", FullPath: "app.war!/some/path/app.zip!/dir/hello.txt"},
|
||||
locations[0].Coordinates)
|
||||
|
||||
reader, err := extracted.FileContentsByLocation(locations[0])
|
||||
require.NoError(t, err)
|
||||
body, err := io.ReadAll(reader)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "hello world", string(body))
|
||||
}
|
||||
|
||||
func TestExtract_heldContentIsReleasedOnceExtracted(t *testing.T) {
|
||||
// the archive's own bytes are only needed to extract and digest it; afterwards the limiter holds
|
||||
// the entries and the index estimate alone
|
||||
data := zipBytes(t, map[string]string{"hello.txt": "hi"})
|
||||
limiter := NewLimiter(Limits{MaxMemoryBytes: -1, MaxDiskBytes: -1})
|
||||
|
||||
extracted, err := Extract(context.Background(), unseekable{bytes.NewReader(data)}, file.NewCoordinates("app.zip", ""), limiter, nil)
|
||||
require.NoError(t, err)
|
||||
t.Cleanup(extracted.Cleanup)
|
||||
|
||||
mem, disk := limiter.InUse()
|
||||
assert.Equal(t, approxIndexBytes(tar.Header{Name: "hello.txt"})+2, mem)
|
||||
assert.Zero(t, disk)
|
||||
}
|
||||
|
||||
func TestExtract_heldContentWrittenToDiskStaysUntilCleanup(t *testing.T) {
|
||||
// only a top-level stream is ever held; a nested archive is read out of its parent, which is held
|
||||
// for the whole descent anyway, so spilled bytes wait for Cleanup with the entries
|
||||
ctx, root := scanContext(t)
|
||||
data := zipBytes(t, map[string]string{"hello.txt": "hi"})
|
||||
limiter := NewLimiter(Limits{MaxMemoryBytes: 0, MaxDiskBytes: -1})
|
||||
|
||||
extracted, err := Extract(ctx, unseekable{bytes.NewReader(data)}, file.NewCoordinates("app.zip", ""), limiter, nil)
|
||||
require.NoError(t, err)
|
||||
t.Cleanup(extracted.Cleanup)
|
||||
|
||||
require.Len(t, filesIn(t, root), 1, "the archive's bytes and its entries share one file")
|
||||
_, disk := limiter.InUse()
|
||||
assert.Equal(t, approxIndexBytes(tar.Header{Name: "hello.txt"})+2+int64(len(data)), disk)
|
||||
|
||||
extracted.Cleanup()
|
||||
assert.Empty(t, filesIn(t, root))
|
||||
_, disk = limiter.InUse()
|
||||
assert.Zero(t, disk)
|
||||
}
|
||||
|
||||
func TestExtract_aTruncatedStreamKeepsWhatWasRead(t *testing.T) {
|
||||
// a .tar.gz cut off partway breaks the stream for every entry after the cut; the ones before it are
|
||||
// still good and must not be thrown away
|
||||
noise := make([]byte, 64*1024)
|
||||
rand.New(rand.NewSource(1)).Read(noise)
|
||||
data := tarGzBytes(t, map[string]string{"a.txt": "first", "b.bin": string(noise)})
|
||||
cut := data[:len(data)/2]
|
||||
|
||||
extracted, err := Extract(context.Background(), bytes.NewReader(cut), file.NewCoordinates("app.tar.gz", ""), NewLimiter(unboundedLimits), nil)
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, extracted)
|
||||
t.Cleanup(extracted.Cleanup)
|
||||
|
||||
assert.Error(t, extracted.Truncated)
|
||||
locations, err := extracted.FilesByGlob("**")
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, []string{"/a.txt"}, localPaths(locations))
|
||||
}
|
||||
|
||||
func TestExtract_aPanicStillReleasesEverything(t *testing.T) {
|
||||
// a decoder panicking on hostile bytes must not leak the charge or the spill file
|
||||
ctx, root := scanContext(t)
|
||||
limiter := NewLimiter(Limits{MaxMemoryBytes: 0, MaxDiskBytes: -1})
|
||||
|
||||
require.Panics(t, func() {
|
||||
_, _ = Extract(ctx, &panicsAfter{n: 4096}, file.NewCoordinates("app.zip", ""), limiter, nil)
|
||||
})
|
||||
|
||||
mem, disk := limiter.InUse()
|
||||
assert.Zero(t, mem)
|
||||
assert.Zero(t, disk)
|
||||
assert.Empty(t, filesIn(t, root))
|
||||
}
|
||||
|
||||
// panicsAfter hands out n zero bytes, then panics.
|
||||
type panicsAfter struct{ n int }
|
||||
|
||||
func (p *panicsAfter) Read(b []byte) (int, error) {
|
||||
if p.n <= 0 {
|
||||
panic("decoder blew up")
|
||||
}
|
||||
n := min(len(b), p.n)
|
||||
clear(b[:n])
|
||||
p.n -= n
|
||||
return n, nil
|
||||
}
|
||||
|
||||
func TestExtract_archiveBytesThatCannotBePlacedAreSkipped(t *testing.T) {
|
||||
data := zipBytes(t, sampleFiles)
|
||||
limiter := NewLimiter(Limits{MaxMemoryBytes: 0, MaxDiskBytes: 0})
|
||||
|
||||
_, err := Extract(context.Background(), unseekable{bytes.NewReader(data)}, file.NewCoordinates("app.zip", ""), limiter, nil)
|
||||
assert.ErrorIs(t, err, ErrDiskLimitReached)
|
||||
|
||||
// read in place, the same bytes cost nothing; only the entries are refused
|
||||
extracted, err := Extract(context.Background(), bytes.NewReader(data), file.NewCoordinates("app.zip", ""), limiter, nil)
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, extracted)
|
||||
t.Cleanup(extracted.Cleanup)
|
||||
assert.Error(t, extracted.Truncated)
|
||||
locations, err := extracted.FilesByGlob("**/*")
|
||||
require.NoError(t, err)
|
||||
assert.Empty(t, locations)
|
||||
}
|
||||
|
||||
func Test_HasZipEndOfCentralDirectory(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
data []byte
|
||||
want bool
|
||||
}{
|
||||
{"a plain zip", zipBytes(t, sampleFiles), true},
|
||||
{"a zip behind a launcher script", append([]byte("#!/bin/bash\nexit 0\n"), zipBytes(t, sampleFiles)...), true},
|
||||
{"prose", []byte("this is not an archive, it is a sentence"), false},
|
||||
{"a tar.gz", tarGzBytes(t, sampleFiles), false},
|
||||
{"empty", nil, false},
|
||||
{"shorter than the signature", []byte("PK"), false},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
assert.Equal(t, tt.want, HasZipEndOfCentralDirectory(bytes.NewReader(tt.data)))
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func Test_MayHideAnAppendedArchive(t *testing.T) {
|
||||
launcher := append([]byte("#!/bin/bash\nexit 0\n"), zipBytes(t, sampleFiles)...)
|
||||
prose := []byte("plain text, whatever the extension claims")
|
||||
|
||||
t.Run("a seekable reader is rewound and returned as is", func(t *testing.T) {
|
||||
reader := bytes.NewReader(launcher)
|
||||
got, found := MayHideAnAppendedArchive(reader)
|
||||
assert.True(t, found)
|
||||
assert.Same(t, reader, got)
|
||||
rest, err := io.ReadAll(got)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, launcher, rest)
|
||||
})
|
||||
|
||||
t.Run("an unseekable reader replays what was consumed", func(t *testing.T) {
|
||||
got, found := MayHideAnAppendedArchive(unseekable{bytes.NewReader(launcher)})
|
||||
assert.True(t, found)
|
||||
rest, err := io.ReadAll(got)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, launcher, rest)
|
||||
})
|
||||
|
||||
t.Run("prose hides nothing", func(t *testing.T) {
|
||||
got, found := MayHideAnAppendedArchive(unseekable{bytes.NewReader(prose)})
|
||||
assert.False(t, found)
|
||||
rest, err := io.ReadAll(got)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, prose, rest)
|
||||
})
|
||||
}
|
||||
|
||||
// zeros is a reader of n zero bytes: the most compressible content there is.
|
||||
func zeros(n int64) io.Reader {
|
||||
return io.LimitReader(zeroSource{}, n)
|
||||
}
|
||||
|
||||
type zeroSource struct{}
|
||||
|
||||
func (zeroSource) Read(p []byte) (int, error) {
|
||||
clear(p)
|
||||
return len(p), nil
|
||||
}
|
||||
|
||||
func TestExtract_decompressionBombIsBoundedByTheLimits(t *testing.T) {
|
||||
// 64 MiB of zeros compresses to a few hundred KiB. The declared size is never trusted: bytes are
|
||||
// charged as they land, so the limits hold whatever the archive claims
|
||||
const inflated = 64 << 20
|
||||
limits := Limits{MaxMemoryBytes: 1 << 20, MaxDiskBytes: 4 << 20}
|
||||
|
||||
var zipBomb bytes.Buffer
|
||||
zw := zip.NewWriter(&zipBomb)
|
||||
w, err := zw.Create("bomb.bin")
|
||||
require.NoError(t, err)
|
||||
_, err = io.Copy(w, zeros(inflated))
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, zw.Close())
|
||||
|
||||
var tarBomb bytes.Buffer
|
||||
gw := gzip.NewWriter(&tarBomb)
|
||||
tw := tar.NewWriter(gw)
|
||||
require.NoError(t, tw.WriteHeader(&tar.Header{Name: "bomb.bin", Mode: 0o644, Size: inflated, Typeflag: tar.TypeReg}))
|
||||
_, err = io.Copy(tw, zeros(inflated))
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, tw.Close())
|
||||
require.NoError(t, gw.Close())
|
||||
|
||||
for name, data := range map[string][]byte{"bomb.zip": zipBomb.Bytes(), "bomb.tar.gz": tarBomb.Bytes()} {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
require.Less(t, len(data), 1<<20, "the fixture must be a bomb: small on disk, huge inflated")
|
||||
ctx, root := scanContext(t)
|
||||
limiter := NewLimiter(limits)
|
||||
|
||||
r, err := Extract(ctx, bytes.NewReader(data), file.NewCoordinates(name, ""), limiter, nil)
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, r)
|
||||
t.Cleanup(r.Cleanup)
|
||||
|
||||
assert.Error(t, r.Truncated)
|
||||
locations, err := r.FilesByGlob("**/*")
|
||||
require.NoError(t, err)
|
||||
assert.Empty(t, locations, "the entry that did not fit is not cataloged in part")
|
||||
|
||||
peakMemory, peakDisk := limiter.Peak()
|
||||
assert.LessOrEqual(t, peakMemory, limits.MaxMemoryBytes)
|
||||
assert.LessOrEqual(t, peakDisk, limits.MaxDiskBytes)
|
||||
|
||||
r.Cleanup()
|
||||
assert.Empty(t, filesIn(t, root))
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestExtract_entryCountBombIsBoundedByTheMemoryLimit(t *testing.T) {
|
||||
// every entry is a node in memory whatever the disk limit says, so a tiny archive of many entries
|
||||
// is bounded by the memory limit alone
|
||||
limits := Limits{MaxMemoryBytes: 1 << 20, MaxDiskBytes: -1}
|
||||
limiter := NewLimiter(limits)
|
||||
|
||||
r, err := Extract(context.Background(), bytes.NewReader(manyEntryZip(t, 20_000)), file.NewCoordinates("many.zip", ""), limiter, nil)
|
||||
require.NoError(t, err)
|
||||
t.Cleanup(r.Cleanup)
|
||||
|
||||
assert.Error(t, r.Truncated)
|
||||
assert.Less(t, r.entries, 20_000)
|
||||
assert.LessOrEqual(t, int64(nodeCount(r))*approxIndexBytesPerEntry, limits.MaxMemoryBytes+approxIndexBytesPerEntry,
|
||||
"nodes held must not exceed what the memory limit paid for")
|
||||
peakMemory, _ := limiter.Peak()
|
||||
assert.LessOrEqual(t, peakMemory, limits.MaxMemoryBytes)
|
||||
}
|
||||
|
||||
func TestExtract_deepPathBombIsBoundedByTheMemoryLimit(t *testing.T) {
|
||||
// 100 entries whose names each imply 250 distinct directories would be 25,000 nodes; the implied
|
||||
// directories are charged as nodes, so the memory limit bounds them too
|
||||
files := map[string]string{}
|
||||
for i := range 100 {
|
||||
var b strings.Builder
|
||||
for j := range maxEntryNameDepth - 6 {
|
||||
fmt.Fprintf(&b, "%03d-%03d/", i, j)
|
||||
}
|
||||
b.WriteString("f")
|
||||
require.LessOrEqual(t, b.Len(), maxEntryNameBytes)
|
||||
files[b.String()] = "x"
|
||||
}
|
||||
limits := Limits{MaxMemoryBytes: 8 << 20, MaxDiskBytes: -1}
|
||||
limiter := NewLimiter(limits)
|
||||
|
||||
r, err := Extract(context.Background(), bytes.NewReader(zipBytes(t, files)), file.NewCoordinates("deep.zip", ""), limiter, nil)
|
||||
require.NoError(t, err)
|
||||
t.Cleanup(r.Cleanup)
|
||||
|
||||
assert.Error(t, r.Truncated)
|
||||
assert.LessOrEqual(t, int64(nodeCount(r))*approxIndexBytesPerEntry, limits.MaxMemoryBytes+approxIndexBytesPerEntry)
|
||||
peakMemory, _ := limiter.Peak()
|
||||
assert.LessOrEqual(t, peakMemory, limits.MaxMemoryBytes)
|
||||
}
|
||||
|
||||
func TestExtract_locationsKeepTheTopLevelRealPath(t *testing.T) {
|
||||
data := zipBytes(t, map[string]string{"dir/hello.txt": "hello world"})
|
||||
|
||||
// a top-level archive's coordinates, with FullPath unset or set to RealPath, name entries alike
|
||||
for _, archive := range []file.Coordinates{
|
||||
{RealPath: "/app.zip"},
|
||||
{RealPath: "/app.zip", FullPath: "/app.zip"},
|
||||
} {
|
||||
extracted, err := Extract(context.Background(), bytes.NewReader(data), archive, nil, nil)
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, extracted)
|
||||
t.Cleanup(extracted.Cleanup)
|
||||
|
||||
locations, err := extracted.FilesByPath("dir/hello.txt")
|
||||
require.NoError(t, err)
|
||||
require.Len(t, locations, 1)
|
||||
assert.Equal(t, file.Coordinates{RealPath: "/app.zip", FullPath: "/app.zip!/dir/hello.txt"}, locations[0].Coordinates)
|
||||
assert.Equal(t, "/dir/hello.txt", locations[0].LocalPath())
|
||||
|
||||
for _, loc := range []file.Location{
|
||||
locations[0],
|
||||
file.NewLocation("dir/hello.txt"),
|
||||
file.NewLocation("/dir/hello.txt"),
|
||||
file.NewLocationFromCoordinates(file.Coordinates{RealPath: "/app.zip", FullPath: "/app.zip!/dir/hello.txt"}),
|
||||
} {
|
||||
reader, err := extracted.FileContentsByLocation(loc)
|
||||
require.NoError(t, err, "location %s", loc)
|
||||
body, err := io.ReadAll(reader)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "hello world", string(body))
|
||||
|
||||
metadata, err := extracted.FileMetadataByLocation(loc)
|
||||
require.NoError(t, err, "location %s", loc)
|
||||
assert.Equal(t, "/app.zip!/dir/hello.txt", metadata.Path)
|
||||
}
|
||||
|
||||
_, err = extracted.FileContentsByLocation(file.NewLocationFromCoordinates(file.Coordinates{RealPath: "/other.zip", FullPath: "/other.zip!/dir/hello.txt"}))
|
||||
assert.Error(t, err, "an entry of another archive is not read from this one")
|
||||
}
|
||||
}
|
||||
@@ -1,344 +0,0 @@
|
||||
package archive
|
||||
|
||||
import (
|
||||
"archive/tar"
|
||||
"archive/zip"
|
||||
"bytes"
|
||||
"compress/gzip"
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"io/fs"
|
||||
"os"
|
||||
"sort"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
stereoscopeFile "github.com/anchore/stereoscope/pkg/file"
|
||||
"github.com/anchore/syft/internal/tmpdir"
|
||||
"github.com/anchore/syft/syft/file"
|
||||
)
|
||||
|
||||
// held reports what this charge is currently holding, in memory and on disk.
|
||||
func (c *charge) held() (memory, disk int64) {
|
||||
if c == nil {
|
||||
return 0, 0
|
||||
}
|
||||
c.limiter.mu.Lock()
|
||||
defer c.limiter.mu.Unlock()
|
||||
return c.inMemory, c.onDisk
|
||||
}
|
||||
|
||||
// heldInMemory reports the content the resolver is still holding in memory.
|
||||
func (r *Resolver) heldInMemory() int64 {
|
||||
var total int64
|
||||
for _, b := range r.held {
|
||||
total += int64(len(b.mem))
|
||||
}
|
||||
return total
|
||||
}
|
||||
|
||||
// unseekable hides a reader's Seek and ReadAt, forcing content through the held path.
|
||||
type unseekable struct{ io.Reader }
|
||||
|
||||
// testEntry is one entry of a fixture archive: a directory when dir is set, a link when link is set
|
||||
// (hard when hard is set, otherwise symbolic), otherwise a regular file.
|
||||
type testEntry struct {
|
||||
name string
|
||||
body string
|
||||
link string
|
||||
hard bool
|
||||
dir bool
|
||||
mode fs.FileMode
|
||||
}
|
||||
|
||||
// header is the tar header this entry would carry.
|
||||
func (e testEntry) header() tar.Header {
|
||||
switch {
|
||||
case e.dir:
|
||||
return tar.Header{Name: e.name, Mode: int64(e.modeOr(0o755)), Typeflag: tar.TypeDir}
|
||||
case e.link != "" && e.hard:
|
||||
return tar.Header{Name: e.name, Mode: 0o644, Typeflag: tar.TypeLink, Linkname: e.link}
|
||||
case e.link != "":
|
||||
return tar.Header{Name: e.name, Mode: 0o777, Typeflag: tar.TypeSymlink, Linkname: e.link}
|
||||
}
|
||||
return tar.Header{Name: e.name, Mode: int64(e.modeOr(0o644)), Size: int64(len(e.body)), Typeflag: tar.TypeReg}
|
||||
}
|
||||
|
||||
func fileEntries(files map[string]string) []testEntry {
|
||||
names := make([]string, 0, len(files))
|
||||
for name := range files {
|
||||
names = append(names, name)
|
||||
}
|
||||
// limits bite as the walk proceeds, so entry order decides what lands before a truncation
|
||||
sort.Strings(names)
|
||||
entries := make([]testEntry, 0, len(names))
|
||||
for _, name := range names {
|
||||
entries = append(entries, testEntry{name: name, body: files[name]})
|
||||
}
|
||||
return entries
|
||||
}
|
||||
|
||||
func zipBytes(t testing.TB, files map[string]string) []byte {
|
||||
return zipFromEntries(t, fileEntries(files))
|
||||
}
|
||||
|
||||
func zipFromEntries(t testing.TB, entries []testEntry) []byte {
|
||||
t.Helper()
|
||||
var buf bytes.Buffer
|
||||
w := zip.NewWriter(&buf)
|
||||
for _, e := range entries {
|
||||
hdr := &zip.FileHeader{Name: e.name}
|
||||
body := e.body
|
||||
switch {
|
||||
case e.dir:
|
||||
hdr.SetMode(fs.ModeDir | e.modeOr(0o755))
|
||||
case e.link != "":
|
||||
hdr.SetMode(fs.ModeSymlink | 0o777)
|
||||
body = e.link
|
||||
default:
|
||||
hdr.SetMode(e.modeOr(0o644))
|
||||
}
|
||||
fw, err := w.CreateHeader(hdr)
|
||||
require.NoError(t, err)
|
||||
_, err = fw.Write([]byte(body))
|
||||
require.NoError(t, err)
|
||||
}
|
||||
require.NoError(t, w.Close())
|
||||
return buf.Bytes()
|
||||
}
|
||||
|
||||
func gzBytes(t testing.TB, body string) []byte {
|
||||
t.Helper()
|
||||
var buf bytes.Buffer
|
||||
gw := gzip.NewWriter(&buf)
|
||||
_, err := gw.Write([]byte(body))
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, gw.Close())
|
||||
return buf.Bytes()
|
||||
}
|
||||
|
||||
func tarGzBytes(t testing.TB, files map[string]string) []byte {
|
||||
return tarGzFromEntries(t, fileEntries(files))
|
||||
}
|
||||
|
||||
func tarGzFromEntries(t testing.TB, entries []testEntry) []byte {
|
||||
t.Helper()
|
||||
var buf bytes.Buffer
|
||||
gw := gzip.NewWriter(&buf)
|
||||
tw := tar.NewWriter(gw)
|
||||
for _, e := range entries {
|
||||
hdr := e.header()
|
||||
require.NoError(t, tw.WriteHeader(&hdr))
|
||||
_, err := tw.Write([]byte(e.body))
|
||||
require.NoError(t, err)
|
||||
}
|
||||
require.NoError(t, tw.Close())
|
||||
require.NoError(t, gw.Close())
|
||||
return buf.Bytes()
|
||||
}
|
||||
|
||||
func (e testEntry) modeOr(fallback fs.FileMode) fs.FileMode {
|
||||
if e.mode != 0 {
|
||||
return e.mode
|
||||
}
|
||||
return fallback
|
||||
}
|
||||
|
||||
func regularHeader(name string, size int64) tar.Header {
|
||||
return tar.Header{Name: name, Size: size, Mode: 0o600, Typeflag: tar.TypeReg}
|
||||
}
|
||||
|
||||
// scanContext returns a context whose temp root is a fresh directory, and that directory.
|
||||
func scanContext(t testing.TB) (context.Context, string) {
|
||||
t.Helper()
|
||||
root := t.TempDir()
|
||||
return tmpdir.WithValue(context.Background(), tmpdir.FromPath(root)), root
|
||||
}
|
||||
|
||||
// filesIn lists the regular files directly in dir.
|
||||
func filesIn(t testing.TB, dir string) []string {
|
||||
t.Helper()
|
||||
entries, err := os.ReadDir(dir)
|
||||
require.NoError(t, err)
|
||||
var names []string
|
||||
for _, e := range entries {
|
||||
if !e.IsDir() {
|
||||
names = append(names, e.Name())
|
||||
}
|
||||
}
|
||||
return names
|
||||
}
|
||||
|
||||
// spillFile is the one file a resolver has written, or "" when it has not.
|
||||
func spillFile(t testing.TB, dir string) string {
|
||||
t.Helper()
|
||||
files := filesIn(t, dir)
|
||||
require.LessOrEqual(t, len(files), 1)
|
||||
if len(files) == 0 {
|
||||
return ""
|
||||
}
|
||||
return files[0]
|
||||
}
|
||||
|
||||
// resolverIn makes an empty resolver over the given charge that spills into dir, along with that
|
||||
// directory. Its entries are added by the test.
|
||||
func resolverIn(t testing.TB, charge *charge) (*Resolver, string) {
|
||||
t.Helper()
|
||||
root := t.TempDir()
|
||||
ctx := tmpdir.WithValue(context.Background(), tmpdir.FromPath(root))
|
||||
r := newResolver(ctx, file.NewCoordinates("test-archive", ""), charge)
|
||||
t.Cleanup(r.Cleanup)
|
||||
return r, root
|
||||
}
|
||||
|
||||
// addAll adds entries in the order given and finishes the resolver.
|
||||
func addAll(t testing.TB, r *Resolver, entries []testEntry) {
|
||||
t.Helper()
|
||||
for _, e := range entries {
|
||||
require.NoError(t, r.add(e.header(), bytes.NewReader([]byte(e.body))))
|
||||
}
|
||||
r.finish()
|
||||
}
|
||||
|
||||
// resolverOver builds a resolver holding the given files within a memory bound.
|
||||
func resolverOver(t testing.TB, maxInMemory int64, files map[string]string) *Resolver {
|
||||
t.Helper()
|
||||
r, _ := resolverIn(t, memCharge(maxInMemory))
|
||||
r.archiveCoordinates = file.NewCoordinates("outer.jar", "")
|
||||
addAll(t, r, fileEntries(files))
|
||||
return r
|
||||
}
|
||||
|
||||
// resolverFrom builds a resolver holding the given entries, links and directories included, with
|
||||
// ample memory.
|
||||
func resolverFrom(t testing.TB, entries ...testEntry) *Resolver {
|
||||
t.Helper()
|
||||
r, _ := resolverIn(t, memCharge(1<<20))
|
||||
r.archiveCoordinates = file.NewCoordinates("outer.jar", "")
|
||||
addAll(t, r, entries)
|
||||
return r
|
||||
}
|
||||
|
||||
// extractBytes extracts an archive into a resolver built over the given charge.
|
||||
func extractBytes(t testing.TB, data []byte, name string, charge *charge) (*Resolver, string) {
|
||||
t.Helper()
|
||||
r, root := resolverIn(t, charge)
|
||||
content := bytes.NewReader(data)
|
||||
format, err := identifyFormat(context.Background(), name, content)
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, format, "fixture must identify as an archive")
|
||||
require.NoError(t, r.extract(context.Background(), format, content, nil))
|
||||
return r, root
|
||||
}
|
||||
|
||||
func readEntry(t testing.TB, r ReaderAtSeeker) string {
|
||||
t.Helper()
|
||||
_, err := r.Seek(0, io.SeekStart)
|
||||
require.NoError(t, err)
|
||||
b, err := io.ReadAll(r)
|
||||
require.NoError(t, err)
|
||||
return string(b)
|
||||
}
|
||||
|
||||
// storedEntry is one entry read back out of a resolver.
|
||||
type storedEntry struct {
|
||||
body string
|
||||
linkTarget string
|
||||
mode fs.FileMode
|
||||
kind stereoscopeFile.Type
|
||||
}
|
||||
|
||||
// readStore reads every stored entry by name: files, links, and directories listed or implied.
|
||||
func readStore(t testing.TB, r *Resolver) map[string]storedEntry {
|
||||
t.Helper()
|
||||
out := map[string]storedEntry{}
|
||||
for _, ref := range r.tree.AllFiles(stereoscopeFile.AllTypes()...) {
|
||||
entry, err := r.index.Get(ref)
|
||||
require.NoError(t, err)
|
||||
out[entry.Metadata.Path] = storedEntry{
|
||||
body: readEntry(t, r.open(r.content[ref.ID()])),
|
||||
linkTarget: entry.Metadata.LinkDestination,
|
||||
mode: entry.Metadata.FileInfo.Mode(),
|
||||
kind: entry.Metadata.Type,
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// storedNames lists every listed entry's name, files and directories alike.
|
||||
func storedNames(t testing.TB, r *Resolver) []string {
|
||||
t.Helper()
|
||||
var names []string
|
||||
for name := range readStore(t, r) {
|
||||
names = append(names, name)
|
||||
}
|
||||
sort.Strings(names)
|
||||
return names
|
||||
}
|
||||
|
||||
// nodeCount is every path the tree holds, including directories the archive never listed.
|
||||
func nodeCount(r *Resolver) int {
|
||||
return len(r.tree.AllRealPaths())
|
||||
}
|
||||
|
||||
// contentAt is the blob held for the entry at a rooted path.
|
||||
func contentAt(t testing.TB, r *Resolver, p string) *blob {
|
||||
t.Helper()
|
||||
_, res, err := r.tree.File(stereoscopeFile.Path(p))
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, res)
|
||||
return r.content[res.Reference.ID()]
|
||||
}
|
||||
|
||||
// memCharge is a charge against a memory bound with disk unbounded.
|
||||
func memCharge(maxMemory int64) *charge {
|
||||
return NewLimiter(Limits{MaxMemoryBytes: maxMemory, MaxDiskBytes: -1}).charge()
|
||||
}
|
||||
|
||||
// diskCharge is a charge against a disk bound with memory refusing everything.
|
||||
func diskCharge(maxDisk int64) *charge {
|
||||
return NewLimiter(Limits{MaxDiskBytes: maxDisk}).charge()
|
||||
}
|
||||
|
||||
// localPaths lists the local entry path of every location (rather than full nested archive path), sorted.
|
||||
func localPaths(locations []file.Location) []string {
|
||||
var out []string
|
||||
for _, loc := range locations {
|
||||
out = append(out, loc.LocalPath())
|
||||
}
|
||||
sort.Strings(out)
|
||||
return out
|
||||
}
|
||||
|
||||
// manyEntryZip is a zip of count one-byte files.
|
||||
func manyEntryZip(t testing.TB, count int) []byte {
|
||||
t.Helper()
|
||||
files := make(map[string]string, count)
|
||||
for i := range count {
|
||||
files[fmt.Sprintf("e%06d.txt", i)] = "x"
|
||||
}
|
||||
return zipBytes(t, files)
|
||||
}
|
||||
|
||||
// extractedResolver extracts one archive through Extract and returns the resolver, along with the scan
|
||||
// temp root its spill file would appear under.
|
||||
func extractedResolver(t testing.TB, data []byte, limits Limits) (*Resolver, string) {
|
||||
t.Helper()
|
||||
ctx, root := scanContext(t)
|
||||
|
||||
r, err := Extract(ctx, bytes.NewReader(data), file.NewCoordinates("app.zip", ""), NewLimiter(limits), nil)
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, r)
|
||||
t.Cleanup(r.Cleanup)
|
||||
|
||||
return r, root
|
||||
}
|
||||
|
||||
var (
|
||||
unboundedLimits = Limits{MaxMemoryBytes: -1, MaxDiskBytes: -1}
|
||||
|
||||
// spillingLimits hold nothing in memory, so every entry lands in the spill file
|
||||
spillingLimits = Limits{MaxMemoryBytes: 0, MaxDiskBytes: -1}
|
||||
)
|
||||
@@ -1,202 +0,0 @@
|
||||
package archive
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"sync"
|
||||
)
|
||||
|
||||
// Budget bounds the bytes decompressed under one top-level archive, summed over every archive nested in
|
||||
// it. Unlike Limits it never refills as archives are released: it bounds work, not what is held at once,
|
||||
// which is what stops a recursive bomb (42.zip) whose every level fits the limits on its own. A nil
|
||||
// *Budget admits everything.
|
||||
//
|
||||
// ponytail: no lock, since the archive walk is sequential; guard it if the walk ever runs concurrently.
|
||||
type Budget struct {
|
||||
remaining int64
|
||||
}
|
||||
|
||||
func NewBudget(bytes int64) *Budget {
|
||||
return &Budget{remaining: bytes}
|
||||
}
|
||||
|
||||
// take spends n bytes, or reports false (and spends the rest) when fewer than n remain.
|
||||
func (b *Budget) take(n int64) bool {
|
||||
if b == nil {
|
||||
return true
|
||||
}
|
||||
if n > b.remaining {
|
||||
b.remaining = 0
|
||||
return false
|
||||
}
|
||||
b.remaining -= n
|
||||
return true
|
||||
}
|
||||
|
||||
var errBudgetSpent = errors.New("archive tree decompression budget spent")
|
||||
|
||||
// ErrDiskLimitReached reports that placing an archive's content would exceed the disk limit.
|
||||
var ErrDiskLimitReached = errors.New("archive content would exceed the disk limit")
|
||||
|
||||
// Limits bound how much archive content one scan holds at once. Each limit falls as archives are
|
||||
// released, so it bounds peak usage rather than the total ever written.
|
||||
//
|
||||
// For both limits: positive is the bound, zero forbids the resource, negative is unbounded.
|
||||
type Limits struct {
|
||||
// MaxMemoryBytes bounds archive content held in memory. Content that does not fit is written to
|
||||
// disk instead.
|
||||
MaxMemoryBytes int64
|
||||
|
||||
// MaxDiskBytes bounds archive content written to disk. Content that does not fit is dropped and
|
||||
// the archive is skipped or truncated.
|
||||
MaxDiskBytes int64
|
||||
}
|
||||
|
||||
// Limiter tracks how much archive content a scan holds right now against its Limits. A nil *Limiter
|
||||
// enforces nothing.
|
||||
type Limiter struct {
|
||||
limits Limits
|
||||
|
||||
mu sync.Mutex
|
||||
memory int64
|
||||
disk int64
|
||||
|
||||
peakMemory int64
|
||||
peakDisk int64
|
||||
}
|
||||
|
||||
func NewLimiter(limits Limits) *Limiter {
|
||||
return &Limiter{limits: limits}
|
||||
}
|
||||
|
||||
// Peak reports the most memory and disk held at any one moment.
|
||||
func (l *Limiter) Peak() (memory, disk int64) {
|
||||
if l == nil {
|
||||
return 0, 0
|
||||
}
|
||||
l.mu.Lock()
|
||||
defer l.mu.Unlock()
|
||||
return l.peakMemory, l.peakDisk
|
||||
}
|
||||
|
||||
// InUse reports the memory and disk held right now.
|
||||
func (l *Limiter) InUse() (memory, disk int64) {
|
||||
if l == nil {
|
||||
return 0, 0
|
||||
}
|
||||
l.mu.Lock()
|
||||
defer l.mu.Unlock()
|
||||
return l.memory, l.disk
|
||||
}
|
||||
|
||||
// charge starts accounting for one archive. Releasing the charge gives back everything it took.
|
||||
func (l *Limiter) charge() *charge {
|
||||
if l == nil {
|
||||
return nil
|
||||
}
|
||||
return &charge{limiter: l}
|
||||
}
|
||||
|
||||
// charge is one archive's draw on a Limiter. A nil *charge admits everything.
|
||||
//
|
||||
// Bytes are charged as they land, never from the sizes an archive declares, since those can lie.
|
||||
type charge struct {
|
||||
limiter *Limiter
|
||||
inMemory int64
|
||||
onDisk int64
|
||||
}
|
||||
|
||||
func admits(limit, held, n int64) bool {
|
||||
switch {
|
||||
case limit < 0:
|
||||
return true
|
||||
case limit == 0:
|
||||
return false
|
||||
default:
|
||||
return held+n <= limit
|
||||
}
|
||||
}
|
||||
|
||||
// memory charges n bytes held in memory, or reports false (charging nothing) when the memory limit
|
||||
// does not admit them.
|
||||
func (c *charge) memory(n int64) bool {
|
||||
if c == nil || n <= 0 {
|
||||
return true
|
||||
}
|
||||
l := c.limiter
|
||||
l.mu.Lock()
|
||||
defer l.mu.Unlock()
|
||||
if !admits(l.limits.MaxMemoryBytes, l.memory, n) {
|
||||
return false
|
||||
}
|
||||
l.memory += n
|
||||
c.inMemory += n
|
||||
l.peakMemory = max(l.peakMemory, l.memory)
|
||||
return true
|
||||
}
|
||||
|
||||
// disk charges n bytes written to disk, or reports false (charging nothing) when the disk limit does
|
||||
// not admit them.
|
||||
func (c *charge) disk(n int64) bool {
|
||||
if c == nil || n <= 0 {
|
||||
return true
|
||||
}
|
||||
l := c.limiter
|
||||
l.mu.Lock()
|
||||
defer l.mu.Unlock()
|
||||
if !admits(l.limits.MaxDiskBytes, l.disk, n) {
|
||||
return false
|
||||
}
|
||||
l.disk += n
|
||||
c.onDisk += n
|
||||
l.peakDisk = max(l.peakDisk, l.disk)
|
||||
return true
|
||||
}
|
||||
|
||||
// index charges n bytes of index bookkeeping, which lives in memory whatever the limits say. A zero
|
||||
// memory limit is a valid configuration that must still index archives, so only then does the cost
|
||||
// fall back to the disk budget; a bounded memory limit bounds the index too.
|
||||
func (c *charge) index(n int64) bool {
|
||||
if c.memory(n) {
|
||||
return true
|
||||
}
|
||||
return c != nil && c.limiter.limits.MaxMemoryBytes == 0 && c.disk(n)
|
||||
}
|
||||
|
||||
// refundMemory gives back n bytes no longer held in memory.
|
||||
func (c *charge) refundMemory(n int64) {
|
||||
if c == nil || n <= 0 {
|
||||
return
|
||||
}
|
||||
l := c.limiter
|
||||
l.mu.Lock()
|
||||
defer l.mu.Unlock()
|
||||
n = min(n, c.inMemory)
|
||||
l.memory -= n
|
||||
c.inMemory -= n
|
||||
}
|
||||
|
||||
// refundDisk gives back n bytes no longer on disk.
|
||||
func (c *charge) refundDisk(n int64) {
|
||||
if c == nil || n <= 0 {
|
||||
return
|
||||
}
|
||||
l := c.limiter
|
||||
l.mu.Lock()
|
||||
defer l.mu.Unlock()
|
||||
n = min(n, c.onDisk)
|
||||
l.disk -= n
|
||||
c.onDisk -= n
|
||||
}
|
||||
|
||||
// release gives back everything this charge took. Safe to call more than once.
|
||||
func (c *charge) release() {
|
||||
if c == nil {
|
||||
return
|
||||
}
|
||||
l := c.limiter
|
||||
l.mu.Lock()
|
||||
defer l.mu.Unlock()
|
||||
l.memory -= c.inMemory
|
||||
l.disk -= c.onDisk
|
||||
c.inMemory, c.onDisk = 0, 0
|
||||
}
|
||||
@@ -1,178 +0,0 @@
|
||||
package archive
|
||||
|
||||
import (
|
||||
"sync"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestLimiter_chargeAndRelease(t *testing.T) {
|
||||
limiter := NewLimiter(Limits{MaxMemoryBytes: 100, MaxDiskBytes: 200})
|
||||
|
||||
first := limiter.charge()
|
||||
require.True(t, first.memory(60))
|
||||
require.True(t, first.disk(150))
|
||||
|
||||
mem, disk := limiter.InUse()
|
||||
assert.Equal(t, int64(60), mem)
|
||||
assert.Equal(t, int64(150), disk)
|
||||
|
||||
second := limiter.charge()
|
||||
assert.False(t, second.memory(50), "60 + 50 is over the memory limit")
|
||||
assert.False(t, second.disk(60), "150 + 60 is over the disk limit")
|
||||
assert.True(t, second.memory(40), "and exactly at the limit is admitted")
|
||||
|
||||
mem, _ = limiter.InUse()
|
||||
assert.Equal(t, int64(100), mem, "a refused charge takes nothing")
|
||||
|
||||
first.release()
|
||||
mem, disk = limiter.InUse()
|
||||
assert.Equal(t, int64(40), mem, "release gives back exactly what that charge took")
|
||||
assert.Zero(t, disk)
|
||||
|
||||
first.release()
|
||||
mem, _ = limiter.InUse()
|
||||
assert.Equal(t, int64(40), mem, "releasing twice must not give back what was never taken")
|
||||
}
|
||||
|
||||
func TestLimiter_refund(t *testing.T) {
|
||||
limiter := NewLimiter(Limits{MaxDiskBytes: 100})
|
||||
charge := limiter.charge()
|
||||
|
||||
require.True(t, charge.disk(80))
|
||||
charge.refundDisk(80)
|
||||
|
||||
_, disk := limiter.InUse()
|
||||
assert.Zero(t, disk)
|
||||
assert.True(t, charge.disk(100), "the refunded room is available again")
|
||||
|
||||
charge.refundDisk(500)
|
||||
_, disk = limiter.InUse()
|
||||
assert.Zero(t, disk, "a refund larger than what was charged cannot drive the limiter negative")
|
||||
}
|
||||
|
||||
func TestLimiter_threeStateReadingIsUniform(t *testing.T) {
|
||||
t.Run("positive is the limit", func(t *testing.T) {
|
||||
c := NewLimiter(Limits{MaxMemoryBytes: 100, MaxDiskBytes: 100}).charge()
|
||||
assert.True(t, c.memory(100))
|
||||
assert.False(t, c.memory(1))
|
||||
assert.True(t, c.disk(100))
|
||||
assert.False(t, c.disk(1))
|
||||
})
|
||||
|
||||
t.Run("zero refuses unconditionally on both", func(t *testing.T) {
|
||||
c := NewLimiter(Limits{MaxMemoryBytes: 0, MaxDiskBytes: 0}).charge()
|
||||
assert.False(t, c.memory(1))
|
||||
assert.False(t, c.disk(1))
|
||||
})
|
||||
|
||||
t.Run("negative admits unconditionally on both", func(t *testing.T) {
|
||||
c := NewLimiter(Limits{MaxMemoryBytes: -1, MaxDiskBytes: -1}).charge()
|
||||
assert.True(t, c.memory(1_000_000_000))
|
||||
assert.True(t, c.disk(1_000_000_000))
|
||||
})
|
||||
}
|
||||
|
||||
func TestLimiter_nilIsUnbounded(t *testing.T) {
|
||||
var limiter *Limiter
|
||||
charge := limiter.charge()
|
||||
assert.Nil(t, charge)
|
||||
|
||||
assert.True(t, charge.memory(1_000_000))
|
||||
assert.True(t, charge.disk(1_000_000))
|
||||
charge.refundDisk(10)
|
||||
charge.release()
|
||||
|
||||
mem, disk := charge.held()
|
||||
assert.Zero(t, mem)
|
||||
assert.Zero(t, disk)
|
||||
|
||||
mem, disk = limiter.InUse()
|
||||
assert.Zero(t, mem)
|
||||
assert.Zero(t, disk)
|
||||
|
||||
mem, disk = limiter.Peak()
|
||||
assert.Zero(t, mem)
|
||||
assert.Zero(t, disk)
|
||||
}
|
||||
|
||||
func TestLimiter_concurrentChargesAgree(t *testing.T) {
|
||||
limiter := NewLimiter(Limits{MaxMemoryBytes: 1_000_000, MaxDiskBytes: -1})
|
||||
|
||||
var wg sync.WaitGroup
|
||||
charges := make([]*charge, 50)
|
||||
for i := range charges {
|
||||
charges[i] = limiter.charge()
|
||||
wg.Add(1)
|
||||
go func(c *charge) {
|
||||
defer wg.Done()
|
||||
for range 100 {
|
||||
c.memory(1)
|
||||
c.disk(2)
|
||||
}
|
||||
}(charges[i])
|
||||
}
|
||||
wg.Wait()
|
||||
|
||||
mem, disk := limiter.InUse()
|
||||
assert.Equal(t, int64(50*100), mem)
|
||||
assert.Equal(t, int64(50*100*2), disk)
|
||||
|
||||
for _, c := range charges {
|
||||
c.release()
|
||||
}
|
||||
mem, disk = limiter.InUse()
|
||||
assert.Zero(t, mem)
|
||||
assert.Zero(t, disk)
|
||||
}
|
||||
|
||||
func TestLimiter_peak(t *testing.T) {
|
||||
t.Run("is a high-water mark", func(t *testing.T) {
|
||||
l := NewLimiter(Limits{MaxMemoryBytes: -1, MaxDiskBytes: -1})
|
||||
|
||||
first := l.charge()
|
||||
require.True(t, first.memory(100))
|
||||
require.True(t, first.disk(300))
|
||||
first.release()
|
||||
|
||||
second := l.charge()
|
||||
require.True(t, second.memory(40))
|
||||
require.True(t, second.disk(50))
|
||||
|
||||
memory, disk := l.InUse()
|
||||
assert.Equal(t, int64(40), memory)
|
||||
assert.Equal(t, int64(50), disk)
|
||||
|
||||
peakMemory, peakDisk := l.Peak()
|
||||
assert.Equal(t, int64(100), peakMemory, "the peak is the most held at once, not the most recently held")
|
||||
assert.Equal(t, int64(300), peakDisk)
|
||||
})
|
||||
|
||||
t.Run("counts concurrent holders together", func(t *testing.T) {
|
||||
l := NewLimiter(Limits{MaxMemoryBytes: -1, MaxDiskBytes: -1})
|
||||
|
||||
outer := l.charge()
|
||||
require.True(t, outer.memory(100))
|
||||
inner := l.charge()
|
||||
require.True(t, inner.memory(60))
|
||||
inner.release()
|
||||
outer.release()
|
||||
|
||||
peakMemory, _ := l.Peak()
|
||||
assert.Equal(t, int64(160), peakMemory)
|
||||
})
|
||||
|
||||
t.Run("a refused charge does not move it", func(t *testing.T) {
|
||||
l := NewLimiter(Limits{MaxMemoryBytes: 50, MaxDiskBytes: 50})
|
||||
|
||||
c := l.charge()
|
||||
assert.False(t, c.memory(500))
|
||||
assert.False(t, c.disk(500))
|
||||
|
||||
peakMemory, peakDisk := l.Peak()
|
||||
assert.Zero(t, peakMemory)
|
||||
assert.Zero(t, peakDisk)
|
||||
})
|
||||
}
|
||||
@@ -1,575 +0,0 @@
|
||||
package archive
|
||||
|
||||
import (
|
||||
"archive/tar"
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"io/fs"
|
||||
"os"
|
||||
"path"
|
||||
"slices"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
"unicode"
|
||||
|
||||
"github.com/dustin/go-humanize"
|
||||
"github.com/mholt/archives"
|
||||
|
||||
stereoscopeFile "github.com/anchore/stereoscope/pkg/file"
|
||||
"github.com/anchore/stereoscope/pkg/filetree"
|
||||
intFile "github.com/anchore/syft/internal/file"
|
||||
"github.com/anchore/syft/internal/log"
|
||||
"github.com/anchore/syft/internal/tmpdir"
|
||||
"github.com/anchore/syft/syft/file"
|
||||
)
|
||||
|
||||
// Resolver is one archive opened as its own filesystem: a file.Resolver over the entries Extract fed
|
||||
// into it, holding their content within the scan's limits. Entries are indexed in a stereoscope file
|
||||
// tree, the same index the directory and image resolvers search.
|
||||
//
|
||||
// Locations it returns keep the outermost archive's RealPath and carry the entry's FullPath. Queries
|
||||
// may name an entry either by its full path or by its path within the archive (see rootedPath).
|
||||
type Resolver struct {
|
||||
// Digests are of the archive file itself.
|
||||
Digests []file.Digest
|
||||
|
||||
// Truncated is non-nil when the resolver covers only part of the archive, and says why: the disk limit
|
||||
// or the decompression budget stopped extraction early, or an entry was too large or unreadable.
|
||||
Truncated error
|
||||
|
||||
archiveCoordinates file.Coordinates
|
||||
charge *charge
|
||||
budget *Budget
|
||||
metered bool // the budget is charged as the archive is decoded (see meteredReader), not in put
|
||||
|
||||
// content is held in memory while the memory limit admits it and in one spill file once it does
|
||||
// not. The file is created on first use, so an archive that stays in memory never touches the
|
||||
// filesystem, and blobs are located in it by offset with no framing of their own.
|
||||
tempDir *tmpdir.TempDir
|
||||
held []*blob // blobs in memory, in the order they were added
|
||||
file *os.File
|
||||
remove func()
|
||||
written int64
|
||||
chunk []byte
|
||||
|
||||
tree *filetree.FileTree
|
||||
index filetree.Index
|
||||
search filetree.Searcher // built by finish, once every entry is in the tree
|
||||
content map[stereoscopeFile.ID]*blob
|
||||
entries int // non-directory entries added
|
||||
}
|
||||
|
||||
var _ file.Resolver = (*Resolver)(nil)
|
||||
|
||||
// newResolver returns an empty resolver whose locations extend the archive's coordinates, spilling into
|
||||
// the scan's temp root when ctx carries one (see internal/tmpdir).
|
||||
func newResolver(ctx context.Context, archiveCoordinates file.Coordinates, charge *charge) *Resolver {
|
||||
return &Resolver{
|
||||
archiveCoordinates: archiveCoordinates,
|
||||
charge: charge,
|
||||
budget: budgetFromContext(ctx),
|
||||
tempDir: tmpdir.FromContext(ctx),
|
||||
tree: filetree.New(),
|
||||
index: filetree.NewIndex(),
|
||||
content: map[stereoscopeFile.ID]*blob{},
|
||||
}
|
||||
}
|
||||
|
||||
// Cleanup removes everything the archive holds on disk and releases its charge against the limiter.
|
||||
// Readers handed out before are no longer valid. Safe to call more than once.
|
||||
func (r *Resolver) Cleanup() {
|
||||
if r == nil {
|
||||
return
|
||||
}
|
||||
r.releaseStorage()
|
||||
r.charge.release()
|
||||
}
|
||||
|
||||
// approxIndexBytesPerEntry estimates what holding one tree node costs beyond its content: the node, its
|
||||
// header and metadata, and its index slots. It is charged for every node an entry creates, including
|
||||
// the directories its path implies, so an archive of many tiny entries or of a few entries with very
|
||||
// deep paths is still bounded by the limits.
|
||||
const approxIndexBytesPerEntry = 2 * 1024
|
||||
|
||||
func approxIndexBytes(hdr tar.Header) int64 {
|
||||
return approxIndexBytesPerEntry + int64(len(hdr.Name)+len(hdr.Linkname))
|
||||
}
|
||||
|
||||
// add stores and indexes one entry, reading its content in full now since archive readers are
|
||||
// sequential. Reaching a limit stores nothing for this entry and returns ErrDiskLimitReached; entries
|
||||
// added before it remain usable. The last entry at a path wins, as it does when the archive is
|
||||
// extracted (tar appends updates, an OCI layer applies in order, and java's legacy zip reading
|
||||
// overwrites too), so a stale earlier copy, such as an older pom.properties, can't stand in for what is
|
||||
// really there.
|
||||
func (r *Resolver) add(hdr tar.Header, content io.Reader) error {
|
||||
entryPath := path.Clean("/" + hdr.Name)
|
||||
if entryPath == "/" {
|
||||
return nil
|
||||
}
|
||||
replacing, ok := r.replace(entryPath, hdr)
|
||||
if !ok {
|
||||
return nil
|
||||
}
|
||||
|
||||
// directories the path implies that do not exist yet are nodes this entry pays for. Walking from the
|
||||
// deepest parent up stops at the first that exists, since its ancestors exist too
|
||||
var newDirs []string
|
||||
for i := strings.LastIndexByte(entryPath, '/'); i > 0; i = strings.LastIndexByte(entryPath[:i], '/') {
|
||||
if r.tree.HasPath(stereoscopeFile.Path(entryPath[:i])) {
|
||||
break
|
||||
}
|
||||
newDirs = append(newDirs, entryPath[:i])
|
||||
}
|
||||
slices.Reverse(newDirs)
|
||||
if !r.chargeIndex(approxIndexBytes(hdr) + int64(len(newDirs))*approxIndexBytesPerEntry) {
|
||||
return ErrDiskLimitReached
|
||||
}
|
||||
// indexed as if listed, since a search for a directory's children looks the directory up by name
|
||||
for _, dir := range newDirs {
|
||||
if err := r.store(dir, tar.Header{Name: dir, Typeflag: tar.TypeDir, Mode: 0o755}, &blob{}); err != nil {
|
||||
log.WithFields("entry", hdr.Name, "error", err).Trace("skipping archive entry")
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
b := &blob{}
|
||||
if hdr.Typeflag == tar.TypeReg && content != nil {
|
||||
// reading one byte past the cap is how an entry over it is told apart from one exactly at it
|
||||
err := r.put(b, io.LimitReader(content, maxEntryBytes+1))
|
||||
if err == nil && b.size > maxEntryBytes {
|
||||
err = errEntryTooLarge
|
||||
}
|
||||
if err != nil {
|
||||
r.discard(b)
|
||||
if errors.Is(err, errEntryTooLarge) {
|
||||
r.truncate(fmt.Errorf("an entry was skipped: %w of %s", err, humanize.IBytes(uint64(maxEntryBytes))))
|
||||
return nil
|
||||
}
|
||||
if errors.Is(err, ErrDiskLimitReached) || errors.Is(err, errBudgetSpent) {
|
||||
return err
|
||||
}
|
||||
// a damaged entry (a CRC mismatch, a corrupt deflate stream) costs only that entry
|
||||
r.truncate(fmt.Errorf("entry %q could not be read: %w", hdr.Name, err))
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
if err := r.store(entryPath, hdr, b); err != nil {
|
||||
// a path the tree cannot hold, such as a file beneath another file's name
|
||||
r.discard(b)
|
||||
log.WithFields("entry", hdr.Name, "error", err).Trace("skipping archive entry")
|
||||
return nil
|
||||
}
|
||||
if hdr.Typeflag != tar.TypeDir && !replacing {
|
||||
r.entries++
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// replace clears the way for an entry at a path that is already held, reporting whether it did and
|
||||
// false when the entry should be dropped. A later non-directory entry replaces an earlier one, refunding
|
||||
// what it held. A file and a directory at one path keep the first, since swapping kinds would mean
|
||||
// re-parenting what is already beneath it.
|
||||
func (r *Resolver) replace(entryPath string, hdr tar.Header) (replacing, ok bool) {
|
||||
exists, res, _ := r.tree.File(stereoscopeFile.Path(entryPath))
|
||||
if !exists || !res.HasReference() {
|
||||
return false, true
|
||||
}
|
||||
earlier, err := r.index.Get(*res.Reference)
|
||||
if err != nil || earlier.IsDir() || hdr.Typeflag == tar.TypeDir {
|
||||
return false, false
|
||||
}
|
||||
// a removed node's index entry is left behind, but searches only answer with references in the tree
|
||||
if err := r.tree.RemovePath(stereoscopeFile.Path(entryPath)); err != nil {
|
||||
log.WithFields("entry", hdr.Name, "error", err).Trace("unable to replace archive entry")
|
||||
return false, false
|
||||
}
|
||||
if b := r.content[res.ID()]; b != nil {
|
||||
r.discard(b)
|
||||
}
|
||||
delete(r.content, res.ID())
|
||||
return true, true
|
||||
}
|
||||
|
||||
func (r *Resolver) store(rooted string, hdr tar.Header, b *blob) error {
|
||||
ref, err := r.addToTree(rooted, hdr)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
metadata := stereoscopeFile.NewMetadata(hdr, r.open(b))
|
||||
metadata.Path = entryPath(rooted)
|
||||
r.index.Add(*ref, metadata)
|
||||
r.content[ref.ID()] = b
|
||||
return nil
|
||||
}
|
||||
|
||||
func (r *Resolver) addToTree(entryPath string, hdr tar.Header) (*stereoscopeFile.Reference, error) {
|
||||
p := stereoscopeFile.Path(entryPath)
|
||||
switch hdr.Typeflag {
|
||||
case tar.TypeDir:
|
||||
return r.tree.AddDir(p)
|
||||
case tar.TypeSymlink:
|
||||
return r.tree.AddSymLink(p, stereoscopeFile.Path(hdr.Linkname))
|
||||
case tar.TypeLink:
|
||||
// a hard link names its target from the archive root
|
||||
return r.tree.AddHardLink(p, stereoscopeFile.Path(path.Join("/", hdr.Linkname)))
|
||||
default:
|
||||
return r.tree.AddFile(p)
|
||||
}
|
||||
}
|
||||
|
||||
// maxEntryBytes caps any one entry, as the legacy java extraction did (see intFile.SafeCopy): without it
|
||||
// a single entry of an overlapping-entry zip bomb can decompress up to the disk limit.
|
||||
var maxEntryBytes int64 = intFile.PerFileReadLimit
|
||||
|
||||
var errEntryTooLarge = errors.New("archive entry exceeds the per-entry size cap")
|
||||
|
||||
// truncate marks the resolver as covering part of the archive, keeping the first reason given.
|
||||
func (r *Resolver) truncate(err error) {
|
||||
if r.Truncated == nil {
|
||||
r.Truncated = err
|
||||
}
|
||||
}
|
||||
|
||||
// chargeIndex charges index bookkeeping, which lives in memory. Content held in memory can move to
|
||||
// disk to make room for it; the index cannot, so when it is refused the entries are.
|
||||
func (r *Resolver) chargeIndex(n int64) bool {
|
||||
if r.charge.index(n) {
|
||||
return true
|
||||
}
|
||||
if len(r.held) == 0 || r.spill() != nil {
|
||||
return false
|
||||
}
|
||||
return r.charge.index(n)
|
||||
}
|
||||
|
||||
// finish builds the search context once every entry is added.
|
||||
func (r *Resolver) finish() {
|
||||
r.search = filetree.NewSearchContext(r.tree, r.index)
|
||||
}
|
||||
|
||||
// linkDestination is the rooted path a link points at inside the archive. A hard link names its
|
||||
// target from the archive root, as does an absolute symlink, so "/etc/passwd" is an entry of this
|
||||
// archive and never the host's. A relative symlink is joined onto the link's own directory, and
|
||||
// cleaning against the root keeps "../../x" inside the archive.
|
||||
func linkDestination(linkPath, linkname string, hard bool) string {
|
||||
if hard || path.IsAbs(linkname) {
|
||||
return path.Join("/", linkname)
|
||||
}
|
||||
return path.Join(path.Dir(linkPath), linkname)
|
||||
}
|
||||
|
||||
// entryPath is a node's path within the archive: archive-relative with no leading slash.
|
||||
func entryPath(rooted string) string {
|
||||
return strings.TrimPrefix(rooted, "/")
|
||||
}
|
||||
|
||||
// fullPath is how paths leave the resolver: the archive's own full path, then the entry path.
|
||||
func (r *Resolver) fullPath(rooted string) string {
|
||||
return r.archiveCoordinates.FullNestedPath() + file.ArchivePathSeparator + entryPath(rooted)
|
||||
}
|
||||
|
||||
// rootedPath is the tree path a query names, which is either form a path takes: a full path is trimmed to
|
||||
// the entry path within this archive, and anything else (a LocalPath, with or without its leading slash)
|
||||
// is taken as archive-relative already, since the tree looks a relative path up from its root. The archive's full path with a trailing "!" (what path.Dir gives for an entry at the root)
|
||||
// names the root. A full path into another archive in the scan names nothing here and reports false.
|
||||
func (r *Resolver) rootedPath(userPath string) (string, bool) {
|
||||
if rest, ok := strings.CutPrefix(userPath, r.archiveCoordinates.FullNestedPath()); ok {
|
||||
if rest == file.ArchivePathSeparator[:1] {
|
||||
return "/", true
|
||||
}
|
||||
if strings.HasPrefix(rest, file.ArchivePathSeparator) {
|
||||
// keep the separator's slash as the rooted path's
|
||||
return path.Clean(rest[len(file.ArchivePathSeparator)-1:]), true
|
||||
}
|
||||
}
|
||||
if r.outside(userPath) {
|
||||
return "", false
|
||||
}
|
||||
return path.Clean(userPath), true
|
||||
}
|
||||
|
||||
// outside reports whether a path not under this archive's full path is a full path from the scan
|
||||
// nonetheless: every full path in this archive tree starts with the outermost archive's real path and a
|
||||
// separator. The real path alone is taken as an entry path, so an entry named like the outermost archive
|
||||
// is found by its LocalPath.
|
||||
func (r *Resolver) outside(userPath string) bool {
|
||||
rest, ok := strings.CutPrefix(userPath, r.archiveCoordinates.RealPath)
|
||||
return ok && strings.HasPrefix(rest, file.ArchivePathSeparator)
|
||||
}
|
||||
|
||||
// coordinates name an entry by the outermost archive's real path, which never includes a path inside an
|
||||
// archive, and its full path through every archive.
|
||||
func (r *Resolver) coordinates(rooted string) file.Coordinates {
|
||||
return file.Coordinates{
|
||||
RealPath: r.archiveCoordinates.RealPath,
|
||||
FileSystemID: r.archiveCoordinates.FileSystemID,
|
||||
FullPath: r.fullPath(rooted),
|
||||
}
|
||||
}
|
||||
|
||||
// location names a resolution: a link is reported at its target's path with the link's own full path
|
||||
// as the access path.
|
||||
func (r *Resolver) location(res stereoscopeFile.Resolution) file.Location {
|
||||
coordinates := r.coordinates(string(res.RealPath))
|
||||
if res.RequestPath == "" || res.RequestPath == res.RealPath {
|
||||
return file.NewLocationFromCoordinates(coordinates)
|
||||
}
|
||||
return file.NewVirtualLocationFromCoordinates(coordinates, r.fullPath(string(res.RequestPath)))
|
||||
}
|
||||
|
||||
// resolve finds the file a path names, following links, and reports false for directories and
|
||||
// paths the archive never listed.
|
||||
func (r *Resolver) resolve(userPath string) (*stereoscopeFile.Resolution, bool) {
|
||||
rooted, ok := r.rootedPath(userPath)
|
||||
if !ok {
|
||||
return nil, false
|
||||
}
|
||||
res, err := r.search.SearchByPath(rooted, filetree.FollowBasenameLinks)
|
||||
if err != nil || res == nil || !res.HasReference() {
|
||||
return nil, false
|
||||
}
|
||||
entry, err := r.index.Get(*res.Reference)
|
||||
if err != nil || entry.IsDir() {
|
||||
return nil, false
|
||||
}
|
||||
return res, true
|
||||
}
|
||||
|
||||
func (r *Resolver) FileContentsByLocation(location file.Location) (io.ReadCloser, error) {
|
||||
res, ok := r.resolve(location.FullNestedPath())
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("no archive entry for path=%q", location.FullNestedPath())
|
||||
}
|
||||
return entryReader{r.open(r.content[res.ID()])}, nil
|
||||
}
|
||||
|
||||
func (r *Resolver) FileMetadataByLocation(location file.Location) (file.Metadata, error) {
|
||||
rooted, ok := r.rootedPath(location.FullNestedPath())
|
||||
if !ok {
|
||||
return file.Metadata{}, fmt.Errorf("no archive entry for path=%q", location.FullNestedPath())
|
||||
}
|
||||
exists, res, err := r.tree.File(stereoscopeFile.Path(rooted))
|
||||
if err != nil || !exists || !res.HasReference() {
|
||||
return file.Metadata{}, fmt.Errorf("no archive entry for path=%q", location.FullNestedPath())
|
||||
}
|
||||
entry, err := r.index.Get(*res.Reference)
|
||||
if err != nil {
|
||||
return file.Metadata{}, err
|
||||
}
|
||||
// the index holds entry paths; callers get full paths, as with locations
|
||||
metadata := entry.Metadata
|
||||
metadata.Path = r.fullPath(rooted)
|
||||
if metadata.LinkDestination != "" {
|
||||
metadata.LinkDestination = r.fullPath(linkDestination(rooted, metadata.LinkDestination, metadata.Type == stereoscopeFile.TypeHardLink))
|
||||
}
|
||||
return metadata, nil
|
||||
}
|
||||
|
||||
func (r *Resolver) HasPath(userPath string) bool {
|
||||
rooted, ok := r.rootedPath(userPath)
|
||||
return ok && r.tree.HasPath(stereoscopeFile.Path(rooted))
|
||||
}
|
||||
|
||||
func (r *Resolver) FilesByPath(paths ...string) ([]file.Location, error) {
|
||||
var out []file.Location
|
||||
var seen []stereoscopeFile.ID // few paths are asked for at once
|
||||
for _, userPath := range paths {
|
||||
res, ok := r.resolve(userPath)
|
||||
if !ok || slices.Contains(seen, res.ID()) {
|
||||
continue
|
||||
}
|
||||
seen = append(seen, res.ID())
|
||||
out = append(out, r.location(*res))
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func (r *Resolver) FilesByGlob(patterns ...string) ([]file.Location, error) {
|
||||
var out []file.Location
|
||||
seen := stereoscopeFile.NewFileReferenceSet()
|
||||
for _, pattern := range patterns {
|
||||
rooted, ok := r.rootedPath(pattern)
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
results, err := r.searchByGlob(rooted)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("invalid glob %q: %w", pattern, err)
|
||||
}
|
||||
for _, res := range results {
|
||||
if !res.HasReference() || seen.Contains(*res.Reference) {
|
||||
continue
|
||||
}
|
||||
entry, err := r.index.Get(*res.Reference)
|
||||
if err != nil || entry.IsDir() {
|
||||
continue
|
||||
}
|
||||
seen.Add(*res.Reference)
|
||||
out = append(out, r.location(res))
|
||||
}
|
||||
}
|
||||
sortLocations(out)
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// searchByGlob is the index search, except that stereoscope answers "<dir>/*" by looking the directory
|
||||
// up by name and the root has none, so "/*" (or "*") walks the tree instead.
|
||||
func (r *Resolver) searchByGlob(pattern string) ([]stereoscopeFile.Resolution, error) {
|
||||
if strings.TrimPrefix(pattern, "/") == "*" {
|
||||
return r.tree.FilesByGlob("/*", filetree.FollowBasenameLinks)
|
||||
}
|
||||
return r.search.SearchByGlob(pattern, filetree.FollowBasenameLinks)
|
||||
}
|
||||
|
||||
// sortLocations orders answers by path, which decides how locations group into packages when results
|
||||
// merge into the SBOM.
|
||||
func sortLocations(locations []file.Location) {
|
||||
sort.Slice(locations, func(i, j int) bool {
|
||||
if a, b := locations[i].FullNestedPath(), locations[j].FullNestedPath(); a != b {
|
||||
return a < b
|
||||
}
|
||||
return locations[i].AccessPath < locations[j].AccessPath
|
||||
})
|
||||
}
|
||||
|
||||
func (r *Resolver) FilesByMIMEType(types ...string) ([]file.Location, error) {
|
||||
results, err := r.search.SearchByMIMEType(types...)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var out []file.Location
|
||||
for _, res := range results {
|
||||
if res.HasReference() {
|
||||
out = append(out, r.location(res))
|
||||
}
|
||||
}
|
||||
sortLocations(out)
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// RelativeFileByPath resolves against the archive root: an archive has no layers to be relative to.
|
||||
func (r *Resolver) RelativeFileByPath(_ file.Location, userPath string) *file.Location {
|
||||
locations, _ := r.FilesByPath(userPath)
|
||||
if len(locations) == 0 {
|
||||
return nil
|
||||
}
|
||||
return &locations[0]
|
||||
}
|
||||
|
||||
func (r *Resolver) AllLocations(ctx context.Context) <-chan file.Location {
|
||||
out := make(chan file.Location)
|
||||
go func() {
|
||||
defer close(out)
|
||||
refs := r.tree.AllFiles(stereoscopeFile.TypeRegular, stereoscopeFile.TypeSymLink, stereoscopeFile.TypeHardLink)
|
||||
sort.Slice(refs, func(i, j int) bool { return refs[i].RealPath < refs[j].RealPath })
|
||||
for _, ref := range refs {
|
||||
select {
|
||||
case out <- r.location(stereoscopeFile.Resolution{RequestPath: ref.RealPath, Reference: &ref}):
|
||||
case <-ctx.Done():
|
||||
return
|
||||
}
|
||||
}
|
||||
}()
|
||||
return out
|
||||
}
|
||||
|
||||
// entryReader gives an entry's reader the Close the resolver interface requires. The resolver owns
|
||||
// the bytes, so there is nothing to close.
|
||||
type entryReader struct {
|
||||
ReaderAtSeeker
|
||||
}
|
||||
|
||||
func (entryReader) Close() error { return nil }
|
||||
|
||||
// maxEntryNameBytes is the longest entry name stored: PATH_MAX. Every path component becomes a node,
|
||||
// so a longer name is a cheap way to inflate the index and no filesystem could hold it anyway.
|
||||
const maxEntryNameBytes = MaxPathBytes
|
||||
|
||||
// MaxPathBytes is PATH_MAX, the longest whole archive chain a scan stores or a decoded document may build.
|
||||
// Each nested file copies its container's chain, so without it a long name shared by many small entries
|
||||
// costs memory quadratic in the document.
|
||||
const MaxPathBytes = 4096
|
||||
|
||||
// MaxDepth is the deepest archives nest: a scan stops there whatever depth is configured, and a decoded
|
||||
// document is held to it. No real nesting comes close, and a hostile archive (a zip quine, a recursive bomb)
|
||||
// must still end.
|
||||
const MaxDepth = 16
|
||||
|
||||
// maxEntryNameDepth is the most path components an entry name may have. Adding an entry costs work per
|
||||
// component, and no real archive nests anywhere near this deep.
|
||||
const maxEntryNameDepth = 256
|
||||
|
||||
// entryHeader builds the header stored for one archive entry, or reports false for an entry not worth
|
||||
// carrying: device nodes, fifos and sockets, which no cataloger reads, and names that cannot be a path
|
||||
// inside the archive. A name refused as unsafe marks the resolver truncated.
|
||||
//
|
||||
// The name is cleaned to an archive-relative path with no leading slash, so "../x" names x at the
|
||||
// archive root rather than anything outside it. Symlinks are stored as headers, never followed on the
|
||||
// host; the resolver resolves their targets inside the archive.
|
||||
func (r *Resolver) entryHeader(f archives.FileInfo) (tar.Header, bool) {
|
||||
mode := f.Mode()
|
||||
if !mode.IsRegular() && !mode.IsDir() && mode.Type()&fs.ModeSymlink == 0 {
|
||||
log.WithFields("entry", f.NameInArchive, "mode", mode).Debug("skipping non-regular archive entry")
|
||||
return tar.Header{}, false
|
||||
}
|
||||
|
||||
name, err := EntryName(f.NameInArchive)
|
||||
if err != nil {
|
||||
r.truncate(fmt.Errorf("an entry was skipped: %w", err))
|
||||
return tar.Header{}, false
|
||||
}
|
||||
if name == "" {
|
||||
return tar.Header{}, false
|
||||
}
|
||||
// held to the chain length a decoder accepts back, so the SBOM round-trips
|
||||
if chain := len(r.archiveCoordinates.FullNestedPath()) + len(file.ArchivePathSeparator) + len(name); chain > MaxPathBytes {
|
||||
r.truncate(fmt.Errorf("an entry was skipped: its path of %d bytes within the archive chain exceeds the %d byte cap", chain, MaxPathBytes))
|
||||
return tar.Header{}, false
|
||||
}
|
||||
|
||||
hdr, err := tar.FileInfoHeader(f.FileInfo, f.LinkTarget)
|
||||
if err != nil {
|
||||
log.WithFields("entry", f.NameInArchive, "error", err).Debug("skipping archive entry with an unusable header")
|
||||
return tar.Header{}, false
|
||||
}
|
||||
hdr.Name = name
|
||||
hdr.Linkname = f.LinkTarget
|
||||
|
||||
// nothing reads these, and the header is held for the life of the archive
|
||||
hdr.PAXRecords = nil
|
||||
hdr.Uname, hdr.Gname = "", ""
|
||||
hdr.AccessTime, hdr.ChangeTime = time.Time{}, time.Time{}
|
||||
hdr.ModTime = hdr.ModTime.Truncate(time.Second)
|
||||
|
||||
return *hdr, true
|
||||
}
|
||||
|
||||
// EntryName cleans an archive entry's name to a path relative to the archive root, returning "" when it
|
||||
// names nothing inside the archive. This is the trust boundary for names: every one ends up in FullPath
|
||||
// and the encoders, so a name that could forge an archive boundary ("!/") or a line in a line-oriented
|
||||
// format (control characters, unicode line separators) is refused. Decoders building nested coordinates
|
||||
// from a document hold its names to the same rules.
|
||||
func EntryName(name string) (string, error) {
|
||||
if len(name) > maxEntryNameBytes {
|
||||
return "", fmt.Errorf("entry name of %d bytes exceeds the %d byte cap", len(name), maxEntryNameBytes)
|
||||
}
|
||||
if strings.IndexFunc(name, isUnsafeNameRune) >= 0 {
|
||||
return "", fmt.Errorf("entry name %q contains a control character", name)
|
||||
}
|
||||
cleaned := strings.TrimPrefix(path.Clean("/"+name), "/")
|
||||
if depth := strings.Count(cleaned, "/") + 1; depth > maxEntryNameDepth {
|
||||
return "", fmt.Errorf("entry name of %d path components exceeds the %d component cap", depth, maxEntryNameDepth)
|
||||
}
|
||||
if strings.Contains(cleaned, file.ArchivePathSeparator) {
|
||||
return "", fmt.Errorf("entry name %q contains the archive path separator %q", name, file.ArchivePathSeparator)
|
||||
}
|
||||
return cleaned, nil
|
||||
}
|
||||
|
||||
func isUnsafeNameRune(c rune) bool {
|
||||
return unicode.IsControl(c) || c == '\u2028' || c == '\u2029'
|
||||
}
|
||||
@@ -1,506 +0,0 @@
|
||||
package archive
|
||||
|
||||
import (
|
||||
"archive/tar"
|
||||
"bytes"
|
||||
"context"
|
||||
"io"
|
||||
"path"
|
||||
"sort"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
stereoscopeFile "github.com/anchore/stereoscope/pkg/file"
|
||||
"github.com/anchore/syft/syft/file"
|
||||
)
|
||||
|
||||
var jarEntriesFixture = map[string]string{
|
||||
"META-INF/MANIFEST.MF": "Manifest-Version: 1.0\n",
|
||||
"META-INF/maven/com.example/lib/pom.properties": "groupId=com.example\n",
|
||||
"META-INF/maven/com.example/lib/pom.xml": "<project/>",
|
||||
"LICENSE": "Apache License 2.0",
|
||||
"BOOT-INF/lib/inner.jar": "PK\x03\x04 not really",
|
||||
"com/example/Thing.class": "bytecode",
|
||||
}
|
||||
|
||||
func TestResolver_add_refusedEntryIsNotStored(t *testing.T) {
|
||||
// a half-stored pom a cataloger parses is worse than an absent one
|
||||
r, _ := resolverIn(t, NewLimiter(Limits{MaxMemoryBytes: 0, MaxDiskBytes: 0}).charge())
|
||||
|
||||
err := r.add(regularHeader("a.txt", 3), bytes.NewReader([]byte("abc")))
|
||||
assert.ErrorIs(t, err, ErrDiskLimitReached)
|
||||
assert.Empty(t, storedNames(t, r))
|
||||
assert.False(t, r.HasPath("a.txt"))
|
||||
}
|
||||
|
||||
func TestResolver_add_refusedContentLeavesNoNodeBehind(t *testing.T) {
|
||||
// the index estimate is admitted, then the content is refused
|
||||
record := approxIndexBytes(regularHeader("dir/a.txt", 0)) + approxIndexBytesPerEntry // the entry and the directory it implies
|
||||
r, _ := resolverIn(t, diskCharge(record+2))
|
||||
|
||||
err := r.add(regularHeader("dir/a.txt", 3), bytes.NewReader([]byte("abc")))
|
||||
assert.ErrorIs(t, err, ErrDiskLimitReached)
|
||||
assert.False(t, r.HasPath("dir/a.txt"))
|
||||
assert.True(t, r.HasPath("dir"), "the directories it implied stay")
|
||||
}
|
||||
|
||||
func TestResolver_add_directoryEntriesHoldNoContent(t *testing.T) {
|
||||
r, _ := resolverIn(t, memCharge(1<<20))
|
||||
|
||||
require.NoError(t, r.add(tar.Header{Name: "lib", Typeflag: tar.TypeDir, Mode: 0o755}, nil))
|
||||
|
||||
assert.Zero(t, r.heldInMemory())
|
||||
assert.Empty(t, readStore(t, r)["lib"].body)
|
||||
}
|
||||
|
||||
func TestResolver_add_indexDoesNotFallBackToDiskWhenMemoryIsBounded(t *testing.T) {
|
||||
// the index lives in memory, so a bounded memory limit bounds it; only a zero limit defers to disk
|
||||
limiter := NewLimiter(Limits{MaxMemoryBytes: 1024, MaxDiskBytes: -1})
|
||||
r, _ := resolverIn(t, limiter.charge())
|
||||
|
||||
err := r.add(regularHeader("a.txt", 0), bytes.NewReader(nil))
|
||||
assert.ErrorIs(t, err, ErrDiskLimitReached)
|
||||
_, disk := limiter.InUse()
|
||||
assert.Zero(t, disk)
|
||||
}
|
||||
|
||||
func TestResolver_add_indexMakesRoomBySpillingHeldContent(t *testing.T) {
|
||||
record := approxIndexBytes(regularHeader("a.txt", 0))
|
||||
r, dir := resolverIn(t, memCharge(2*record+50))
|
||||
|
||||
require.NoError(t, r.add(regularHeader("a.txt", 100), bytes.NewReader(bytes.Repeat([]byte("a"), 100))))
|
||||
require.Equal(t, int64(100), r.heldInMemory())
|
||||
|
||||
// b's index does not fit beside a's content, so a's content moves to disk and b is indexed
|
||||
require.NoError(t, r.add(regularHeader("b.txt", 3), bytes.NewReader([]byte("bbb"))))
|
||||
r.finish()
|
||||
|
||||
assert.Equal(t, int64(3), r.heldInMemory())
|
||||
assert.NotEmpty(t, spillFile(t, dir))
|
||||
assert.Equal(t, []string{"a.txt", "b.txt"}, storedNames(t, r))
|
||||
assert.Equal(t, strings.Repeat("a", 100), readStore(t, r)["a.txt"].body)
|
||||
}
|
||||
|
||||
func TestResolver_add_lastEntryAtAPathWins(t *testing.T) {
|
||||
r, _ := resolverIn(t, memCharge(1<<20))
|
||||
|
||||
require.NoError(t, r.add(regularHeader("a.txt", 5), bytes.NewReader([]byte("first"))))
|
||||
require.NoError(t, r.add(regularHeader("a.txt", 6), bytes.NewReader([]byte("second"))))
|
||||
require.NoError(t, r.add(regularHeader("b.txt", 1), bytes.NewReader([]byte("b"))))
|
||||
require.NoError(t, r.add(tar.Header{Name: "b.txt", Typeflag: tar.TypeSymlink, Linkname: "a.txt"}, nil))
|
||||
r.finish()
|
||||
|
||||
assert.Equal(t, "second", readStore(t, r)["a.txt"].body)
|
||||
assert.Equal(t, "a.txt", readStore(t, r)["b.txt"].linkTarget, "a later entry may change kind")
|
||||
assert.Equal(t, 2, r.entries)
|
||||
assert.Equal(t, int64(len("second")), r.heldInMemory(), "the earlier copy is refunded")
|
||||
|
||||
locations, err := r.FilesByGlob("**/a.txt")
|
||||
require.NoError(t, err)
|
||||
require.Len(t, locations, 1)
|
||||
reader, err := r.FileContentsByLocation(locations[0])
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "second", readEntry(t, reader.(entryReader).ReaderAtSeeker))
|
||||
}
|
||||
|
||||
func TestResolver_add_fileAndDirectoryAtOnePathKeepsTheFirst(t *testing.T) {
|
||||
r, _ := resolverIn(t, memCharge(1<<20))
|
||||
|
||||
require.NoError(t, r.add(regularHeader("a/b", 1), bytes.NewReader([]byte("x"))))
|
||||
require.NoError(t, r.add(regularHeader("a", 4), bytes.NewReader([]byte("file"))))
|
||||
require.NoError(t, r.add(regularHeader("c", 4), bytes.NewReader([]byte("file"))))
|
||||
require.NoError(t, r.add(tar.Header{Name: "c", Typeflag: tar.TypeDir, Mode: 0o755}, nil))
|
||||
r.finish()
|
||||
|
||||
got := readStore(t, r)
|
||||
assert.Equal(t, stereoscopeFile.TypeDirectory, got["a"].kind)
|
||||
assert.Equal(t, "file", got["c"].body)
|
||||
}
|
||||
|
||||
func TestResolver_pathsAndContents(t *testing.T) {
|
||||
r := resolverOver(t, 1<<20, jarEntriesFixture)
|
||||
|
||||
locations, err := r.FilesByPath("/META-INF/MANIFEST.MF")
|
||||
require.NoError(t, err)
|
||||
require.Len(t, locations, 1)
|
||||
|
||||
assert.True(t, r.HasPath("/META-INF/MANIFEST.MF"))
|
||||
assert.True(t, r.HasPath("META-INF/MANIFEST.MF"), "a path may arrive with or without the leading slash")
|
||||
assert.False(t, r.HasPath("/nope"))
|
||||
|
||||
reader, err := r.FileContentsByLocation(locations[0])
|
||||
require.NoError(t, err)
|
||||
body, err := io.ReadAll(reader)
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, reader.Close())
|
||||
assert.Equal(t, "Manifest-Version: 1.0\n", string(body))
|
||||
}
|
||||
|
||||
func TestResolver_fullPathsOutsideTheArchiveNameNothing(t *testing.T) {
|
||||
r, _ := resolverIn(t, memCharge(1<<20))
|
||||
r.archiveCoordinates = file.Coordinates{RealPath: "/app.war", FullPath: "/app.war!/WEB-INF/lib/dep.jar"}
|
||||
addAll(t, r, fileEntries(map[string]string{"/app.war": "same name", "/META-INF/MANIFEST.MF": "m"}))
|
||||
|
||||
locations, err := r.FilesByPath("/app.war!/WEB-INF/lib/dep.jar!/META-INF/MANIFEST.MF")
|
||||
require.NoError(t, err)
|
||||
assert.Len(t, locations, 1, "a full path in this archive")
|
||||
|
||||
for _, p := range []string{"/app.war!/WEB-INF/lib/dep.jar", "/app.war!/WEB-INF/lib/other.jar!/META-INF/MANIFEST.MF"} {
|
||||
locations, err = r.FilesByPath(p)
|
||||
require.NoError(t, err)
|
||||
assert.Empty(t, locations, "%s is outside the archive", p)
|
||||
assert.False(t, r.HasPath(p), p)
|
||||
}
|
||||
|
||||
for _, p := range []string{"app.war", "/app.war"} {
|
||||
locations, err = r.FilesByPath(p)
|
||||
require.NoError(t, err)
|
||||
require.Len(t, locations, 1, "%s is an entry path named like the outermost archive", p)
|
||||
assert.Equal(t, "/app.war!/WEB-INF/lib/dep.jar!/app.war", locations[0].FullPath, p)
|
||||
}
|
||||
|
||||
again, err := r.FilesByPath(locations[0].LocalPath())
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, locations, again, "a LocalPath finds its own entry")
|
||||
}
|
||||
|
||||
func TestResolver_mimeTypesAreSniffedFromContent(t *testing.T) {
|
||||
r := resolverOver(t, 1<<20, map[string]string{
|
||||
"notes.txt": "just some text",
|
||||
"run.sh": "#!/bin/bash\necho hello\n",
|
||||
})
|
||||
|
||||
locations, err := r.FilesByMIMEType("text/x-shellscript")
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, []string{"/run.sh"}, localPaths(locations))
|
||||
}
|
||||
|
||||
func TestResolver_answersAreOrdered(t *testing.T) {
|
||||
// order decides which locations group into which package when results merge into the SBOM
|
||||
entries := map[string]string{}
|
||||
for _, name := range []string{
|
||||
"b/z.jar", "a/y.jar", "c/x.jar", "a/b/c/w.jar", "z.jar", "a.jar",
|
||||
} {
|
||||
entries[name] = "PK\x03\x04 not really"
|
||||
}
|
||||
r := resolverOver(t, 1<<20, entries)
|
||||
|
||||
first, err := r.FilesByGlob("**/*.jar")
|
||||
require.NoError(t, err)
|
||||
require.Len(t, first, len(entries))
|
||||
|
||||
byMIME, err := r.FilesByMIMEType("application/octet-stream")
|
||||
require.NoError(t, err)
|
||||
|
||||
for range 8 {
|
||||
again, err := r.FilesByGlob("**/*.jar")
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, first, again)
|
||||
|
||||
mimeAgain, err := r.FilesByMIMEType("application/octet-stream")
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, byMIME, mimeAgain)
|
||||
|
||||
var all []string
|
||||
for loc := range r.AllLocations(context.Background()) {
|
||||
all = append(all, loc.LocalPath())
|
||||
}
|
||||
assert.True(t, sort.StringsAreSorted(all))
|
||||
}
|
||||
|
||||
var paths []string
|
||||
for _, loc := range first {
|
||||
paths = append(paths, loc.LocalPath())
|
||||
}
|
||||
assert.True(t, sort.StringsAreSorted(paths), "ordered by path, not merely stable")
|
||||
}
|
||||
|
||||
func TestResolver_impliedDirectoriesArePathsButNotFiles(t *testing.T) {
|
||||
r := resolverOver(t, 1<<20, jarEntriesFixture)
|
||||
|
||||
assert.True(t, r.HasPath("META-INF"))
|
||||
assert.True(t, r.HasPath("/META-INF/maven"))
|
||||
|
||||
locations, err := r.FilesByPath("META-INF")
|
||||
require.NoError(t, err)
|
||||
assert.Empty(t, locations)
|
||||
locations, err = r.FilesByGlob("**/maven")
|
||||
require.NoError(t, err)
|
||||
assert.Empty(t, locations)
|
||||
}
|
||||
|
||||
// stereoscope finds a directory's children by looking the directory up by name: directories the archive
|
||||
// never listed must be indexed, and the root, which has no name, answered another way
|
||||
func TestResolver_globsForADirectorysChildren(t *testing.T) {
|
||||
r := resolverOver(t, 1<<20, map[string]string{
|
||||
"LICENSE": "root",
|
||||
"META-INF/LICENSE.txt": "nested",
|
||||
"META-INF/maven/g/a/pom.xml": "deeper",
|
||||
})
|
||||
|
||||
assert.Equal(t, []string{"/LICENSE"}, globPaths(t, r, "/*"))
|
||||
assert.Equal(t, []string{"/LICENSE"}, globPaths(t, r, "*"))
|
||||
assert.Equal(t, []string{"/META-INF/LICENSE.txt"}, globPaths(t, r, "/META-INF/*"))
|
||||
assert.Equal(t, []string{"/META-INF/LICENSE.txt"}, globPaths(t, r, "META-INF/*"))
|
||||
assert.Equal(t, []string{"/META-INF/maven/g/a/pom.xml"}, globPaths(t, r, "**/a/*"))
|
||||
}
|
||||
|
||||
func TestResolver_invalidGlobIsAnError(t *testing.T) {
|
||||
r := resolverOver(t, 1<<20, jarEntriesFixture)
|
||||
_, err := r.FilesByGlob("**/[")
|
||||
assert.Error(t, err)
|
||||
}
|
||||
|
||||
func globPaths(t *testing.T, r *Resolver, pattern string) []string {
|
||||
t.Helper()
|
||||
locations, err := r.FilesByGlob(pattern)
|
||||
require.NoError(t, err)
|
||||
return localPaths(locations)
|
||||
}
|
||||
|
||||
// several paths to one file answer as one location: the file's own path when it matched, otherwise
|
||||
// the lowest-sorting link
|
||||
func TestResolver_linksCollapseToOnePath(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
entries []testEntry
|
||||
pattern string
|
||||
want []string
|
||||
reason string
|
||||
}{
|
||||
{
|
||||
name: "the file itself beats every link to it",
|
||||
entries: []testEntry{
|
||||
{name: "opt/real.jar", body: "PK\x03\x04"},
|
||||
{name: "opt/a-sym.jar", link: "real.jar"},
|
||||
{name: "opt/z-sym.jar", link: "real.jar"},
|
||||
{name: "opt/hard.jar", link: "opt/real.jar", hard: true},
|
||||
},
|
||||
pattern: "**/*.jar",
|
||||
want: []string{"/opt/real.jar"},
|
||||
reason: "one file, so one location - and it is named by the path that holds it",
|
||||
},
|
||||
{
|
||||
name: "with no candidate naming the file, the lowest-sorting link wins",
|
||||
entries: []testEntry{
|
||||
{name: "opt/real.bin", body: "PK\x03\x04"},
|
||||
{name: "opt/z-sym.jar", link: "real.bin"},
|
||||
{name: "opt/a-sym.jar", link: "real.bin"},
|
||||
{name: "opt/m-sym.jar", link: "real.bin"},
|
||||
},
|
||||
pattern: "**/*.jar",
|
||||
want: []string{"/opt/real.bin"},
|
||||
reason: "the content is the target's, so that is the real path the answer carries",
|
||||
},
|
||||
{
|
||||
name: "distinct files stay distinct",
|
||||
entries: []testEntry{
|
||||
{name: "opt/one.jar", body: "one"},
|
||||
{name: "opt/two.jar", body: "two"},
|
||||
{name: "opt/one-sym.jar", link: "one.jar"},
|
||||
},
|
||||
pattern: "**/*.jar",
|
||||
want: []string{"/opt/one.jar", "/opt/two.jar"},
|
||||
reason: "collapsing is per file, not per pattern",
|
||||
},
|
||||
{
|
||||
name: "a dangling link is not a file",
|
||||
entries: []testEntry{
|
||||
{name: "opt/real.jar", body: "PK\x03\x04"},
|
||||
{name: "opt/gone.jar", link: "missing.jar"},
|
||||
},
|
||||
pattern: "**/*.jar",
|
||||
want: []string{"/opt/real.jar"},
|
||||
reason: "it names no content",
|
||||
},
|
||||
{
|
||||
name: "two dangling links are two paths, not one",
|
||||
entries: []testEntry{
|
||||
{name: "opt/a.jar", link: "missing.jar"},
|
||||
{name: "opt/b.jar", link: "missing.jar"},
|
||||
},
|
||||
pattern: "**/*.jar",
|
||||
want: nil,
|
||||
reason: "a link to nothing is not a file",
|
||||
},
|
||||
{
|
||||
name: "a chain of links collapses onto the file at its end",
|
||||
entries: []testEntry{
|
||||
{name: "opt/real.jar", body: "PK\x03\x04"},
|
||||
{name: "opt/mid.jar", link: "real.jar"},
|
||||
{name: "opt/outer.jar", link: "mid.jar"},
|
||||
},
|
||||
pattern: "**/*.jar",
|
||||
want: []string{"/opt/real.jar"},
|
||||
reason: "following one hop at a time reaches the same file from every path",
|
||||
},
|
||||
{
|
||||
name: "a cycle resolves to nothing rather than looping",
|
||||
entries: []testEntry{
|
||||
{name: "opt/a.jar", link: "b.jar"},
|
||||
{name: "opt/b.jar", link: "a.jar"},
|
||||
},
|
||||
pattern: "**/*.jar",
|
||||
want: nil,
|
||||
reason: "neither reaches a file",
|
||||
},
|
||||
{
|
||||
name: "a hard link resolves from the archive root, not from its own directory",
|
||||
entries: []testEntry{
|
||||
{name: "opt/real.jar", body: "PK\x03\x04"},
|
||||
{name: "opt/nested/hard.jar", link: "opt/real.jar", hard: true},
|
||||
},
|
||||
pattern: "**/*.jar",
|
||||
want: []string{"/opt/real.jar"},
|
||||
reason: "read as relative it would resolve to opt/nested/opt/real.jar and dangle",
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
r := resolverFrom(t, tt.entries...)
|
||||
assert.Equal(t, tt.want, globPaths(t, r, tt.pattern), tt.reason)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestResolver_collapsedLocationReadsTheTarget(t *testing.T) {
|
||||
r := resolverFrom(t,
|
||||
testEntry{name: "opt/real.bin", body: "PK\x03\x04 real contents"},
|
||||
testEntry{name: "opt/a-sym.jar", link: "real.bin"},
|
||||
testEntry{name: "opt/z-sym.jar", link: "real.bin"},
|
||||
)
|
||||
|
||||
locations, err := r.FilesByGlob("**/*.jar")
|
||||
require.NoError(t, err)
|
||||
require.Len(t, locations, 1)
|
||||
|
||||
assert.Equal(t, "/opt/real.bin", locations[0].LocalPath(), "content comes from the file")
|
||||
assert.Equal(t, "outer.jar!/opt/a-sym.jar", locations[0].AccessPath, "the path searched by is the one matched")
|
||||
|
||||
reader, err := r.FileContentsByLocation(locations[0])
|
||||
require.NoError(t, err)
|
||||
t.Cleanup(func() { require.NoError(t, reader.Close()) })
|
||||
|
||||
body, err := io.ReadAll(reader)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "PK\x03\x04 real contents", string(body),
|
||||
"a location standing for a link must still read the file's bytes, not the link's none")
|
||||
}
|
||||
|
||||
func TestResolver_fullPathsAreTrimmedToEntryPaths(t *testing.T) {
|
||||
// catalogers build queries from the full paths the resolver hands out, e.g. path.Dir plus a sibling
|
||||
r := resolverOver(t, 1<<20, map[string]string{"pom.xml": "a", "lib/dep.jar": "b"})
|
||||
|
||||
locations, err := r.FilesByPath("outer.jar!/lib/dep.jar")
|
||||
require.NoError(t, err)
|
||||
require.Len(t, locations, 1)
|
||||
assert.Equal(t, "outer.jar!/lib/dep.jar", locations[0].AccessPath)
|
||||
|
||||
locations, err = r.FilesByPath(path.Join(path.Dir(locations[0].AccessPath), "../pom.xml"))
|
||||
require.NoError(t, err)
|
||||
require.Len(t, locations, 1, "a sibling at the archive root is reached through the full path")
|
||||
assert.Equal(t, "outer.jar!/pom.xml", locations[0].FullNestedPath())
|
||||
|
||||
assert.True(t, r.HasPath(path.Dir("outer.jar!/pom.xml")), "the directory of a root entry is the root")
|
||||
assert.True(t, r.HasPath("lib/dep.jar"), "an entry path is still accepted")
|
||||
|
||||
locations, err = r.FilesByGlob("outer.jar!/**/*.jar")
|
||||
require.NoError(t, err)
|
||||
require.Len(t, locations, 1)
|
||||
}
|
||||
|
||||
func TestResolver_danglingLinkHasNoContent(t *testing.T) {
|
||||
r := resolverFrom(t, testEntry{name: "opt/gone.jar", link: "missing.jar"})
|
||||
|
||||
var locations []file.Location
|
||||
for l := range r.AllLocations(context.Background()) {
|
||||
locations = append(locations, l)
|
||||
}
|
||||
require.Len(t, locations, 1)
|
||||
assert.Equal(t, "/opt/gone.jar", locations[0].LocalPath())
|
||||
|
||||
_, err := r.FileContentsByLocation(locations[0])
|
||||
assert.Error(t, err, "a link to nothing has no content to read")
|
||||
|
||||
metadata, err := r.FileMetadataByLocation(locations[0])
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "outer.jar!/opt/missing.jar", metadata.LinkDestination,
|
||||
"where it pointed is still reported, resolved against the link's own directory")
|
||||
}
|
||||
|
||||
func TestResolver_mimeTypeAnswersOnlyWithFiles(t *testing.T) {
|
||||
r := resolverFrom(t,
|
||||
testEntry{name: "opt/real.jar", body: "PK\x03\x04 zip contents here"},
|
||||
testEntry{name: "opt/sym.jar", link: "real.jar"},
|
||||
testEntry{name: "opt/hard.jar", link: "opt/real.jar", hard: true},
|
||||
testEntry{name: "opt/gone.jar", link: "missing.jar"},
|
||||
testEntry{name: "opt/notes.txt", body: "hello world"},
|
||||
)
|
||||
|
||||
byMIME := func(types ...string) []string {
|
||||
t.Helper()
|
||||
locations, err := r.FilesByMIMEType(types...)
|
||||
require.NoError(t, err)
|
||||
return localPaths(locations)
|
||||
}
|
||||
|
||||
assert.Equal(t, []string{"/opt/real.jar"}, byMIME("application/zip"),
|
||||
"the file answers, not the two links to it")
|
||||
assert.Equal(t, []string{"/opt/notes.txt"}, byMIME("text/plain"))
|
||||
assert.Empty(t, byMIME(""),
|
||||
"a link sniffs to no type, and must not be reachable by asking for that")
|
||||
assert.Equal(t, []string{"/opt/notes.txt", "/opt/real.jar"}, byMIME("application/zip", "text/plain"))
|
||||
}
|
||||
|
||||
// AllLocations enumerates what the archive holds rather than searching for files: the file metadata
|
||||
// cataloger records a row per path, links included
|
||||
func TestResolver_allLocationsKeepsEveryPath(t *testing.T) {
|
||||
r := resolverFrom(t,
|
||||
testEntry{name: "opt/real.jar", body: "PK\x03\x04"},
|
||||
testEntry{name: "opt/sym.jar", link: "real.jar"},
|
||||
testEntry{name: "opt/gone.jar", link: "missing.jar"},
|
||||
)
|
||||
|
||||
var got []string
|
||||
for l := range r.AllLocations(context.Background()) {
|
||||
got = append(got, l.LocalPath())
|
||||
}
|
||||
assert.Equal(t, []string{"/opt/gone.jar", "/opt/real.jar", "/opt/sym.jar"}, got,
|
||||
"every path the archive names, links included")
|
||||
}
|
||||
|
||||
func TestResolver_add_anEntryOverTheCapIsSkippedAndReadNoFurther(t *testing.T) {
|
||||
// one entry of an overlapping-entry bomb can inflate without bound; it must stop at the cap, cost
|
||||
// the archive only that entry, and say so
|
||||
orig := maxEntryBytes
|
||||
maxEntryBytes = 1024
|
||||
t.Cleanup(func() { maxEntryBytes = orig })
|
||||
|
||||
r, _ := resolverIn(t, memCharge(-1))
|
||||
endless := &countingReader{r: zeroSource{}}
|
||||
require.NoError(t, r.add(tar.Header{Name: "bomb.bin", Typeflag: tar.TypeReg, Mode: 0o644}, endless))
|
||||
require.NoError(t, r.add(tar.Header{Name: "ok.txt", Typeflag: tar.TypeReg, Mode: 0o644}, strings.NewReader("ok")))
|
||||
r.finish()
|
||||
|
||||
assert.LessOrEqual(t, endless.n, 1024+copyChunkSize, "reading stops just past the cap")
|
||||
assert.ErrorIs(t, r.Truncated, errEntryTooLarge)
|
||||
assert.ErrorContains(t, r.Truncated, "per-entry size cap of 1.0 KiB")
|
||||
locations, err := r.FilesByGlob("**")
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, []string{"/ok.txt"}, localPaths(locations))
|
||||
}
|
||||
|
||||
type countingReader struct {
|
||||
r io.Reader
|
||||
n int
|
||||
}
|
||||
|
||||
func (c *countingReader) Read(p []byte) (int, error) {
|
||||
n, err := c.r.Read(p)
|
||||
c.n += n
|
||||
return n, err
|
||||
}
|
||||
@@ -1,217 +0,0 @@
|
||||
package archive
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
|
||||
"github.com/anchore/syft/internal/log"
|
||||
)
|
||||
|
||||
// copyChunkSize is how many bytes are read and charged at a time, bounding how far a charge runs
|
||||
// ahead of what is actually held.
|
||||
const copyChunkSize = 32 * 1024
|
||||
|
||||
// spillFilePattern names the one file an archive's content spills into, as in os.CreateTemp.
|
||||
const spillFilePattern = "archive-spill-*"
|
||||
|
||||
// ReaderAtSeeker is the random access an archive format needs: a zip is read from its central
|
||||
// directory at the end of the stream.
|
||||
type ReaderAtSeeker interface {
|
||||
io.Reader
|
||||
io.ReaderAt
|
||||
io.Seeker
|
||||
}
|
||||
|
||||
// blob is one piece of content the resolver holds: in memory, or at an offset in its spill file once
|
||||
// memory refused it. Moving changes nothing that points at the blob.
|
||||
type blob struct {
|
||||
mem []byte // content while held in memory
|
||||
off int64 // where content starts in the spill file once written there
|
||||
size int64
|
||||
}
|
||||
|
||||
// put reads content in full into b, since archive readers are sequential. Each chunk is charged before it
|
||||
// is held, so a stream larger than the memory limit is never held in full. Reaching the disk limit
|
||||
// returns ErrDiskLimitReached; what landed in the file before that stays there and stays charged.
|
||||
func (r *Resolver) put(b *blob, content io.Reader) error {
|
||||
if r.chunk == nil {
|
||||
r.chunk = make([]byte, copyChunkSize)
|
||||
}
|
||||
for {
|
||||
n, readErr := content.Read(r.chunk)
|
||||
if n > 0 {
|
||||
// content read through a meteredReader was charged as it was decoded
|
||||
if !r.metered && !r.budget.take(int64(n)) {
|
||||
return errBudgetSpent
|
||||
}
|
||||
if err := r.write(b, r.chunk[:n]); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
if readErr == nil {
|
||||
continue
|
||||
}
|
||||
if errors.Is(readErr, io.EOF) {
|
||||
break
|
||||
}
|
||||
return readErr
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// meteredReader charges the decompression budget for every byte read through it and stops once ctx is
|
||||
// done, so nothing a decoder produces gets around the budget.
|
||||
type meteredReader struct {
|
||||
ctx context.Context
|
||||
reader io.Reader
|
||||
budget *Budget
|
||||
}
|
||||
|
||||
func (m *meteredReader) Read(p []byte) (int, error) {
|
||||
if err := m.ctx.Err(); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
n, err := m.reader.Read(p)
|
||||
if n > 0 && !m.budget.take(int64(n)) {
|
||||
return 0, errBudgetSpent
|
||||
}
|
||||
return n, err
|
||||
}
|
||||
|
||||
// write appends p to b: into memory while the memory limit admits it, otherwise into the spill file
|
||||
// after everything held in memory is moved there first.
|
||||
func (r *Resolver) write(b *blob, p []byte) error {
|
||||
n := int64(len(p))
|
||||
onDisk := b.mem == nil && b.size > 0
|
||||
if !onDisk {
|
||||
if r.charge.memory(n) {
|
||||
if b.mem == nil {
|
||||
r.held = append(r.held, b)
|
||||
}
|
||||
b.mem = append(b.mem, p...)
|
||||
b.size += n
|
||||
return nil
|
||||
}
|
||||
if err := r.spill(); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
if err := r.openFile(); err != nil {
|
||||
return err
|
||||
}
|
||||
if !r.charge.disk(n) {
|
||||
return ErrDiskLimitReached
|
||||
}
|
||||
if b.size == 0 {
|
||||
b.off = r.written
|
||||
}
|
||||
// b is always the blob most recently written, so it ends where the file does
|
||||
w, err := r.file.WriteAt(p, r.written)
|
||||
r.written += int64(w)
|
||||
b.size += int64(w)
|
||||
r.charge.refundDisk(n - int64(w))
|
||||
if err != nil {
|
||||
return fmt.Errorf("unable to write archive content to disk: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// spill moves every blob held in memory into the spill file, charging the disk limit as bytes land
|
||||
// and refunding the memory they leave.
|
||||
func (r *Resolver) spill() error {
|
||||
if len(r.held) == 0 {
|
||||
return nil
|
||||
}
|
||||
if err := r.openFile(); err != nil {
|
||||
return err
|
||||
}
|
||||
log.WithFields("archive", r.archiveCoordinates.FullNestedPath(), "held", len(r.held), "bytes", r.written).
|
||||
Debug("archive content does not fit in memory; writing it to disk")
|
||||
|
||||
for len(r.held) > 0 {
|
||||
b := r.held[0]
|
||||
n := int64(len(b.mem))
|
||||
if !r.charge.disk(n) {
|
||||
return ErrDiskLimitReached
|
||||
}
|
||||
if _, err := r.file.WriteAt(b.mem, r.written); err != nil {
|
||||
return fmt.Errorf("unable to write archive content to disk: %w", err)
|
||||
}
|
||||
b.off, b.mem = r.written, nil
|
||||
r.written += n
|
||||
r.charge.refundMemory(n)
|
||||
r.held = r.held[1:]
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (r *Resolver) openFile() error {
|
||||
if r.file != nil {
|
||||
return nil
|
||||
}
|
||||
var err error
|
||||
if r.tempDir != nil {
|
||||
r.file, r.remove, err = r.tempDir.NewFile(spillFilePattern)
|
||||
} else {
|
||||
r.file, err = os.CreateTemp("", spillFilePattern)
|
||||
r.remove = func() {
|
||||
if err := os.Remove(r.file.Name()); err != nil {
|
||||
log.WithFields("path", r.file.Name(), "error", err).Trace("unable to remove archive temp file")
|
||||
}
|
||||
}
|
||||
}
|
||||
if err != nil {
|
||||
return fmt.Errorf("unable to create archive temp file: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// discard drops a blob's bytes held in memory and refunds them. Bytes it holds on disk stay until
|
||||
// Cleanup.
|
||||
func (r *Resolver) discard(b *blob) {
|
||||
for i, held := range r.held {
|
||||
if held == b {
|
||||
r.held = append(r.held[:i], r.held[i+1:]...)
|
||||
break
|
||||
}
|
||||
}
|
||||
r.charge.refundMemory(int64(len(b.mem)))
|
||||
b.mem, b.size = nil, 0
|
||||
}
|
||||
|
||||
// open returns a reader over a blob's content, wherever it is. The reader supports Read, Seek and
|
||||
// ReadAt, so a nested archive can be read where it lies.
|
||||
func (r *Resolver) open(b *blob) ReaderAtSeeker {
|
||||
switch {
|
||||
case b.mem != nil:
|
||||
return bytes.NewReader(b.mem)
|
||||
case b.size == 0:
|
||||
return bytes.NewReader(nil)
|
||||
default:
|
||||
return io.NewSectionReader(r.file, b.off, b.size)
|
||||
}
|
||||
}
|
||||
|
||||
// releaseStorage refunds every blob held in memory and closes, removes and refunds the spill file.
|
||||
// Readers from open must not be used after this. Safe to call more than once.
|
||||
func (r *Resolver) releaseStorage() {
|
||||
for _, b := range r.held {
|
||||
r.charge.refundMemory(int64(len(b.mem)))
|
||||
b.mem = nil
|
||||
}
|
||||
r.held = nil
|
||||
if r.file == nil {
|
||||
return
|
||||
}
|
||||
if err := r.file.Close(); err != nil {
|
||||
log.WithFields("archive", r.archiveCoordinates.FullNestedPath(), "error", err).Trace("unable to close archive temp file")
|
||||
}
|
||||
r.remove()
|
||||
r.charge.refundDisk(r.written)
|
||||
r.file, r.remove, r.written = nil, nil, 0
|
||||
}
|
||||
@@ -1,276 +0,0 @@
|
||||
package archive
|
||||
|
||||
import (
|
||||
"archive/zip"
|
||||
"bytes"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestResolver_put_holdsSmallContentInMemory(t *testing.T) {
|
||||
s, dir := resolverIn(t, memCharge(1024))
|
||||
|
||||
var b blob
|
||||
require.NoError(t, s.put(&b, bytes.NewReader([]byte("hello"))))
|
||||
|
||||
assert.Equal(t, int64(5), s.heldInMemory())
|
||||
assert.Empty(t, filesIn(t, dir))
|
||||
assert.Equal(t, "hello", readEntry(t, s.open(&b)))
|
||||
}
|
||||
|
||||
func TestResolver_put_spillsToDiskAndKeepsTheSameBlobs(t *testing.T) {
|
||||
// nodes pointing at a blob are not rebuilt when its content moves
|
||||
s, dir := resolverIn(t, memCharge(8))
|
||||
|
||||
var first, second blob
|
||||
require.NoError(t, s.put(&first, bytes.NewReader([]byte("aaaa"))))
|
||||
require.Equal(t, int64(4), s.heldInMemory())
|
||||
|
||||
require.NoError(t, s.put(&second, bytes.NewReader([]byte("bbbbbb"))))
|
||||
|
||||
assert.Zero(t, s.heldInMemory(), "everything moves out, not just the blob that did not fit")
|
||||
assert.NotEmpty(t, spillFile(t, dir))
|
||||
assert.Equal(t, int64(10), s.written)
|
||||
assert.Equal(t, "aaaa", readEntry(t, s.open(&first)))
|
||||
assert.Equal(t, "bbbbbb", readEntry(t, s.open(&second)))
|
||||
}
|
||||
|
||||
func TestResolver_put_threeStateLimits(t *testing.T) {
|
||||
body := []byte(strings.Repeat("e", 500))
|
||||
|
||||
t.Run("a zero memory limit sends everything to disk", func(t *testing.T) {
|
||||
charge := NewLimiter(Limits{MaxMemoryBytes: 0, MaxDiskBytes: 10_000}).charge()
|
||||
s, dir := resolverIn(t, charge)
|
||||
|
||||
var b blob
|
||||
require.NoError(t, s.put(&b, bytes.NewReader(body)))
|
||||
|
||||
assert.Zero(t, s.heldInMemory())
|
||||
assert.NotEmpty(t, spillFile(t, dir))
|
||||
assert.Equal(t, body, []byte(readEntry(t, s.open(&b))))
|
||||
mem, disk := charge.held()
|
||||
assert.Zero(t, mem)
|
||||
assert.Equal(t, int64(500), disk)
|
||||
})
|
||||
|
||||
t.Run("a negative memory limit holds content regardless of how much is already held", func(t *testing.T) {
|
||||
charge := NewLimiter(Limits{MaxMemoryBytes: -1, MaxDiskBytes: 10_000}).charge()
|
||||
require.True(t, charge.memory(1_000_000))
|
||||
s, dir := resolverIn(t, charge)
|
||||
|
||||
var b blob
|
||||
require.NoError(t, s.put(&b, bytes.NewReader(body)))
|
||||
|
||||
assert.Empty(t, filesIn(t, dir))
|
||||
mem, disk := charge.held()
|
||||
assert.Equal(t, int64(1_000_000+len(body)), mem)
|
||||
assert.Zero(t, disk)
|
||||
})
|
||||
|
||||
t.Run("a zero disk limit still holds what memory admits", func(t *testing.T) {
|
||||
charge := NewLimiter(Limits{MaxMemoryBytes: 10_000, MaxDiskBytes: 0}).charge()
|
||||
s, dir := resolverIn(t, charge)
|
||||
|
||||
var b blob
|
||||
require.NoError(t, s.put(&b, bytes.NewReader(body)))
|
||||
|
||||
assert.Empty(t, filesIn(t, dir))
|
||||
mem, disk := charge.held()
|
||||
assert.Equal(t, int64(len(body)), mem)
|
||||
assert.Zero(t, disk)
|
||||
})
|
||||
|
||||
t.Run("a negative disk limit writes with no ceiling", func(t *testing.T) {
|
||||
s, dir := resolverIn(t, NewLimiter(Limits{MaxMemoryBytes: 0, MaxDiskBytes: -1}).charge())
|
||||
|
||||
var b blob
|
||||
require.NoError(t, s.put(&b, bytes.NewReader(body)))
|
||||
|
||||
assert.NotEmpty(t, spillFile(t, dir))
|
||||
assert.Equal(t, body, []byte(readEntry(t, s.open(&b))))
|
||||
})
|
||||
|
||||
t.Run("a nil charge holds everything in memory", func(t *testing.T) {
|
||||
s, dir := resolverIn(t, nil)
|
||||
|
||||
var b blob
|
||||
require.NoError(t, s.put(&b, bytes.NewReader(body)))
|
||||
|
||||
assert.Empty(t, filesIn(t, dir))
|
||||
assert.Equal(t, body, []byte(readEntry(t, s.open(&b))))
|
||||
})
|
||||
}
|
||||
|
||||
func TestResolver_put_bytesReadBeforeTheLimitRefusedAreNotLost(t *testing.T) {
|
||||
body := []byte(strings.Repeat("b", 250))
|
||||
charge := NewLimiter(Limits{MaxMemoryBytes: 100, MaxDiskBytes: 10_000}).charge()
|
||||
s, _ := resolverIn(t, charge)
|
||||
|
||||
var b blob
|
||||
require.NoError(t, s.put(&b, bytes.NewReader(body)))
|
||||
|
||||
assert.Equal(t, body, []byte(readEntry(t, s.open(&b))))
|
||||
mem, disk := charge.held()
|
||||
assert.Zero(t, mem, "nothing stays charged to a limit that refused")
|
||||
assert.Equal(t, int64(250), disk)
|
||||
}
|
||||
|
||||
// peakTrackingReader records the highest memory charge observed while being read.
|
||||
type peakTrackingReader struct {
|
||||
r io.Reader
|
||||
charge *charge
|
||||
peak int64
|
||||
}
|
||||
|
||||
func (p *peakTrackingReader) Read(b []byte) (int, error) {
|
||||
n, err := p.r.Read(b)
|
||||
if mem, _ := p.charge.held(); mem > p.peak {
|
||||
p.peak = mem
|
||||
}
|
||||
return n, err
|
||||
}
|
||||
|
||||
func TestResolver_put_neverBuffersBeyondTheMemoryLimit(t *testing.T) {
|
||||
const limit = int64(3 * copyChunkSize)
|
||||
body := []byte(strings.Repeat("c", 10*copyChunkSize))
|
||||
charge := NewLimiter(Limits{MaxMemoryBytes: limit, MaxDiskBytes: int64(len(body)) + 1}).charge()
|
||||
tracked := &peakTrackingReader{r: bytes.NewReader(body), charge: charge}
|
||||
s, _ := resolverIn(t, charge)
|
||||
|
||||
var b blob
|
||||
require.NoError(t, s.put(&b, tracked))
|
||||
|
||||
assert.Equal(t, body, []byte(readEntry(t, s.open(&b))))
|
||||
assert.LessOrEqual(t, tracked.peak, limit, "memory charged must never exceed the limit, even transiently")
|
||||
mem, disk := charge.held()
|
||||
assert.Zero(t, mem)
|
||||
assert.Equal(t, int64(len(body)), disk)
|
||||
}
|
||||
|
||||
func TestResolver_put_diskLimitIsTerminal(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
limits Limits
|
||||
}{
|
||||
{"a disk limit that is exceeded", Limits{MaxDiskBytes: 100}},
|
||||
{"a zero disk limit", Limits{MaxMemoryBytes: 10, MaxDiskBytes: 0}},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
charge := NewLimiter(tt.limits).charge()
|
||||
s, dir := resolverIn(t, charge)
|
||||
|
||||
var b blob
|
||||
err := s.put(&b, bytes.NewReader([]byte(strings.Repeat("d", 500))))
|
||||
require.ErrorIs(t, err, ErrDiskLimitReached)
|
||||
|
||||
s.Cleanup()
|
||||
assert.Empty(t, filesIn(t, dir), "cleanup removes the partial file")
|
||||
mem, disk := charge.held()
|
||||
assert.Zero(t, mem)
|
||||
assert.Zero(t, disk, "and refunds it")
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestResolver_cleanupRefundsEverythingAndIsIdempotent(t *testing.T) {
|
||||
charge := NewLimiter(Limits{MaxMemoryBytes: 8, MaxDiskBytes: -1}).charge()
|
||||
s, dir := resolverIn(t, charge)
|
||||
|
||||
var spilled, held blob
|
||||
require.NoError(t, s.put(&spilled, bytes.NewReader([]byte("spilled to disk"))))
|
||||
require.NoError(t, s.put(&held, bytes.NewReader([]byte("held"))))
|
||||
mem, disk := charge.held()
|
||||
require.Equal(t, int64(4), mem)
|
||||
require.Equal(t, int64(15), disk)
|
||||
require.Equal(t, "held", readEntry(t, s.open(&held)))
|
||||
|
||||
s.Cleanup()
|
||||
|
||||
assert.Empty(t, filesIn(t, dir))
|
||||
mem, disk = charge.held()
|
||||
assert.Zero(t, mem)
|
||||
assert.Zero(t, disk)
|
||||
_, err := io.ReadAll(s.open(&spilled))
|
||||
assert.Error(t, err, "readers over the spill file are no longer valid")
|
||||
s.Cleanup()
|
||||
mem, disk = charge.held()
|
||||
assert.Zero(t, mem, "a second cleanup refunds nothing twice")
|
||||
assert.Zero(t, disk)
|
||||
}
|
||||
|
||||
func TestResolver_put_discardRefundsWhatIsHeldInMemory(t *testing.T) {
|
||||
charge := memCharge(1024)
|
||||
s, _ := resolverIn(t, charge)
|
||||
|
||||
var keep, drop blob
|
||||
require.NoError(t, s.put(&keep, bytes.NewReader([]byte("keep"))))
|
||||
require.NoError(t, s.put(&drop, bytes.NewReader([]byte("dropped"))))
|
||||
|
||||
s.discard(&drop)
|
||||
|
||||
mem, _ := charge.held()
|
||||
assert.Equal(t, int64(4), mem)
|
||||
assert.Equal(t, int64(4), s.heldInMemory())
|
||||
assert.Empty(t, readEntry(t, s.open(&drop)))
|
||||
assert.Equal(t, "keep", readEntry(t, s.open(&keep)))
|
||||
}
|
||||
|
||||
func TestResolver_filesystemCostDoesNotTrackEntryCount(t *testing.T) {
|
||||
small, smallRoot := extractedResolver(t, manyEntryZip(t, 2), spillingLimits)
|
||||
large, largeRoot := extractedResolver(t, manyEntryZip(t, 5000), spillingLimits)
|
||||
|
||||
assert.Len(t, filesIn(t, smallRoot), 1)
|
||||
assert.Len(t, filesIn(t, largeRoot), 1, "5000 entries cost the same filesystem entries as 2")
|
||||
|
||||
assert.Equal(t, 2, small.entries)
|
||||
assert.Equal(t, 5000, large.entries)
|
||||
|
||||
// entries cost their content and nothing else on disk: no headers, no padding
|
||||
assert.Equal(t, int64(2), small.written)
|
||||
assert.Equal(t, int64(5000), large.written)
|
||||
}
|
||||
|
||||
func TestResolver_anEntryAtTheEndIsReadBySeeking(t *testing.T) {
|
||||
// the reader over an entry is a bounded window at its offset, not a re-read of what precedes it
|
||||
const filler = 64 * 1024
|
||||
|
||||
var buf bytes.Buffer
|
||||
zw := zip.NewWriter(&buf)
|
||||
for i := range 8 {
|
||||
w, err := zw.Create(fmt.Sprintf("filler%d.bin", i))
|
||||
require.NoError(t, err)
|
||||
_, err = w.Write(bytes.Repeat([]byte{byte(i)}, filler))
|
||||
require.NoError(t, err)
|
||||
}
|
||||
last, err := zw.Create("zz-last.txt")
|
||||
require.NoError(t, err)
|
||||
wanted := strings.Repeat("last", 64)
|
||||
_, err = last.Write([]byte(wanted))
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, zw.Close())
|
||||
|
||||
r, root := extractedResolver(t, buf.Bytes(), spillingLimits)
|
||||
|
||||
body, err := os.ReadFile(filepath.Join(root, spillFile(t, root)))
|
||||
require.NoError(t, err)
|
||||
offset := int64(bytes.Index(body, []byte(wanted)))
|
||||
require.Greater(t, offset, int64(len(body))*9/10, "the last entry must really be at the end of the file")
|
||||
|
||||
reader := r.open(contentAt(t, r, "/zz-last.txt"))
|
||||
end, err := reader.Seek(0, io.SeekEnd)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, int64(len(wanted)), end)
|
||||
|
||||
head := make([]byte, 8)
|
||||
_, err = reader.ReadAt(head, 0)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, wanted[:8], string(head))
|
||||
}
|
||||
@@ -1,194 +0,0 @@
|
||||
package archive
|
||||
|
||||
import (
|
||||
"context"
|
||||
"path"
|
||||
"slices"
|
||||
"strings"
|
||||
|
||||
"github.com/anchore/syft/syft/file"
|
||||
"github.com/anchore/syft/syft/pkg"
|
||||
)
|
||||
|
||||
// Traversal describes the archive whose contents a cataloger is running against. The archive
|
||||
// cataloger task puts one on the context before running catalogers inside an archive, so a cataloger
|
||||
// such as java can describe the archive it is inside without owning the recursion.
|
||||
type Traversal struct {
|
||||
// Location is the archive file's location within its parent filesystem, including the access path
|
||||
// it was reached by, so VirtualPath(Location) is the chain from the scan root as seen through that
|
||||
// access path. The files inside it extend its real coordinates instead (see ContainerPath).
|
||||
Location file.Location
|
||||
|
||||
// Digests are of the archive file itself, taken during extraction; a cataloger running inside the
|
||||
// archive cannot compute them since the archive is not inside itself.
|
||||
Digests []file.Digest
|
||||
|
||||
// Parent is the traversal of the archive this one is nested in, nil for an archive at the scan root.
|
||||
Parent *Traversal
|
||||
|
||||
// Packages are those the catalogers found describing this archive (located at it), filled in once
|
||||
// they have run. Catalogers inside a nested archive read them through Parent, as java does to link a
|
||||
// nested jar to the jar that holds it.
|
||||
Packages []pkg.Package
|
||||
}
|
||||
|
||||
// FullPathIn is the full path of the entry at entryPath inside the archive whose full path is
|
||||
// containerPath, or entryPath itself for a blank containerPath (the scan root).
|
||||
func FullPathIn(containerPath, entryPath string) string {
|
||||
if containerPath == "" {
|
||||
return entryPath
|
||||
}
|
||||
// cleaning an already clean path allocates nothing
|
||||
return containerPath + file.ArchivePathSeparator + strings.TrimPrefix(path.Clean(entryPath), "/")
|
||||
}
|
||||
|
||||
// OwnerContainers are the containers a package owns files within: the container of each of its
|
||||
// locations, each once, as coordinates whose FullPath is the container's full path (blank at the scan
|
||||
// root). A package without locations owns files at the scan root. See OwnedCoordinates.
|
||||
func OwnerContainers(p pkg.Package) []file.Coordinates {
|
||||
var out []file.Coordinates
|
||||
for _, l := range p.Locations.ToUnorderedSlice() {
|
||||
c := file.Coordinates{FileSystemID: l.FileSystemID}
|
||||
if container := ContainerPath(l.Coordinates); container != "" {
|
||||
c.RealPath, c.FullPath = l.RealPath, container
|
||||
}
|
||||
if !slices.Contains(out, c) {
|
||||
out = append(out, c)
|
||||
}
|
||||
}
|
||||
if len(out) == 0 {
|
||||
out = append(out, file.Coordinates{})
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// OwnedCoordinates are the coordinates of the file at ownedPath within container, one of
|
||||
// OwnerContainers. At the scan root the path may still need resolving (it may pass through links).
|
||||
func OwnedCoordinates(container file.Coordinates, ownedPath string) file.Coordinates {
|
||||
if container.FullPath == "" {
|
||||
return file.Coordinates{RealPath: ownedPath, FileSystemID: container.FileSystemID}
|
||||
}
|
||||
return file.Coordinates{
|
||||
RealPath: container.RealPath,
|
||||
FileSystemID: container.FileSystemID,
|
||||
FullPath: FullPathIn(container.FullPath, ownedPath),
|
||||
}
|
||||
}
|
||||
|
||||
// FullTraversalArchivePath is the archive's own full path, which is the ContainerPath of every file inside it. Outside
|
||||
// any archive (a nil Traversal) it is blank, as for files on the scanned filesystem.
|
||||
func (t *Traversal) FullTraversalArchivePath() string {
|
||||
if t == nil {
|
||||
return ""
|
||||
}
|
||||
return t.Location.FullNestedPath()
|
||||
}
|
||||
|
||||
// ContainerPath returns the full path of the archive directly holding the file at c, or blank for a
|
||||
// file on the scanned filesystem, whether or not its FullPath is set.
|
||||
func ContainerPath(c file.Coordinates) string {
|
||||
entries, ok := archiveEntries(c)
|
||||
if !ok {
|
||||
return ""
|
||||
}
|
||||
if i := strings.LastIndex(entries, file.ArchivePathSeparator); i >= 0 {
|
||||
return c.RealPath + file.ArchivePathSeparator + entries[:i]
|
||||
}
|
||||
return c.RealPath
|
||||
}
|
||||
|
||||
// ContainingArchive returns the coordinates of the archive directly holding the file at c, reporting false
|
||||
// for a file on the scanned filesystem.
|
||||
func ContainingArchive(c file.Coordinates) (file.Coordinates, bool) {
|
||||
containerPath := ContainerPath(c)
|
||||
if containerPath == "" {
|
||||
return file.Coordinates{}, false
|
||||
}
|
||||
container := file.Coordinates{RealPath: c.RealPath, FileSystemID: c.FileSystemID}
|
||||
if containerPath != c.RealPath {
|
||||
container.FullPath = containerPath
|
||||
}
|
||||
return container, true
|
||||
}
|
||||
|
||||
// archiveEntries is the "!/"-delimited chain of entry paths after RealPath, reporting false for a file
|
||||
// not inside an archive. RealPath is split off first, so a "!/" in it is not taken for a boundary.
|
||||
func archiveEntries(c file.Coordinates) (string, bool) {
|
||||
remain, ok := strings.CutPrefix(c.FullNestedPath(), c.RealPath)
|
||||
if !ok {
|
||||
return "", false
|
||||
}
|
||||
return strings.CutPrefix(remain, file.ArchivePathSeparator)
|
||||
}
|
||||
|
||||
// VirtualPath returns the colon-delimited chain of archives from the scan root to loc, ending in loc
|
||||
// itself as accessed: "app.war:WEB-INF/lib/dep.jar". Outside any archive it is the location's path
|
||||
// unchanged. In an entry name '%' is escaped as "%25" and then ':' as "%3A", so splitting on ':'
|
||||
// recovers the archive boundaries and no two entry names share an escaped form. Leading slashes are
|
||||
// dropped to match the java cataloger, which joins slash-less zip entry names onto the containing
|
||||
// archive's path.
|
||||
func VirtualPath(loc file.Location) string {
|
||||
accessPath := loc.Path()
|
||||
entries, ok := archiveEntries(loc.Coordinates)
|
||||
if !ok {
|
||||
return accessPath
|
||||
}
|
||||
segments := strings.Split(entries, file.ArchivePathSeparator)
|
||||
if rest, ok := strings.CutPrefix(accessPath, ContainerPath(loc.Coordinates)); ok && strings.HasPrefix(rest, file.ArchivePathSeparator) {
|
||||
accessPath = rest[len(file.ArchivePathSeparator):]
|
||||
}
|
||||
segments[len(segments)-1] = accessPath
|
||||
for i, s := range segments {
|
||||
segments[i] = strings.TrimPrefix(escapeEntryPath(s), "/")
|
||||
}
|
||||
return loc.RealPath + ":" + strings.Join(segments, ":")
|
||||
}
|
||||
|
||||
var entryPathEscaper = strings.NewReplacer("%", "%25", ":", "%3A")
|
||||
|
||||
func escapeEntryPath(p string) string {
|
||||
if !strings.ContainsAny(p, "%:") {
|
||||
return p
|
||||
}
|
||||
return entryPathEscaper.Replace(p)
|
||||
}
|
||||
|
||||
type traversalCtxKey struct{}
|
||||
|
||||
func WithTraversal(ctx context.Context, t *Traversal) context.Context {
|
||||
return context.WithValue(ctx, traversalCtxKey{}, t)
|
||||
}
|
||||
|
||||
// TraversalFromContext returns the archive traversal on the context, or nil outside an archive.
|
||||
func TraversalFromContext(ctx context.Context) *Traversal {
|
||||
t, _ := ctx.Value(traversalCtxKey{}).(*Traversal)
|
||||
return t
|
||||
}
|
||||
|
||||
type budgetCtxKey struct{}
|
||||
|
||||
// WithBudget sets the decompression budget that Extract spends from, for this archive and every archive
|
||||
// extracted beneath it.
|
||||
func WithBudget(ctx context.Context, b *Budget) context.Context {
|
||||
return context.WithValue(ctx, budgetCtxKey{}, b)
|
||||
}
|
||||
|
||||
func budgetFromContext(ctx context.Context) *Budget {
|
||||
b, _ := ctx.Value(budgetCtxKey{}).(*Budget)
|
||||
return b
|
||||
}
|
||||
|
||||
type nestedCatalogingCtxKey struct{}
|
||||
|
||||
// WithNestedCataloging marks the context of a scan whose archive cataloger task will extract every
|
||||
// archive and run the catalogers inside it. syft.CreateSBOM sets it before any task runs, so a
|
||||
// cataloger that would otherwise open archives itself can leave them to the task instead.
|
||||
func WithNestedCataloging(ctx context.Context) context.Context {
|
||||
return context.WithValue(ctx, nestedCatalogingCtxKey{}, true)
|
||||
}
|
||||
|
||||
// NestedCatalogingEnabled reports whether the archive cataloger task is extracting archives in this scan.
|
||||
func NestedCatalogingEnabled(ctx context.Context) bool {
|
||||
enabled, _ := ctx.Value(nestedCatalogingCtxKey{}).(bool)
|
||||
return enabled
|
||||
}
|
||||
@@ -1,113 +0,0 @@
|
||||
package archive
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"github.com/anchore/syft/syft/file"
|
||||
)
|
||||
|
||||
func TestTraversalContextRoundTrip(t *testing.T) {
|
||||
assert.Nil(t, TraversalFromContext(context.Background()))
|
||||
|
||||
trav := &Traversal{Location: file.NewLocation("app.war")}
|
||||
ctx := WithTraversal(context.Background(), trav)
|
||||
require.Same(t, trav, TraversalFromContext(ctx))
|
||||
|
||||
child := &Traversal{Location: locationIn("app.war", "WEB-INF/lib/dep.jar")}
|
||||
ctx = WithTraversal(ctx, child)
|
||||
require.Same(t, child, TraversalFromContext(ctx))
|
||||
}
|
||||
|
||||
// locationIn is a location as the Resolver returns it: the outermost archive's real path, and the chain
|
||||
// of entry paths within it.
|
||||
func locationIn(realPath, archivePath string) file.Location {
|
||||
return file.NewLocationFromCoordinates(file.Coordinates{RealPath: realPath, FullPath: realPath + file.ArchivePathSeparator + archivePath})
|
||||
}
|
||||
|
||||
func TestContainerPath(t *testing.T) {
|
||||
assert.Equal(t, "", ContainerPath(file.NewCoordinates("/app.war", "")))
|
||||
assert.Equal(t, "", ContainerPath(file.Coordinates{RealPath: "/app.war", FullPath: "/app.war"}), "a top-level file whose FullPath is set")
|
||||
assert.Equal(t, "", ContainerPath(file.Coordinates{RealPath: "/odd!/name"}), "a separator in a top-level real path is not an archive boundary")
|
||||
assert.Equal(t, "", ContainerPath(file.Coordinates{RealPath: "/odd!/name", FullPath: "/odd!/name"}), "nor when FullPath is set")
|
||||
assert.Equal(t, "/odd!/name.zip", ContainerPath(locationIn("/odd!/name.zip", "a.txt").Coordinates))
|
||||
assert.Equal(t, "/odd!/name.zip!/b.jar", ContainerPath(locationIn("/odd!/name.zip", "b.jar!/a.txt").Coordinates))
|
||||
assert.Equal(t, "/app.war", ContainerPath(locationIn("/app.war", "WEB-INF/lib/dep.jar").Coordinates))
|
||||
assert.Equal(t, "/app.war!/WEB-INF/lib/dep.jar",
|
||||
ContainerPath(locationIn("/app.war", "WEB-INF/lib/dep.jar!/META-INF/MANIFEST.MF").Coordinates))
|
||||
}
|
||||
|
||||
func TestVirtualPath(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
location file.Location
|
||||
expected string
|
||||
}{
|
||||
{
|
||||
name: "outside an archive returns the path unchanged",
|
||||
location: file.NewLocation("/WEB-INF/lib/dep.jar"),
|
||||
expected: "/WEB-INF/lib/dep.jar",
|
||||
},
|
||||
{
|
||||
name: "outside an archive preserves a relative path",
|
||||
location: file.NewLocation("some/dir/example.jar"),
|
||||
expected: "some/dir/example.jar",
|
||||
},
|
||||
{
|
||||
name: "outside an archive with FullPath set returns the path unchanged",
|
||||
location: file.NewLocationFromCoordinates(file.Coordinates{RealPath: "/odd!/name.jar", FullPath: "/odd!/name.jar"}),
|
||||
expected: "/odd!/name.jar",
|
||||
},
|
||||
{
|
||||
name: "single level trims leading slash",
|
||||
location: locationIn("app.war", "/WEB-INF/lib/dep.jar"),
|
||||
expected: "app.war:WEB-INF/lib/dep.jar",
|
||||
},
|
||||
{
|
||||
name: "single level with slash-less entry",
|
||||
location: locationIn("app.war", "WEB-INF/lib/dep.jar"),
|
||||
expected: "app.war:WEB-INF/lib/dep.jar",
|
||||
},
|
||||
{
|
||||
name: "multi-level chain",
|
||||
location: locationIn("app.war", "WEB-INF/lib/dep.jar!/inner/most.jar"),
|
||||
expected: "app.war:WEB-INF/lib/dep.jar:inner/most.jar",
|
||||
},
|
||||
{
|
||||
// a colon in the entry name must not be mistaken for a chain delimiter
|
||||
name: "colon in entry path is escaped",
|
||||
location: locationIn("app.war", "WEB-INF/lib/dep:special.jar"),
|
||||
expected: "app.war:WEB-INF/lib/dep%3Aspecial.jar",
|
||||
},
|
||||
{
|
||||
name: "every colon in the entry path is escaped",
|
||||
location: locationIn("app.war", "lib/a:b:c.jar"),
|
||||
expected: "app.war:lib/a%3Ab%3Ac.jar",
|
||||
},
|
||||
{
|
||||
// or the literal text %3A would collide with an escaped colon
|
||||
name: "the escape character is itself escaped",
|
||||
location: locationIn("outer.zip", "a%3Ab.jar"),
|
||||
expected: "outer.zip:a%253Ab.jar",
|
||||
},
|
||||
{
|
||||
name: "access path is preferred over real path",
|
||||
location: file.NewVirtualLocation("/real/dep.jar", "/link/dep.jar"),
|
||||
expected: "/link/dep.jar",
|
||||
},
|
||||
{
|
||||
name: "access path within an archive replaces the last entry",
|
||||
location: file.NewVirtualLocationFromCoordinates(
|
||||
file.Coordinates{RealPath: "app.war", FullPath: "app.war!/WEB-INF/lib/real.jar"}, "app.war!/WEB-INF/lib/link.jar"),
|
||||
expected: "app.war:WEB-INF/lib/link.jar",
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
assert.Equal(t, tt.expected, VirtualPath(tt.location))
|
||||
})
|
||||
}
|
||||
}
|
||||
+1
-79
@@ -1,9 +1,6 @@
|
||||
package bus
|
||||
|
||||
import (
|
||||
"context"
|
||||
"sync"
|
||||
|
||||
"github.com/wagoodman/go-partybus"
|
||||
"github.com/wagoodman/go-progress"
|
||||
|
||||
@@ -39,82 +36,7 @@ func Notify(message string) {
|
||||
})
|
||||
}
|
||||
|
||||
type catalogerTaskRegistryKey struct{}
|
||||
|
||||
// catalogerTaskRegistry remembers the progress row each cataloger publishes. A cataloger runs once
|
||||
// over the scan root and again inside every archive the archive cataloger walks; a re-run reports
|
||||
// into the row it already owns rather than publishing another, and republishing an ID would replace
|
||||
// the live row with one the UI never sees completed.
|
||||
type catalogerTaskRegistry struct {
|
||||
mu sync.Mutex
|
||||
rows map[string]*monitor.TaskProgress
|
||||
}
|
||||
|
||||
// WithCatalogerTaskRegistry returns a context under which each cataloger publishes one progress row
|
||||
// however many times it runs. A context that already carries a registry is returned unchanged.
|
||||
func WithCatalogerTaskRegistry(ctx context.Context) context.Context {
|
||||
if catalogerTaskRegistryFromContext(ctx) != nil {
|
||||
return ctx
|
||||
}
|
||||
return context.WithValue(ctx, catalogerTaskRegistryKey{}, &catalogerTaskRegistry{
|
||||
rows: make(map[string]*monitor.TaskProgress),
|
||||
})
|
||||
}
|
||||
|
||||
func catalogerTaskRegistryFromContext(ctx context.Context) *catalogerTaskRegistry {
|
||||
reg, _ := ctx.Value(catalogerTaskRegistryKey{}).(*catalogerTaskRegistry)
|
||||
return reg
|
||||
}
|
||||
|
||||
// catalogerTaskKey identifies the row a task owns: its ID when it has one, otherwise its parent and
|
||||
// title (file catalogers publish no ID).
|
||||
func catalogerTaskKey(info monitor.GenericTask) string {
|
||||
if info.ID != "" {
|
||||
return "id:" + info.ID
|
||||
}
|
||||
return "title:" + info.ParentID + "/" + info.Title.Default
|
||||
}
|
||||
|
||||
func StartCatalogerTask(ctx context.Context, info monitor.GenericTask, size int64, initialStage string) *monitor.TaskProgress {
|
||||
reg := catalogerTaskRegistryFromContext(ctx)
|
||||
if reg == nil {
|
||||
return publishCatalogerTask(info, size, initialStage)
|
||||
}
|
||||
|
||||
reg.mu.Lock()
|
||||
defer reg.mu.Unlock()
|
||||
|
||||
key := catalogerTaskKey(info)
|
||||
if row, ok := reg.rows[key]; ok {
|
||||
resumeCatalogerTask(row, size, initialStage)
|
||||
return row
|
||||
}
|
||||
|
||||
t := publishCatalogerTask(info, size, initialStage)
|
||||
reg.rows[key] = t
|
||||
|
||||
return t
|
||||
}
|
||||
|
||||
// resumeCatalogerTask reopens a row an earlier run completed and grows its total by what this run
|
||||
// adds, so the row does not read as finished mid-walk. A row carrying a real failure keeps it.
|
||||
func resumeCatalogerTask(row *monitor.TaskProgress, size int64, initialStage string) {
|
||||
if size < 0 || row.Size() < 0 {
|
||||
row.SetTotal(-1)
|
||||
} else {
|
||||
row.SetTotal(row.Size() + size)
|
||||
}
|
||||
|
||||
if progress.IsErrCompleted(row.Error()) {
|
||||
row.SetError(nil)
|
||||
}
|
||||
|
||||
if initialStage != "" {
|
||||
row.AtomicStage.Set(initialStage)
|
||||
}
|
||||
}
|
||||
|
||||
func publishCatalogerTask(info monitor.GenericTask, size int64, initialStage string) *monitor.TaskProgress {
|
||||
func StartCatalogerTask(info monitor.GenericTask, size int64, initialStage string) *monitor.TaskProgress {
|
||||
t := &monitor.TaskProgress{
|
||||
AtomicStage: progress.NewAtomicStage(initialStage),
|
||||
Manual: progress.NewManual(size),
|
||||
|
||||
@@ -1,152 +0,0 @@
|
||||
package bus
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
"github.com/wagoodman/go-progress"
|
||||
|
||||
"github.com/anchore/syft/syft/event/monitor"
|
||||
)
|
||||
|
||||
func digestsTask() monitor.GenericTask {
|
||||
return monitor.GenericTask{
|
||||
Title: monitor.Title{Default: "File digests"},
|
||||
ParentID: monitor.TopLevelCatalogingTaskID,
|
||||
}
|
||||
}
|
||||
|
||||
func TestStartCatalogerTask_withoutARegistryEveryRunPublishes(t *testing.T) {
|
||||
// the right answer for a caller driving a single pass: nothing re-runs, so nothing should be reused
|
||||
ctx := context.Background()
|
||||
|
||||
first := StartCatalogerTask(ctx, digestsTask(), 2, "")
|
||||
second := StartCatalogerTask(ctx, digestsTask(), 3, "")
|
||||
|
||||
assert.NotSame(t, first, second)
|
||||
assert.Equal(t, int64(2), first.Size())
|
||||
assert.Equal(t, int64(3), second.Size())
|
||||
}
|
||||
|
||||
func TestStartCatalogerTask_reRunReportsIntoTheRowItAlreadyOwns(t *testing.T) {
|
||||
ctx := WithCatalogerTaskRegistry(context.Background())
|
||||
|
||||
first := StartCatalogerTask(ctx, digestsTask(), 2, "")
|
||||
first.Increment()
|
||||
first.Increment()
|
||||
first.SetCompleted()
|
||||
|
||||
// the archive walk reaches the same cataloger again, over an archive's contents this time
|
||||
second := StartCatalogerTask(ctx, digestsTask(), 3, "")
|
||||
require.Same(t, first, second, "a re-run must find the row it already published")
|
||||
|
||||
assert.Equal(t, int64(5), second.Size(), "the total must cover both runs, not just the first")
|
||||
assert.False(t, progress.IsCompleted(second), "a row with work still to do must not read as complete")
|
||||
|
||||
for i := 0; i < 3; i++ {
|
||||
second.Increment()
|
||||
}
|
||||
second.SetCompleted()
|
||||
|
||||
assert.Equal(t, int64(5), second.Current())
|
||||
assert.Equal(t, int64(5), second.Size(), "the count must not run past the total")
|
||||
assert.True(t, progress.IsCompleted(second))
|
||||
}
|
||||
|
||||
func TestStartCatalogerTask_aTotalResolvedOnCompletionIsAddable(t *testing.T) {
|
||||
// a run that could not say how much it would do says so when it signs off (Manual.SetCompleted
|
||||
// backfills the total from the count), so by the time a re-run arrives there is a real number to
|
||||
// add to. This is the file metadata cataloger's shape: it publishes -1 every run.
|
||||
ctx := WithCatalogerTaskRegistry(context.Background())
|
||||
|
||||
row := StartCatalogerTask(ctx, digestsTask(), -1, "")
|
||||
row.Increment()
|
||||
row.SetCompleted()
|
||||
require.Equal(t, int64(1), row.Size(), "signing off must resolve the total to what the run did")
|
||||
|
||||
require.Same(t, row, StartCatalogerTask(ctx, digestsTask(), -1, ""))
|
||||
assert.Equal(t, int64(-1), row.Size(), "a re-run that cannot say its own size makes the row indeterminate again")
|
||||
|
||||
row.Increment()
|
||||
row.SetCompleted()
|
||||
assert.Equal(t, int64(2), row.Current())
|
||||
assert.Equal(t, int64(2), row.Size(), "and signing off resolves it again, over both runs")
|
||||
}
|
||||
|
||||
func TestStartCatalogerTask_indeterminateTotalsStayIndeterminate(t *testing.T) {
|
||||
// a run whose size is unknown cannot be added to one that is known
|
||||
tests := []struct {
|
||||
name string
|
||||
first, next int64
|
||||
complete func(row *monitor.TaskProgress)
|
||||
}{
|
||||
{
|
||||
name: "the re-run cannot say its own size",
|
||||
first: 2, next: -1,
|
||||
complete: func(row *monitor.TaskProgress) { row.Increment(); row.SetCompleted() },
|
||||
},
|
||||
{
|
||||
name: "the first run did nothing, so completing resolved nothing",
|
||||
first: -1, next: 3,
|
||||
complete: func(row *monitor.TaskProgress) { row.SetCompleted() },
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
ctx := WithCatalogerTaskRegistry(context.Background())
|
||||
|
||||
row := StartCatalogerTask(ctx, digestsTask(), tt.first, "")
|
||||
tt.complete(row)
|
||||
|
||||
require.Same(t, row, StartCatalogerTask(ctx, digestsTask(), tt.next, ""))
|
||||
assert.Equal(t, int64(-1), row.Size())
|
||||
assert.False(t, progress.IsCompleted(row), "an indeterminate row is not complete until it says so")
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestStartCatalogerTask_aFailedRowKeepsItsFailure(t *testing.T) {
|
||||
// a later run finding nothing wrong does not mean the earlier one succeeded
|
||||
ctx := WithCatalogerTaskRegistry(context.Background())
|
||||
failure := errors.New("unable to read the thing")
|
||||
|
||||
row := StartCatalogerTask(ctx, digestsTask(), 2, "")
|
||||
row.SetError(failure)
|
||||
|
||||
require.Same(t, row, StartCatalogerTask(ctx, digestsTask(), 3, ""))
|
||||
assert.ErrorIs(t, row.Error(), failure)
|
||||
}
|
||||
|
||||
func TestStartCatalogerTask_rowsAreKeyedByTaskIdentity(t *testing.T) {
|
||||
ctx := WithCatalogerTaskRegistry(context.Background())
|
||||
|
||||
byID := func(id string) monitor.GenericTask {
|
||||
return monitor.GenericTask{ID: id, Title: monitor.Title{Default: "shared title"}}
|
||||
}
|
||||
|
||||
assert.Same(t, StartCatalogerTask(ctx, byID("a"), -1, ""), StartCatalogerTask(ctx, byID("a"), -1, ""))
|
||||
assert.NotSame(t, StartCatalogerTask(ctx, byID("a"), -1, ""), StartCatalogerTask(ctx, byID("b"), -1, ""),
|
||||
"an ID is the identity where there is one, even when two tasks share a title")
|
||||
|
||||
// a task publishing without an ID, as every file cataloger does, is keyed by where it sits and what
|
||||
// it is called
|
||||
underCataloging := monitor.GenericTask{Title: monitor.Title{Default: "Files"}, ParentID: "cataloging"}
|
||||
underSomethingElse := monitor.GenericTask{Title: monitor.Title{Default: "Files"}, ParentID: "other"}
|
||||
|
||||
assert.Same(t, StartCatalogerTask(ctx, underCataloging, -1, ""), StartCatalogerTask(ctx, underCataloging, -1, ""))
|
||||
assert.NotSame(t, StartCatalogerTask(ctx, underCataloging, -1, ""), StartCatalogerTask(ctx, underSomethingElse, -1, ""))
|
||||
}
|
||||
|
||||
func TestWithCatalogerTaskRegistry_isIdempotent(t *testing.T) {
|
||||
// anything that re-runs catalogers may ask for a registry; a second one would hold none of the rows
|
||||
// already published and every re-run would publish again
|
||||
ctx := WithCatalogerTaskRegistry(context.Background())
|
||||
row := StartCatalogerTask(ctx, digestsTask(), 2, "")
|
||||
|
||||
nested := WithCatalogerTaskRegistry(ctx)
|
||||
assert.Same(t, row, StartCatalogerTask(nested, digestsTask(), 2, ""))
|
||||
}
|
||||
Vendored
-4
@@ -6,7 +6,6 @@ import (
|
||||
"net/url"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"runtime"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
@@ -96,9 +95,6 @@ func Test_makeDiskKey(t *testing.T) {
|
||||
}
|
||||
|
||||
func Test_errors(t *testing.T) {
|
||||
if runtime.GOOS == "windows" {
|
||||
t.Skip("windows ignores unix permission bits on directories")
|
||||
}
|
||||
tmp := t.TempDir()
|
||||
cache := filepath.Join(tmp, "cache")
|
||||
// make a non-writable directory
|
||||
|
||||
@@ -15,9 +15,7 @@ application: # AUTO-GENERATED - application-level config keys
|
||||
- key: dotnet.relax-dll-claims-when-bundling-detected
|
||||
description: show all packages from the deps.json if bundling tooling is present as a dependency (e.g. ILRepack)
|
||||
- key: golang.capture-symbols
|
||||
description: 'capture function symbols from the binary symbol table (pclntab). valid values are: "none" (disabled), "stdlib" (only the synthetic stdlib package), "extended-stdlib" (stdlib plus every module under golang.org/x/), and "all" (all module packages plus stdlib)'
|
||||
- key: golang.capture-symbols-modules
|
||||
description: glob patterns matched against go module paths (e.g. github.com/klauspost/**) that should have symbols captured in addition to whatever capture-symbols selects. ** crosses path separators, * does not. a trailing major version suffix is ignored when matching, so github.com/foo/* covers github.com/foo/bar/v2; spelling a suffix out in the pattern selects only that major version. this can only widen the selection, never narrow it, and is inert when capture-symbols is none
|
||||
description: 'capture function symbols from the binary symbol table (pclntab). valid values are: "none" (disabled), "stdlib" (only the synthetic stdlib package), and "all" (all module packages plus stdlib)'
|
||||
- key: golang.local-mod-cache-dir
|
||||
description: specify an explicit go mod cache directory, if unset this defaults to $GOPATH/pkg/mod or $HOME/go/pkg/mod
|
||||
- key: golang.local-vendor-dir
|
||||
|
||||
@@ -9,7 +9,7 @@ import (
|
||||
"sort"
|
||||
|
||||
"github.com/scylladb/go-set/strset"
|
||||
"go.yaml.in/yaml/v3"
|
||||
"gopkg.in/yaml.v3"
|
||||
|
||||
"github.com/anchore/syft/internal/task"
|
||||
)
|
||||
|
||||
@@ -9,7 +9,6 @@ import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"slices"
|
||||
"strings"
|
||||
|
||||
"github.com/anchore/syft/internal/capabilities/internal"
|
||||
@@ -32,10 +31,6 @@ type ConfigInfo struct {
|
||||
|
||||
var appConfigAnnotationPattern = regexp.MustCompile(`^//\s*app-config:\s*(.+)$`)
|
||||
|
||||
// embeddedFieldsAnnotationPattern matches "// capabilities-fields: A, B" on an embedded struct, which limits
|
||||
// the embedded fields listed to the ones named (for a shared type the cataloger reads only part of).
|
||||
var embeddedFieldsAnnotationPattern = regexp.MustCompile(`^//\s*capabilities-fields:\s*(.+)$`)
|
||||
|
||||
// DiscoverConfigs walks the cataloger directory and discovers all configuration structs
|
||||
// Returns map where key is "packageName.StructName" (e.g., "golang.CatalogerConfig")
|
||||
func DiscoverConfigs(repoRoot string) (map[string]ConfigInfo, error) {
|
||||
@@ -165,9 +160,6 @@ func extractCatalogerConfigFieldsRecursive(structType *ast.StructType, visitedTy
|
||||
if len(field.Names) == 0 {
|
||||
// this is an embedded field - resolve and extract its fields
|
||||
embeddedFields := resolveEmbeddedStructFields(field.Type, visitedTypes)
|
||||
if only := embeddedFieldAllowlist(field.Doc); only != nil {
|
||||
embeddedFields = slices.DeleteFunc(embeddedFields, func(f ConfigField) bool { return !only[f.Name] })
|
||||
}
|
||||
fields = append(fields, embeddedFields...)
|
||||
continue
|
||||
}
|
||||
@@ -203,26 +195,6 @@ func extractCatalogerConfigFieldsRecursive(structType *ast.StructType, visitedTy
|
||||
return fields
|
||||
}
|
||||
|
||||
// embeddedFieldAllowlist returns the field names from a "capabilities-fields:" annotation, or nil when there
|
||||
// is none (every exported embedded field is listed).
|
||||
func embeddedFieldAllowlist(commentGroup *ast.CommentGroup) map[string]bool {
|
||||
if commentGroup == nil {
|
||||
return nil
|
||||
}
|
||||
for _, comment := range commentGroup.List {
|
||||
matches := embeddedFieldsAnnotationPattern.FindStringSubmatch(comment.Text)
|
||||
if len(matches) < 2 {
|
||||
continue
|
||||
}
|
||||
only := make(map[string]bool)
|
||||
for _, name := range strings.Split(matches[1], ",") {
|
||||
only[strings.TrimSpace(name)] = true
|
||||
}
|
||||
return only
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// resolveEmbeddedStructFields resolves an embedded struct type and extracts its fields recursively
|
||||
func resolveEmbeddedStructFields(fieldType ast.Expr, visitedTypes map[string]bool) []ConfigField {
|
||||
// extract the type name from the expression
|
||||
|
||||
@@ -457,32 +457,3 @@ func TestDiscoverAllowedConfigStructs(t *testing.T) {
|
||||
t.Logf(" - %s", configType)
|
||||
}
|
||||
}
|
||||
|
||||
func TestEmbeddedFieldAllowlist(t *testing.T) {
|
||||
comments := func(lines ...string) *ast.CommentGroup {
|
||||
g := &ast.CommentGroup{}
|
||||
for _, l := range lines {
|
||||
g.List = append(g.List, &ast.Comment{Text: l})
|
||||
}
|
||||
return g
|
||||
}
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
doc *ast.CommentGroup
|
||||
want map[string]bool
|
||||
}{
|
||||
{name: "no doc lists every field", doc: nil, want: nil},
|
||||
{name: "no annotation lists every field", doc: comments("// Deprecated: use something else."), want: nil},
|
||||
{
|
||||
name: "annotation limits to the named fields",
|
||||
doc: comments("// Deprecated: use something else.", "// capabilities-fields: IncludeIndexedArchives, IncludeUnindexedArchives"),
|
||||
want: map[string]bool{"IncludeIndexedArchives": true, "IncludeUnindexedArchives": true},
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
require.Equal(t, tt.want, embeddedFieldAllowlist(tt.doc))
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
@@ -212,7 +212,7 @@ func parseGenericCatalogerFunction(funcDecl *ast.FuncDecl, filePath, repoRoot st
|
||||
// extractPackageNameFromPath extracts the package name from a cataloger source file path
|
||||
// e.g., "syft/pkg/cataloger/swift/cataloger.go" -> "swift"
|
||||
func extractPackageNameFromPath(filePath string) string {
|
||||
parts := strings.Split(filepath.ToSlash(filePath), "/")
|
||||
parts := strings.Split(filePath, string(filepath.Separator))
|
||||
|
||||
// find the LAST occurrence of "cataloger" in the path
|
||||
// (to handle test fixtures with multiple "cataloger" segments)
|
||||
|
||||
@@ -4,7 +4,6 @@ import (
|
||||
"go/ast"
|
||||
"go/parser"
|
||||
"go/token"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/require"
|
||||
@@ -338,7 +337,7 @@ func TestResolveImportPath(t *testing.T) {
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got := resolveImportPath(tt.importPath, testRepoRoot)
|
||||
require.Equal(t, filepath.FromSlash(tt.want), got)
|
||||
require.Equal(t, tt.want, got)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
// this file handles YAML file reading and writing with comment preservation, using go.yaml.in/yaml/v3's node tree to maintain all existing comments during regeneration.
|
||||
// this file handles YAML file reading and writing with comment preservation, using gopkg.in/yaml.v3's node tree to maintain all existing comments during regeneration.
|
||||
package main
|
||||
|
||||
import (
|
||||
@@ -6,7 +6,7 @@ import (
|
||||
"os"
|
||||
"regexp"
|
||||
|
||||
"go.yaml.in/yaml/v3"
|
||||
"gopkg.in/yaml.v3"
|
||||
|
||||
"github.com/anchore/syft/internal/capabilities"
|
||||
"github.com/anchore/syft/internal/capabilities/internal"
|
||||
|
||||
@@ -4,7 +4,7 @@ import (
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/require"
|
||||
"go.yaml.in/yaml/v3"
|
||||
"gopkg.in/yaml.v3"
|
||||
)
|
||||
|
||||
func TestFindFieldValue(t *testing.T) {
|
||||
|
||||
@@ -502,19 +502,6 @@ func convertToJSONSchemaTypesFromMetadata(metadataTypes []string) []string {
|
||||
return result
|
||||
}
|
||||
|
||||
// packageTypeConstName renders a package type using its go constant name, which is what the detector
|
||||
// "type" field reports (e.g. "BinaryPkg").
|
||||
// a switch rather than a derivation, since pkg.Type values don't reliably map back to their constant
|
||||
// names (e.g. "java-archive" is JavaPkg); add an arm when a classifier declares a new type.
|
||||
func packageTypeConstName(t pkg.Type) string {
|
||||
switch t {
|
||||
case pkg.CpanPkg:
|
||||
return "CpanPkg"
|
||||
default:
|
||||
return "BinaryPkg"
|
||||
}
|
||||
}
|
||||
|
||||
// EnrichWithBinaryClassifier enriches an entry with binary classifier detectors if it's the binary-classifier-cataloger
|
||||
func (e *EnrichmentData) EnrichWithBinaryClassifier(catalogerName string, entry *capabilities.CatalogerEntry) {
|
||||
// special handling for binary-classifier-cataloger: auto-generate one detector per classifier
|
||||
@@ -536,7 +523,7 @@ func (e *EnrichmentData) EnrichWithBinaryClassifier(catalogerName string, entry
|
||||
Name: classifier.Package,
|
||||
PURL: purlStr,
|
||||
CPEs: cpeStrings,
|
||||
Type: packageTypeConstName(classifier.PackageType()),
|
||||
Type: "BinaryPkg",
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
@@ -5,7 +5,7 @@ import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
|
||||
"go.yaml.in/yaml/v3"
|
||||
"gopkg.in/yaml.v3"
|
||||
|
||||
"github.com/anchore/syft/internal/capabilities"
|
||||
)
|
||||
|
||||
@@ -5,7 +5,7 @@ import (
|
||||
|
||||
"github.com/google/go-cmp/cmp"
|
||||
"github.com/stretchr/testify/require"
|
||||
"go.yaml.in/yaml/v3"
|
||||
"gopkg.in/yaml.v3"
|
||||
)
|
||||
|
||||
func TestConfigSchemaExtensions(t *testing.T) {
|
||||
|
||||
@@ -69,5 +69,5 @@ func locationSorter(a, b file.Location) int {
|
||||
return strings.Compare(a.AccessPath, b.AccessPath)
|
||||
}
|
||||
|
||||
return strings.Compare(a.FullNestedPath(), b.FullNestedPath())
|
||||
return strings.Compare(a.RealPath, b.RealPath)
|
||||
}
|
||||
|
||||
@@ -14,5 +14,5 @@ func DefaultLocationComparer(x, y file.Location) bool {
|
||||
}
|
||||
|
||||
func LocationComparerWithoutLayer(x, y file.Location) bool {
|
||||
return cmp.Equal(x.FullNestedPath(), y.FullNestedPath()) && cmp.Equal(x.AccessPath, y.AccessPath)
|
||||
return cmp.Equal(x.RealPath, y.RealPath) && cmp.Equal(x.AccessPath, y.AccessPath)
|
||||
}
|
||||
|
||||
@@ -3,7 +3,7 @@ package internal
|
||||
const (
|
||||
// JSONSchemaVersion is the current schema version output by the JSON encoder
|
||||
// This is roughly following the "SchemaVer" guidelines for versioning the JSON schema. Please see schema/json/README.md for details on how to increment.
|
||||
JSONSchemaVersion = "16.1.12"
|
||||
JSONSchemaVersion = "16.1.10"
|
||||
|
||||
// Changelog
|
||||
// 16.1.0 - reformulated the python pdm fields (added "URL" and removed the unused "path" field).
|
||||
@@ -17,6 +17,4 @@ const (
|
||||
// 16.1.8 - add VcpkgManifest metadata type for vcpkg manifest support
|
||||
// 16.1.9 - add Symbols (grouped by owning package import path) to GolangBinaryBuildinfoEntry metadata
|
||||
// 16.1.10 - add SafeTensorsModelInfo metadata type for the safetensors AI model cataloger
|
||||
// 16.1.11 - add CpanDistribution and CpanUnpackedRelease metadata types for the perl catalogers
|
||||
// 16.1.12 - add fullPath to file coordinates, naming the nested archive chain a file was found in (blank for files not within an archive)
|
||||
)
|
||||
|
||||
@@ -6,13 +6,13 @@ import (
|
||||
"io"
|
||||
)
|
||||
|
||||
// PerFileReadLimit is the most read from any one archive entry, guarding against decompression bombs.
|
||||
const PerFileReadLimit = 2 * GB
|
||||
const perFileReadLimit = 2 * GB
|
||||
|
||||
// SafeCopy bounds the copy to guard against decompression bombs when extracting from archives.
|
||||
func SafeCopy(writer io.Writer, reader io.Reader) error {
|
||||
numBytes, err := io.Copy(writer, io.LimitReader(reader, PerFileReadLimit))
|
||||
if numBytes >= PerFileReadLimit {
|
||||
// safeCopy limits the copy from the reader. This is useful when extracting files from archives to
|
||||
// protect against decompression bomb attacks.
|
||||
func safeCopy(writer io.Writer, reader io.Reader) error {
|
||||
numBytes, err := io.Copy(writer, io.LimitReader(reader, perFileReadLimit))
|
||||
if numBytes >= perFileReadLimit {
|
||||
return fmt.Errorf("zip read limit hit (potential decompression bomb attack)")
|
||||
}
|
||||
// Propagate decompression / read errors up to the caller. io.Copy
|
||||
|
||||
@@ -29,7 +29,7 @@ func (r *errReader) Read(p []byte) (int, error) {
|
||||
func TestSafeCopy(t *testing.T) {
|
||||
t.Run("clean copy returns nil", func(t *testing.T) {
|
||||
var buf bytes.Buffer
|
||||
if err := SafeCopy(&buf, strings.NewReader("hello")); err != nil {
|
||||
if err := safeCopy(&buf, strings.NewReader("hello")); err != nil {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
if got := buf.String(); got != "hello" {
|
||||
@@ -38,12 +38,12 @@ func TestSafeCopy(t *testing.T) {
|
||||
})
|
||||
|
||||
t.Run("propagates decompression error", func(t *testing.T) {
|
||||
// #4806: SafeCopy used to drop non-EOF errors, so the caller
|
||||
// #4806: safeCopy used to drop non-EOF errors, so the caller
|
||||
// would persist a partial buffer as a successful extract and
|
||||
// downstream catalogers silently read empty manifests.
|
||||
sentinel := errors.New("flate: corrupt input before offset 42")
|
||||
var buf bytes.Buffer
|
||||
err := SafeCopy(&buf, &errReader{data: []byte("partial"), err: sentinel})
|
||||
err := safeCopy(&buf, &errReader{data: []byte("partial"), err: sentinel})
|
||||
if err == nil {
|
||||
t.Fatalf("expected error to be returned, got nil")
|
||||
}
|
||||
@@ -56,7 +56,7 @@ func TestSafeCopy(t *testing.T) {
|
||||
// The old code had a dead io.EOF branch that labelled clean
|
||||
// reads as decompression bombs; keep the happy path clean.
|
||||
var buf bytes.Buffer
|
||||
err := SafeCopy(&buf, io.LimitReader(strings.NewReader("abc"), 3))
|
||||
err := safeCopy(&buf, io.LimitReader(strings.NewReader("abc"), 3))
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error on clean copy: %v", err)
|
||||
}
|
||||
|
||||
@@ -1,66 +0,0 @@
|
||||
package file
|
||||
|
||||
import "io"
|
||||
|
||||
// ReaderSize reports the number of bytes behind r. The bool is false when that cannot be determined, and
|
||||
// callers must branch on it rather than treating the zero as a size.
|
||||
//
|
||||
// Binary parsers read sizes out of the files they are parsing and allocate against them. Those sizes are
|
||||
// 32- and 64-bit fields under the control of whoever produced the file, so a truncated or hostile input
|
||||
// can declare gigabytes that are not there. The read would fail afterwards either way; what matters is
|
||||
// refusing before the allocation, and that needs a real byte count to weigh the claim against.
|
||||
//
|
||||
// The bool is the whole point of the signature. An earlier version returned 0 for every failure mode, and
|
||||
// a caller that forgot to special-case it got a bound of "nothing is allowed" or, more often, silently
|
||||
// fell back to an unbounded path. Making the caller name the failure keeps that from happening by
|
||||
// omission.
|
||||
//
|
||||
// Both *bytes.Reader and *io.SectionReader answer directly. A file handle only has Seek, so fall back to
|
||||
// a save-and-restore seek, which leaves the caller's cursor where it found it. That fallback is not
|
||||
// atomic: it moves the cursor and puts it back, so a reader being read concurrently can observe the
|
||||
// intermediate position.
|
||||
//
|
||||
// Neither answer is taken on faith, because a reader can report a length it cannot deliver: an
|
||||
// *io.SectionReader answers with the length it was constructed with, and debug/elf and friends construct
|
||||
// theirs with a nominal one, so io.NewSectionReader(r, 0, 1<<63-1) would otherwise report a bound that
|
||||
// permits everything as if it had been measured. The last byte is read back to confirm the count, which is
|
||||
// one ReadAt whatever the size, and the read is what the caller was going to weigh anyway.
|
||||
//
|
||||
// A wrapper that embeds io.ReaderAt as an interface promotes only ReadAt, so it answers no size here
|
||||
// however sizable the reader underneath it is. A type meant to be bounded needs to forward Size() itself.
|
||||
func ReaderSize(r io.ReaderAt) (int64, bool) {
|
||||
size, ok := reportedSize(r)
|
||||
if !ok {
|
||||
return 0, false
|
||||
}
|
||||
// ReadAt may return io.EOF alongside a full read, so the count is what says the byte is there
|
||||
var last [1]byte
|
||||
if n, _ := r.ReadAt(last[:], size-1); n != len(last) {
|
||||
return 0, false
|
||||
}
|
||||
return size, true
|
||||
}
|
||||
|
||||
// reportedSize is what the reader says about itself, before ReaderSize checks whether it can back it up.
|
||||
func reportedSize(r io.ReaderAt) (int64, bool) {
|
||||
if sr, ok := r.(interface{ Size() int64 }); ok {
|
||||
size := sr.Size()
|
||||
return size, size > 0
|
||||
}
|
||||
s, ok := r.(io.Seeker)
|
||||
if !ok {
|
||||
return 0, false
|
||||
}
|
||||
cur, err := s.Seek(0, io.SeekCurrent)
|
||||
if err != nil {
|
||||
return 0, false
|
||||
}
|
||||
size, err := s.Seek(0, io.SeekEnd)
|
||||
if err != nil {
|
||||
return 0, false
|
||||
}
|
||||
if _, err := s.Seek(cur, io.SeekStart); err != nil {
|
||||
return 0, false
|
||||
}
|
||||
return size, size > 0
|
||||
}
|
||||
@@ -1,140 +0,0 @@
|
||||
package file
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestReaderSize(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
r io.ReaderAt
|
||||
want int64
|
||||
wantOK bool
|
||||
}{
|
||||
{
|
||||
name: "bytes.Reader answers directly",
|
||||
r: bytes.NewReader(make([]byte, 1234)),
|
||||
want: 1234,
|
||||
wantOK: true,
|
||||
},
|
||||
{
|
||||
// an empty reader is not a size to bound against, so it reports not-ok along with everything
|
||||
// else that cannot be measured
|
||||
name: "empty bytes.Reader",
|
||||
r: bytes.NewReader(nil),
|
||||
want: 0,
|
||||
},
|
||||
{
|
||||
name: "SectionReader over a real length",
|
||||
r: io.NewSectionReader(bytes.NewReader(make([]byte, 500)), 100, 300),
|
||||
want: 300,
|
||||
wantOK: true,
|
||||
},
|
||||
{
|
||||
name: "seek-only reader falls back to seeking",
|
||||
r: seekOnly{bytes.NewReader(make([]byte, 4096))},
|
||||
want: 4096,
|
||||
wantOK: true,
|
||||
},
|
||||
{
|
||||
name: "reader with neither Size nor Seek cannot be measured",
|
||||
r: readAtOnly{bytes.NewReader(make([]byte, 4096))},
|
||||
want: 0,
|
||||
},
|
||||
{
|
||||
// the shape debug/elf, debug/pe and debug/macho build internally. Taking this at its word hands
|
||||
// back a bound that permits everything, reported as if it had been measured.
|
||||
name: "SectionReader over a nominal length reports what it can deliver",
|
||||
r: io.NewSectionReader(bytes.NewReader(make([]byte, 500)), 0, 1<<63-1),
|
||||
want: 0,
|
||||
},
|
||||
{
|
||||
// a Size() a type simply got wrong is the same failure: the bytes are what decides
|
||||
name: "a size the reader cannot back up is refused",
|
||||
r: overstatedSize{ReaderAt: bytes.NewReader(make([]byte, 10)), size: 1 << 30},
|
||||
want: 0,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
size, ok := ReaderSize(tt.r)
|
||||
assert.Equal(t, tt.want, size)
|
||||
assert.Equal(t, tt.wantOK, ok)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestReaderSize_FileHandle(t *testing.T) {
|
||||
path := filepath.Join(t.TempDir(), "f.bin")
|
||||
require.NoError(t, os.WriteFile(path, make([]byte, 7777), 0o600))
|
||||
f, err := os.Open(path)
|
||||
require.NoError(t, err)
|
||||
defer f.Close()
|
||||
|
||||
// the cursor has to come back where it was found: the caller may be mid-read, and an *os.File is the
|
||||
// one input here that carries a shared position
|
||||
_, err = f.Seek(101, io.SeekStart)
|
||||
require.NoError(t, err)
|
||||
|
||||
size, ok := ReaderSize(f)
|
||||
assert.True(t, ok)
|
||||
assert.Equal(t, int64(7777), size)
|
||||
|
||||
at, err := f.Seek(0, io.SeekCurrent)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, int64(101), at, "measuring the file must not move the caller's cursor")
|
||||
}
|
||||
|
||||
// TestReaderSize_WrapperMustForwardSize pins the shape that made every bound expressed against this
|
||||
// helper go silently inert. A type that embeds io.ReaderAt as an interface promotes only ReadAt, so it
|
||||
// answers 0 here however sizable the reader underneath is, and a caller that treats 0 as "fall back to
|
||||
// unbounded" then stops bounding anything. Forwarding Size() is what fixes it, and this is the assertion
|
||||
// that a new wrapper has to satisfy.
|
||||
func TestReaderSize_WrapperMustForwardSize(t *testing.T) {
|
||||
inner := bytes.NewReader(make([]byte, 2048))
|
||||
|
||||
size, ok := ReaderSize(embedsReaderAt{inner})
|
||||
assert.False(t, ok, "a wrapper that only embeds the interface cannot be measured")
|
||||
assert.Equal(t, int64(0), size)
|
||||
|
||||
size, ok = ReaderSize(forwardsSize{inner})
|
||||
assert.True(t, ok, "a wrapper meant to be bounded has to forward Size itself")
|
||||
assert.Equal(t, int64(2048), size)
|
||||
}
|
||||
|
||||
type seekOnly struct{ inner *bytes.Reader }
|
||||
|
||||
func (s seekOnly) ReadAt(p []byte, off int64) (int, error) { return s.inner.ReadAt(p, off) }
|
||||
func (s seekOnly) Seek(off int64, whence int) (int64, error) {
|
||||
return s.inner.Seek(off, whence)
|
||||
}
|
||||
|
||||
type readAtOnly struct{ inner *bytes.Reader }
|
||||
|
||||
func (r readAtOnly) ReadAt(p []byte, off int64) (int, error) { return r.inner.ReadAt(p, off) }
|
||||
|
||||
type embedsReaderAt struct{ io.ReaderAt }
|
||||
|
||||
type forwardsSize struct{ io.ReaderAt }
|
||||
|
||||
func (f forwardsSize) Size() int64 {
|
||||
size, _ := ReaderSize(f.ReaderAt)
|
||||
return size
|
||||
}
|
||||
|
||||
// overstatedSize answers with a size larger than the bytes behind it, the way a wrapper that returns a
|
||||
// declared length rather than a measured one would.
|
||||
type overstatedSize struct {
|
||||
io.ReaderAt
|
||||
size int64
|
||||
}
|
||||
|
||||
func (o overstatedSize) Size() int64 { return o.size }
|
||||
@@ -4,7 +4,7 @@ import (
|
||||
"errors"
|
||||
"io/fs"
|
||||
"os"
|
||||
"path"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
|
||||
"github.com/diskfs/go-diskfs/filesystem"
|
||||
@@ -25,8 +25,7 @@ func WalkDiskDir(fsys filesystem.FileSystem, root string, fn WalkDiskDirFunc) er
|
||||
}
|
||||
|
||||
for _, info := range infos {
|
||||
// squashfs paths are posix regardless of host os
|
||||
p := path.Join(root, info.Name())
|
||||
p := filepath.Join(root, info.Name())
|
||||
err = walkDiskDir(fsys, p, info, fn)
|
||||
if err != nil {
|
||||
if errors.Is(err, fs.SkipDir) {
|
||||
@@ -75,7 +74,7 @@ func walkDiskDir(fsys filesystem.FileSystem, name string, d os.FileInfo, walkDir
|
||||
}
|
||||
|
||||
for _, d1 := range dirs {
|
||||
name1 := path.Join(name, d1.Name())
|
||||
name1 := filepath.Join(name, d1.Name())
|
||||
if err := walkDiskDir(fsys, name1, d1, walkDirFn); err != nil {
|
||||
if errors.Is(err, fs.SkipDir) {
|
||||
break
|
||||
|
||||
@@ -19,8 +19,6 @@ func createTestFS(t *testing.T) filesystem.FileSystem {
|
||||
filename := "test.squashfs"
|
||||
f, err := os.Create(filepath.Join(dir, filename))
|
||||
require.NoError(t, err)
|
||||
// windows can't remove the temp dir while the backing file is open
|
||||
t.Cleanup(func() { _ = f.Close() })
|
||||
|
||||
b := file.New(f, false)
|
||||
fsys, err := squashfs.Create(b, 0, 0, 4096)
|
||||
|
||||
@@ -70,7 +70,7 @@ func ExtractGlobsFromTarToUniqueTempFile(ctx context.Context, archivePath, dir s
|
||||
}
|
||||
defer internal.CloseAndLogError(packedFile, archivePath)
|
||||
|
||||
if err := SafeCopy(tempFile, packedFile); err != nil {
|
||||
if err := safeCopy(tempFile, packedFile); err != nil {
|
||||
return fmt.Errorf("unable to copy source=%q for tar=%q: %w", file.Name(), archivePath, err)
|
||||
}
|
||||
|
||||
@@ -84,8 +84,7 @@ func ExtractGlobsFromTarToUniqueTempFile(ctx context.Context, archivePath, dir s
|
||||
|
||||
func matchesAnyGlob(name string, globs ...string) bool {
|
||||
for _, glob := range globs {
|
||||
// tar entry names are always posix, so don't match with the host separator (PathMatch)
|
||||
if matches, err := doublestar.Match(glob, name); err == nil && matches {
|
||||
if matches, err := doublestar.PathMatch(glob, name); err == nil && matches {
|
||||
return true
|
||||
}
|
||||
}
|
||||
|
||||
@@ -101,7 +101,7 @@ func ExtractFromZipToUniqueTempFile(ctx context.Context, archivePath, dir string
|
||||
return fmt.Errorf("unable to extract directories, only files: %s", file.NameInArchive)
|
||||
}
|
||||
|
||||
if err := SafeCopy(tempFile, zippedFile); err != nil {
|
||||
if err := safeCopy(tempFile, zippedFile); err != nil {
|
||||
return fmt.Errorf("unable to copy source=%q for zip=%q: %w", file.NameInArchive, archivePath, err)
|
||||
}
|
||||
|
||||
@@ -138,7 +138,7 @@ func ContentsFromZip(ctx context.Context, archivePath string, paths ...string) (
|
||||
}
|
||||
|
||||
var buffer bytes.Buffer
|
||||
if err := SafeCopy(&buffer, zippedFile); err != nil {
|
||||
if err := safeCopy(&buffer, zippedFile); err != nil {
|
||||
return fmt.Errorf("unable to copy source=%q for zip=%q: %w", file.NameInArchive, archivePath, err)
|
||||
}
|
||||
|
||||
@@ -210,7 +210,7 @@ func extractSingleFile(file archives.FileInfo, expandedFilePath, archivePath str
|
||||
}
|
||||
}()
|
||||
|
||||
if err := SafeCopy(outputFile, zippedFile); err != nil {
|
||||
if err := safeCopy(outputFile, zippedFile); err != nil {
|
||||
return fmt.Errorf("unable to copy source=%q to dest=%q for zip=%q: %w", file.NameInArchive, outputFile.Name(), archivePath, err)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,19 +0,0 @@
|
||||
package internal
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"io"
|
||||
)
|
||||
|
||||
// maxScannedLineSize is the longest line NewLineScanner will return before Scan stops with bufio.ErrTooLong.
|
||||
// The buffer only grows to this size when a line needs it.
|
||||
const maxScannedLineSize = 1024 * 1024
|
||||
|
||||
// NewLineScanner returns a bufio.Scanner that accepts lines up to 1MB rather than the 64KB default, since
|
||||
// package metadata files carry long single-line fields (descriptions, license text, dependency lists). Callers
|
||||
// must still check Err() after scanning.
|
||||
func NewLineScanner(r io.Reader) *bufio.Scanner {
|
||||
s := bufio.NewScanner(r)
|
||||
s.Buffer(nil, maxScannedLineSize)
|
||||
return s
|
||||
}
|
||||
@@ -1,24 +0,0 @@
|
||||
package internal
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestNewLineScanner(t *testing.T) {
|
||||
long := strings.Repeat("a", 100*1024)
|
||||
s := NewLineScanner(strings.NewReader(long + "\nb\n"))
|
||||
require.True(t, s.Scan())
|
||||
require.Equal(t, long, s.Text())
|
||||
require.True(t, s.Scan())
|
||||
require.Equal(t, "b", s.Text())
|
||||
require.False(t, s.Scan())
|
||||
require.NoError(t, s.Err())
|
||||
|
||||
s = NewLineScanner(strings.NewReader(strings.Repeat("a", maxScannedLineSize+1)))
|
||||
require.False(t, s.Scan())
|
||||
require.ErrorIs(t, s.Err(), bufio.ErrTooLong)
|
||||
}
|
||||
@@ -6,7 +6,6 @@ import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"regexp"
|
||||
"slices"
|
||||
|
||||
"github.com/anchore/syft/internal"
|
||||
"github.com/anchore/syft/internal/log"
|
||||
@@ -49,7 +48,7 @@ func findRhelFeatures(resolver file.Resolver, release *linux.Release) error {
|
||||
for _, contentManifestFile := range contentManifestFiles {
|
||||
found, err := fileMatchesAny(resolver, contentManifestFile, rhelEUSPatterns...)
|
||||
if err != nil {
|
||||
return fmt.Errorf("unable to read content manifest from %s: %w", contentManifestFile.FullNestedPath(), err)
|
||||
return fmt.Errorf("unable to read content manifest from %s: %w", contentManifestFile.RealPath, err)
|
||||
}
|
||||
if found {
|
||||
release.ExtendedSupport = true
|
||||
@@ -163,9 +162,9 @@ func hasActiveUbuntuESMService(resolver file.Resolver) (bool, error) {
|
||||
func hasEnabledESMServiceInStatus(resolver file.Resolver, location file.Location) (bool, error) {
|
||||
contents, err := resolver.FileContentsByLocation(location)
|
||||
if err != nil {
|
||||
return false, fmt.Errorf("unable to read ubuntu-advantage status from %s: %w", location.FullNestedPath(), err)
|
||||
return false, fmt.Errorf("unable to read ubuntu-advantage status from %s: %w", location.RealPath, err)
|
||||
}
|
||||
defer internal.CloseAndLogError(contents, location.FullNestedPath())
|
||||
defer internal.CloseAndLogError(contents, location.RealPath)
|
||||
|
||||
var status struct {
|
||||
Services []struct {
|
||||
@@ -174,7 +173,7 @@ func hasEnabledESMServiceInStatus(resolver file.Resolver, location file.Location
|
||||
} `json:"services"`
|
||||
}
|
||||
if err := json.NewDecoder(contents).Decode(&status); err != nil {
|
||||
return false, fmt.Errorf("unable to parse ubuntu-advantage status from %s: %w", location.FullNestedPath(), err)
|
||||
return false, fmt.Errorf("unable to parse ubuntu-advantage status from %s: %w", location.RealPath, err)
|
||||
}
|
||||
|
||||
// pro status values are enabled/disabled/n-a/warning/—; only "enabled" proves the stream is active.
|
||||
@@ -192,16 +191,19 @@ func hasInstalledESMPackage(packages *pkg.Collection) bool {
|
||||
if packages == nil {
|
||||
return false
|
||||
}
|
||||
return slices.ContainsFunc(packages.Sorted(pkg.DebPkg), func(p pkg.Package) bool {
|
||||
return esmVersionPattern.MatchString(p.Version)
|
||||
})
|
||||
for p := range packages.Enumerate(pkg.DebPkg) {
|
||||
if esmVersionPattern.MatchString(p.Version) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func fileMatchesAny(resolver file.Resolver, location file.Location, patterns ...*regexp.Regexp) (bool, error) {
|
||||
contents, err := resolver.FileContentsByLocation(location)
|
||||
if err != nil {
|
||||
return false, fmt.Errorf("unable to read %s: %w", location.FullNestedPath(), err)
|
||||
return false, fmt.Errorf("unable to read %s: %w", location.RealPath, err)
|
||||
}
|
||||
defer internal.CloseAndLogError(contents, location.FullNestedPath())
|
||||
defer internal.CloseAndLogError(contents, location.RealPath)
|
||||
return internal.MatchAnyFromReader(contents, patterns...)
|
||||
}
|
||||
|
||||
@@ -8,7 +8,6 @@ import (
|
||||
"path/filepath"
|
||||
"slices"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
@@ -16,7 +15,6 @@ import (
|
||||
"github.com/anchore/syft/internal/os"
|
||||
"github.com/anchore/syft/internal/sbomsync"
|
||||
"github.com/anchore/syft/internal/task"
|
||||
"github.com/anchore/syft/internal/testutils"
|
||||
"github.com/anchore/syft/syft"
|
||||
"github.com/anchore/syft/syft/file"
|
||||
"github.com/anchore/syft/syft/linux"
|
||||
@@ -85,7 +83,6 @@ func Test_EnvironmentTask(t *testing.T) {
|
||||
|
||||
for _, test := range tests {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
testutils.SkipWithoutLinuxContainers(t)
|
||||
tarPath := imagetest.GetFixtureImageTarPath(t, test.name)
|
||||
|
||||
// get the source
|
||||
@@ -235,33 +232,6 @@ func Test_findUbuntuFeatures(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// returning early from Collection.Enumerate leaks its goroutine and read lock, so any later write blocks forever.
|
||||
func Test_findUbuntuFeatures_releasesPackageCollectionLock(t *testing.T) {
|
||||
resolver := fixtureResolverForDir(t, "testdata/ubuntu_plain")
|
||||
|
||||
s := sbom.SBOM{}
|
||||
s.Artifacts.LinuxDistribution = &linux.Release{ID: "ubuntu"}
|
||||
s.Artifacts.Packages = pkg.NewCollection(
|
||||
pkg.Package{Name: "openssl", Version: "1.1.1f-1ubuntu2.19+esm1", Type: pkg.DebPkg},
|
||||
pkg.Package{Name: "libcap2", Version: "1:2.32-1ubuntu0.1~esm1", Type: pkg.DebPkg},
|
||||
)
|
||||
|
||||
require.NoError(t, os.DetectFeatures(context.Background(), resolver, sbomsync.NewBuilder(&s)))
|
||||
require.True(t, s.Artifacts.LinuxDistribution.ExtendedSupport)
|
||||
|
||||
added := make(chan struct{})
|
||||
go func() {
|
||||
s.Artifacts.Packages.Add(pkg.Package{Name: "zlib1g", Version: "1:1.2.11.dfsg-2ubuntu1", Type: pkg.DebPkg})
|
||||
close(added)
|
||||
}()
|
||||
|
||||
select {
|
||||
case <-added:
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatal("package collection is still read-locked after feature detection")
|
||||
}
|
||||
}
|
||||
|
||||
// a read error in the apt-source signal must not suppress the ubuntu-advantage status signal; the three
|
||||
// signals are meant to be independent fallbacks.
|
||||
func Test_findUbuntuFeatures_signalErrorDoesNotSuppressOthers(t *testing.T) {
|
||||
|
||||
@@ -19,8 +19,6 @@ func AllTypes() []any {
|
||||
pkg.ConanfileEntry{},
|
||||
pkg.ConaninfoEntry{},
|
||||
pkg.CondaMetaPackage{},
|
||||
pkg.CpanDistribution{},
|
||||
pkg.CpanUnpackedRelease{},
|
||||
pkg.DartPubspec{},
|
||||
pkg.DartPubspecLockEntry{},
|
||||
pkg.DenoLockEntry{},
|
||||
|
||||
@@ -71,8 +71,6 @@ var jsonTypes = makeJSONTypes(
|
||||
jsonNames(pkg.ConanV2LockEntry{}, "c-conan-lock-v2-entry"),
|
||||
jsonNames(pkg.ConanfileEntry{}, "c-conan-file-entry", "ConanMetadataType"),
|
||||
jsonNames(pkg.ConaninfoEntry{}, "c-conan-info-entry"),
|
||||
jsonNames(pkg.CpanDistribution{}, "cpan-distribution"),
|
||||
jsonNames(pkg.CpanUnpackedRelease{}, "cpan-unpacked-release"),
|
||||
jsonNames(pkg.DartPubspecLockEntry{}, "dart-pubspec-lock-entry", "DartPubMetadata"),
|
||||
jsonNames(pkg.DartPubspec{}, "dart-pubspec"),
|
||||
jsonNames(pkg.DenoLockEntry{}, "deno-lock-entry"),
|
||||
|
||||
@@ -1,74 +0,0 @@
|
||||
package packagemetadata
|
||||
|
||||
import (
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// untypedFieldAllowlist holds package metadata fields that are allowed to carry untyped (interface) values.
|
||||
// Values in these fields come straight from whatever a parser decoded, so they are not shaped by syft
|
||||
// and can hold anything an encoder or hasher chokes on. Every entry is part of existing SBOM output and
|
||||
// cannot be retyped without breaking it; new entries need a reason and a review.
|
||||
var untypedFieldAllowlist = map[string]string{
|
||||
"GGUFFileHeader.RemainingKeyValues": "raw GGUF header key-values; retyping changes the header output and metadataHash",
|
||||
}
|
||||
|
||||
// TestNoUntypedMetadataFields fails when a package metadata type gains a field typed any, []any,
|
||||
// map[string]any (or any other interface) that is not on the allowlist.
|
||||
func TestNoUntypedMetadataFields(t *testing.T) {
|
||||
var hits []string
|
||||
for _, ty := range AllTypes() {
|
||||
rt := reflect.TypeOf(ty)
|
||||
collectUntyped(rt, rt.Name(), map[reflect.Type]bool{}, &hits)
|
||||
}
|
||||
|
||||
seen := map[string]bool{}
|
||||
for _, h := range hits {
|
||||
seen[h] = true
|
||||
if _, ok := untypedFieldAllowlist[h]; !ok {
|
||||
t.Errorf("untyped field %s: use a concrete type, or add it to untypedFieldAllowlist with a reason", h)
|
||||
}
|
||||
}
|
||||
for k := range untypedFieldAllowlist {
|
||||
if !seen[k] {
|
||||
t.Errorf("stale untypedFieldAllowlist entry %s: field no longer exists or is now typed", k)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// collectUntyped records "Struct.Field" for every field whose type reaches an interface, descending
|
||||
// through pointers, slices, arrays, maps and nested syft-owned structs.
|
||||
func collectUntyped(rt reflect.Type, path string, visited map[reflect.Type]bool, hits *[]string) {
|
||||
switch rt.Kind() {
|
||||
case reflect.Interface:
|
||||
*hits = append(*hits, path)
|
||||
case reflect.Pointer, reflect.Slice, reflect.Array:
|
||||
collectUntyped(rt.Elem(), path, visited, hits)
|
||||
case reflect.Map:
|
||||
collectUntyped(rt.Key(), path, visited, hits)
|
||||
collectUntyped(rt.Elem(), path, visited, hits)
|
||||
case reflect.Struct:
|
||||
if visited[rt] {
|
||||
return
|
||||
}
|
||||
if rt.Name() != "" && !strings.HasPrefix(rt.PkgPath(), "github.com/anchore/syft/") {
|
||||
// a third-party type in the model carries whatever shape the library gives it (the same leak as an
|
||||
// interface field), so it counts as untyped. The standard library is left alone.
|
||||
if strings.Contains(strings.Split(rt.PkgPath(), "/")[0], ".") {
|
||||
*hits = append(*hits, path)
|
||||
}
|
||||
return
|
||||
}
|
||||
visited[rt] = true
|
||||
for i := 0; i < rt.NumField(); i++ {
|
||||
f := rt.Field(i)
|
||||
// unexported embedded structs still promote their exported fields to the encoders and the hasher. A
|
||||
// field is only out of reach when it is neither encoded nor hashed by SetID.
|
||||
if (!f.IsExported() && !f.Anonymous) || (f.Tag.Get("json") == "-" && f.Tag.Get("hash") == "ignore") {
|
||||
continue
|
||||
}
|
||||
collectUntyped(f.Type, rt.Name()+"."+f.Name, visited, hits)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -13,14 +13,6 @@ func Set(s redact.Store) {
|
||||
store = s
|
||||
}
|
||||
|
||||
// Reset clears the global redaction store. It must only be called once a command
|
||||
// run has fully finished (nothing left to Add or Apply), so that a later run can
|
||||
// Set its own store. Calling it while a run is in flight would let a second Set
|
||||
// split secrets across two stores and leak the ones in the dropped store.
|
||||
func Reset() {
|
||||
store = nil
|
||||
}
|
||||
|
||||
func Get() redact.Store {
|
||||
return store
|
||||
}
|
||||
|
||||
@@ -71,7 +71,7 @@ func compareElfBinaryPackages(s *sbom.SBOM) []artifact.ID {
|
||||
pkgsToDelete := make([]artifact.ID, 0)
|
||||
for _, elfPkg := range allElfPackages(s) {
|
||||
for _, loc := range onlyPrimaryEvidenceLocations(elfPkg) {
|
||||
for _, otherPkg := range s.Artifacts.Packages.PackagesByPath(loc.FullNestedPath()) {
|
||||
for _, otherPkg := range s.Artifacts.Packages.PackagesByPath(loc.RealPath) {
|
||||
// we only care about comparing binary packages to each other (not other types)
|
||||
if otherPkg.Type != pkg.BinaryPkg {
|
||||
continue
|
||||
@@ -130,7 +130,7 @@ func populateRelationships(exec file.Executable, parentPkg pkg.Package, resolver
|
||||
|
||||
for _, loc := range libLocations {
|
||||
// are you in our index?
|
||||
realBaseName := path.Base(loc.FullNestedPath())
|
||||
realBaseName := path.Base(loc.RealPath)
|
||||
pkgCollection := index.owningLibraryPackage(realBaseName)
|
||||
if pkgCollection.PackageCount() < 1 {
|
||||
addRelationship(
|
||||
|
||||
@@ -3,7 +3,6 @@ package binary
|
||||
import (
|
||||
"path"
|
||||
|
||||
"github.com/anchore/syft/internal/archive"
|
||||
"github.com/anchore/syft/internal/log"
|
||||
"github.com/anchore/syft/internal/sbomsync"
|
||||
"github.com/anchore/syft/syft/file"
|
||||
@@ -64,7 +63,7 @@ func locationsThatProvideLibraries(accessor sbomsync.Accessor) (map[string]file.
|
||||
continue
|
||||
}
|
||||
|
||||
basename := path.Base(coord.FullNestedPath())
|
||||
basename := path.Base(coord.RealPath)
|
||||
set := libLocationsByBasename[basename]
|
||||
set.Add(coord)
|
||||
allLibLocations.Add(coord)
|
||||
@@ -83,11 +82,6 @@ func packagesWithLibraryOwnership(resolver file.Resolver, accessor sbomsync.Acce
|
||||
var ownedFilePaths []string
|
||||
if p.Type == pkg.BinaryPkg {
|
||||
for _, loc := range p.Locations.ToSlice() {
|
||||
if archive.ContainerPath(loc.Coordinates) != "" {
|
||||
// a path within an archive cannot be resolved from the scan root; the location is the library
|
||||
addLibraryOwner(packagesByLibraryPath, allLibLocations, loc.Coordinates, p)
|
||||
continue
|
||||
}
|
||||
ownedFilePaths = append(ownedFilePaths, loc.Path())
|
||||
}
|
||||
} else {
|
||||
@@ -120,19 +114,18 @@ func populatePackagesByLibraryPath(
|
||||
}
|
||||
|
||||
for _, loc := range ownedLocation {
|
||||
addLibraryOwner(packagesByLibraryPath, allLibLocations, loc.Coordinates, p)
|
||||
// if the location is a library, add the package to the set of packages that own the library
|
||||
if !allLibLocations.Contains(loc.Coordinates) {
|
||||
continue
|
||||
}
|
||||
|
||||
if _, ok := packagesByLibraryPath[loc.Coordinates]; !ok {
|
||||
packagesByLibraryPath[loc.Coordinates] = pkg.NewCollection()
|
||||
}
|
||||
|
||||
// we have a library path, add the package to the set of packages that own the library
|
||||
packagesByLibraryPath[loc.Coordinates].Add(p)
|
||||
}
|
||||
}
|
||||
return packagesByLibraryPath
|
||||
}
|
||||
|
||||
// addLibraryOwner records p as an owner of the file at coordinates when that file is a library.
|
||||
func addLibraryOwner(packagesByLibraryPath map[file.Coordinates]*pkg.Collection, allLibLocations file.CoordinateSet, coordinates file.Coordinates, p pkg.Package) {
|
||||
if !allLibLocations.Contains(coordinates) {
|
||||
return
|
||||
}
|
||||
if _, ok := packagesByLibraryPath[coordinates]; !ok {
|
||||
packagesByLibraryPath[coordinates] = pkg.NewCollection()
|
||||
}
|
||||
packagesByLibraryPath[coordinates].Add(p)
|
||||
}
|
||||
|
||||
@@ -6,7 +6,6 @@ import (
|
||||
"github.com/bmatcuk/doublestar/v4"
|
||||
"github.com/scylladb/go-set/strset"
|
||||
|
||||
"github.com/anchore/syft/internal/archive"
|
||||
"github.com/anchore/syft/internal/log"
|
||||
"github.com/anchore/syft/internal/sbomsync"
|
||||
"github.com/anchore/syft/syft/artifact"
|
||||
@@ -101,7 +100,7 @@ func findOwnershipByFilesRelationships(resolver file.Resolver, catalog *pkg.Coll
|
||||
// Now establish relationships, considering symlinks
|
||||
for _, candidateOwnerPkg := range catalog.Sorted() {
|
||||
id := candidateOwnerPkg.ID()
|
||||
if candidateOwnerPkg.Metadata == nil || withinArchive(candidateOwnerPkg) {
|
||||
if candidateOwnerPkg.Metadata == nil {
|
||||
continue
|
||||
}
|
||||
|
||||
@@ -165,7 +164,7 @@ func directOwnersByPath(catalog *pkg.Collection) map[pkg.Type]*strset.Set {
|
||||
|
||||
// First, identify direct ownership of all files
|
||||
for _, p := range catalog.Sorted() {
|
||||
if p.Metadata == nil || withinArchive(p) {
|
||||
if p.Metadata == nil {
|
||||
continue
|
||||
}
|
||||
|
||||
@@ -193,21 +192,6 @@ func directOwnersByPath(catalog *pkg.Collection) map[pkg.Type]*strset.Set {
|
||||
return directOwnership
|
||||
}
|
||||
|
||||
// withinArchive reports whether every location of p is inside an archive. Such a package's owned files
|
||||
// are paths within its archive, not the scan root, so it is not treated as a file owner.
|
||||
func withinArchive(p pkg.Package) bool {
|
||||
locations := p.Locations.ToSlice()
|
||||
if len(locations) == 0 {
|
||||
return false
|
||||
}
|
||||
for _, l := range locations {
|
||||
if archive.ContainerPath(l.Coordinates) == "" {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func resolvePaths(ownedFilePath string, resolver file.Resolver) []string {
|
||||
// though we have a string path, we need to resolve symlinks and other filesystem oddities since we cannot assume this is a real path
|
||||
var locs []file.Location
|
||||
@@ -222,8 +206,6 @@ func resolvePaths(ownedFilePath string, resolver file.Resolver) []string {
|
||||
|
||||
ownedFilePaths := strset.New(ownedFilePath)
|
||||
for _, loc := range locs {
|
||||
// Owned file paths are currently based on the top-level resolver, which is always the RealPath and includes
|
||||
// all files found within archives at that location
|
||||
ownedFilePaths.Add(loc.RealPath)
|
||||
}
|
||||
return ownedFilePaths.List()
|
||||
|
||||
@@ -78,56 +78,6 @@ func TestOwnershipByFilesRelationship(t *testing.T) {
|
||||
return []pkg.Package{parent, child}, []artifact.Relationship{relationship}
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "package-within-archive-is-not-a-file-owner",
|
||||
setup: func(t testing.TB) ([]pkg.Package, []artifact.Relationship) {
|
||||
nestedOwner := pkg.Package{
|
||||
Locations: file.NewLocationSet(file.NewLocationFromCoordinates(file.Coordinates{
|
||||
RealPath: "/rootfs.tar",
|
||||
FullPath: "/rootfs.tar!/var/lib/rpm/Packages",
|
||||
})),
|
||||
Type: pkg.RpmPkg,
|
||||
Metadata: pkg.RpmDBEntry{
|
||||
Files: []pkg.RpmFileRecord{{Path: "/d/path"}},
|
||||
},
|
||||
}
|
||||
nestedOwner.SetID()
|
||||
|
||||
child := pkg.Package{
|
||||
Locations: file.NewLocationSet(file.NewLocation("/d/path")),
|
||||
Type: pkg.BinaryPkg,
|
||||
}
|
||||
child.SetID()
|
||||
|
||||
return []pkg.Package{nestedOwner, child}, nil
|
||||
},
|
||||
},
|
||||
{
|
||||
// the archive's own package (when it has one) is found by its path; what is inside it is
|
||||
// linked to that package by CONTAINS, not owned by whoever owns the archive file
|
||||
name: "top-level-owner-of-an-archive-does-not-own-packages-within-it",
|
||||
setup: func(t testing.TB) ([]pkg.Package, []artifact.Relationship) {
|
||||
parent := pkg.Package{
|
||||
Locations: file.NewLocationSet(file.NewLocation("/var/lib/rpm/Packages")),
|
||||
Type: pkg.RpmPkg,
|
||||
Metadata: pkg.RpmDBEntry{
|
||||
Files: []pkg.RpmFileRecord{{Path: "/opt/bundle.tar"}},
|
||||
},
|
||||
}
|
||||
parent.SetID()
|
||||
|
||||
child := pkg.Package{
|
||||
Locations: file.NewLocationSet(file.NewLocationFromCoordinates(file.Coordinates{
|
||||
RealPath: "/opt/bundle.tar",
|
||||
FullPath: "/opt/bundle.tar!/lib/thing.jar",
|
||||
})),
|
||||
Type: pkg.JavaPkg,
|
||||
}
|
||||
child.SetID()
|
||||
|
||||
return []pkg.Package{parent, child}, nil
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "only-real-file-owner-gets-ownership-relationship-not-symlink-owner",
|
||||
resolver: mockFR{
|
||||
@@ -329,46 +279,6 @@ func TestOwnershipByFilesRelationship(t *testing.T) {
|
||||
return []pkg.Package{parent, child}, nil
|
||||
},
|
||||
},
|
||||
{
|
||||
// a CPAN distribution owns whatever its .packlist recorded, which is how a distribution that
|
||||
// installed a binary or a shared object is related to the package cataloged from it
|
||||
name: "cpan-distribution-owns-packlist-paths",
|
||||
setup: func(t testing.TB) ([]pkg.Package, []artifact.Relationship) {
|
||||
parent := pkg.Package{
|
||||
Locations: file.NewLocationSet(
|
||||
file.NewLocation("/usr/local/lib/perl5/site_perl/5.40.4/x86_64-linux/auto/Net/SSLeay/.packlist"),
|
||||
),
|
||||
Type: pkg.CpanPkg,
|
||||
Metadata: pkg.CpanDistribution{
|
||||
MainModule: "Net::SSLeay",
|
||||
Files: []string{
|
||||
"/usr/local/lib/perl5/site_perl/5.40.4/x86_64-linux/Net/SSLeay.pm",
|
||||
"/usr/local/lib/perl5/site_perl/5.40.4/x86_64-linux/auto/Net/SSLeay/SSLeay.so",
|
||||
},
|
||||
},
|
||||
}
|
||||
parent.SetID()
|
||||
|
||||
child := pkg.Package{
|
||||
Locations: file.NewLocationSet(
|
||||
file.NewLocation("/usr/local/lib/perl5/site_perl/5.40.4/x86_64-linux/auto/Net/SSLeay/SSLeay.so"),
|
||||
),
|
||||
Type: pkg.BinaryPkg,
|
||||
}
|
||||
child.SetID()
|
||||
|
||||
return []pkg.Package{parent, child}, []artifact.Relationship{
|
||||
{
|
||||
From: parent,
|
||||
To: child,
|
||||
Type: artifact.OwnershipByFileOverlapRelationship,
|
||||
Data: ownershipByFilesMetadata{
|
||||
Files: []string{"/usr/local/lib/perl5/site_perl/5.40.4/x86_64-linux/auto/Net/SSLeay/SSLeay.so"},
|
||||
},
|
||||
},
|
||||
}
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
for _, test := range tests {
|
||||
|
||||
@@ -15,9 +15,18 @@ func TestRelationshipsEvidentBy(t *testing.T) {
|
||||
|
||||
c := pkg.NewCollection()
|
||||
|
||||
coordA := file.NewCoordinates("/somewhere/real", "abc")
|
||||
coordC := file.NewCoordinates("/somewhere/real", "abc")
|
||||
coordD := file.NewCoordinates("/somewhere/real", "abc")
|
||||
coordA := file.Coordinates{
|
||||
RealPath: "/somewhere/real",
|
||||
FileSystemID: "abc",
|
||||
}
|
||||
coordC := file.Coordinates{
|
||||
RealPath: "/somewhere/real",
|
||||
FileSystemID: "abc",
|
||||
}
|
||||
coordD := file.Coordinates{
|
||||
RealPath: "/somewhere/real",
|
||||
FileSystemID: "abc",
|
||||
}
|
||||
pkgA := pkg.Package{
|
||||
Locations: file.NewLocationSet(
|
||||
// added!
|
||||
@@ -30,7 +39,10 @@ func TestRelationshipsEvidentBy(t *testing.T) {
|
||||
pkgA.SetID()
|
||||
c.Add(pkgA)
|
||||
|
||||
coordB := file.NewCoordinates("/somewhere-else/real", "def")
|
||||
coordB := file.Coordinates{
|
||||
RealPath: "/somewhere-else/real",
|
||||
FileSystemID: "def",
|
||||
}
|
||||
pkgB := pkg.Package{
|
||||
Locations: file.NewLocationSet(
|
||||
// added!
|
||||
|
||||
@@ -21,8 +21,12 @@ func Test_Index(t *testing.T) {
|
||||
p3 := pkg.Package{
|
||||
Name: "pkg-3",
|
||||
}
|
||||
c1 := file.NewCoordinates("/coords/1", "")
|
||||
c2 := file.NewCoordinates("/coords/2", "")
|
||||
c1 := file.Coordinates{
|
||||
RealPath: "/coords/1",
|
||||
}
|
||||
c2 := file.Coordinates{
|
||||
RealPath: "/coords/2",
|
||||
}
|
||||
|
||||
for _, p := range []*pkg.Package{&p1, &p2, &p3} {
|
||||
p.SetID()
|
||||
@@ -152,10 +156,18 @@ func Test_Coordinates(t *testing.T) {
|
||||
p3 := pkg.Package{
|
||||
Name: "pkg-3",
|
||||
}
|
||||
c1 := file.NewCoordinates("/coords/1", "")
|
||||
c2 := file.NewCoordinates("/coords/2", "")
|
||||
c3 := file.NewCoordinates("/coords/3", "")
|
||||
c4 := file.NewCoordinates("/coords/4", "")
|
||||
c1 := file.Coordinates{
|
||||
RealPath: "/coords/1",
|
||||
}
|
||||
c2 := file.Coordinates{
|
||||
RealPath: "/coords/2",
|
||||
}
|
||||
c3 := file.Coordinates{
|
||||
RealPath: "/coords/3",
|
||||
}
|
||||
c4 := file.Coordinates{
|
||||
RealPath: "/coords/4",
|
||||
}
|
||||
|
||||
for _, p := range []*pkg.Package{&p1, &p2, &p3} {
|
||||
p.SetID()
|
||||
@@ -225,10 +237,10 @@ func TestRemove(t *testing.T) {
|
||||
p1 := pkg.Package{Name: "pkg-1"}
|
||||
p2 := pkg.Package{Name: "pkg-2"}
|
||||
p3 := pkg.Package{Name: "pkg-3"}
|
||||
c1 := file.NewCoordinates("/coords/1", "")
|
||||
c2 := file.NewCoordinates("/coords/2", "")
|
||||
c3 := file.NewCoordinates("/coords/3", "")
|
||||
c4 := file.NewCoordinates("/coords/4", "")
|
||||
c1 := file.Coordinates{RealPath: "/coords/1"}
|
||||
c2 := file.Coordinates{RealPath: "/coords/2"}
|
||||
c3 := file.Coordinates{RealPath: "/coords/3"}
|
||||
c4 := file.Coordinates{RealPath: "/coords/4"}
|
||||
|
||||
for _, p := range []*pkg.Package{&p1, &p2, &p3} {
|
||||
p.SetID()
|
||||
|
||||
@@ -1,7 +0,0 @@
|
||||
package internal
|
||||
|
||||
// MaxSafeTensorsHeaderSize caps a safetensors header's JSON (excluding its
|
||||
// 8-byte length prefix). Shared by the ai cataloger and the OCI model source so
|
||||
// dir and OCI scans accept the same headers. Real-world shard headers are well
|
||||
// under 1 MB.
|
||||
const MaxSafeTensorsHeaderSize = 8 * 1024 * 1024
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user