Compare commits

..
17 Commits
Author SHA1 Message Date
Alex Goodman fdc1effd6e add test and linting fixes
Signed-off-by: Alex Goodman <alex.goodman@anchore.com>
2021-10-10 19:50:50 -07:00
Alex Goodman 055542694f remove encoder notes (have already been incorporated into lib interface)
Signed-off-by: Alex Goodman <alex.goodman@anchore.com>
2021-10-10 19:02:00 -07:00
Alex Goodman c6447cb62b update anchore import to use the new format option
Signed-off-by: Alex Goodman <alex.goodman@anchore.com>
2021-10-10 18:59:46 -07:00
Alex Goodman bd89e22fe5 update the power-user JSON document to use the format model directory
Signed-off-by: Alex Goodman <alex.goodman@anchore.com>
2021-10-10 18:59:13 -07:00
Alex Goodman 3ca797dc53 update existing presenter tests to use the shared format test helpers
Signed-off-by: Alex Goodman <alex.goodman@anchore.com>
2021-10-10 18:58:34 -07:00
Alex Goodman 50408b52d1 update integration test to reference new format options
Signed-off-by: Alex Goodman <alex.goodman@anchore.com>
2021-10-10 18:57:44 -07:00
Alex Goodman c00afc7fa9 add formats helper test fixtures
Signed-off-by: Alex Goodman <alex.goodman@anchore.com>
2021-10-10 18:56:43 -07:00
Alex Goodman 4a82e4843a add encoder tests for spdx-json and syft json
Signed-off-by: Alex Goodman <alex.goodman@anchore.com>
2021-10-10 18:56:16 -07:00
Alex Goodman d474281731 migrate and split common spdx format helpers
Signed-off-by: Alex Goodman <alex.goodman@anchore.com>
2021-10-10 18:40:27 -07:00
Alex Goodman 0c0890ef44 remove spdx json presenter + model
Signed-off-by: Alex Goodman <alex.goodman@anchore.com>
2021-10-10 18:38:28 -07:00
Alex Goodman 8d522efcf6 remove json presenter
Signed-off-by: Alex Goodman <alex.goodman@anchore.com>
2021-10-10 18:36:23 -07:00
Alex Goodman 124cfd86c7 replace presenter.Option with format.Option
Signed-off-by: Alex Goodman <alex.goodman@anchore.com>
2021-10-10 18:35:02 -07:00
Alex Goodman 12b44af471 add encode and decode to the syft lib interface
Signed-off-by: Alex Goodman <alex.goodman@anchore.com>
2021-10-10 18:32:42 -07:00
Alex Goodman bcdebcadaf add internal.formats helper functions + identify tests
Signed-off-by: Alex Goodman <alex.goodman@anchore.com>
2021-10-10 18:32:02 -07:00
Alex Goodman f6cc0c8628 migrate spdxjson presenter to spdxjson format object
Signed-off-by: Alex Goodman <alex.goodman@anchore.com>
2021-10-10 18:30:30 -07:00
Alex Goodman c1346ad62c migrate json presenter to json format object
Signed-off-by: Alex Goodman <alex.goodman@anchore.com>
2021-10-10 18:30:04 -07:00
Alex Goodman 9ded1c4c22 add basic format abstraction
Signed-off-by: Alex Goodman <alex.goodman@anchore.com>
2021-10-10 18:28:40 -07:00
425 changed files with 10811 additions and 34727 deletions
-5
View File
@@ -15,8 +15,3 @@ ignore-packages:
# * GNU General Public License, version 2.0 or later (GPL-2.0-or-later).
# (we choose Apache-2.0)
- github.com/spdx/tools-golang
# from: https://github.com/xi2/xz/blob/master/LICENSE
# All these files have been put into the public domain.
# You can do whatever you want with these files.
- github.com/xi2/xz
-398
View File
@@ -1,398 +0,0 @@
#!/bin/sh
set -e
# Code generated by godownloader on 2019-12-25T12:47:14Z. DO NOT EDIT.
#
usage() {
this=$1
cat <<EOF
$this: download go binaries for goreleaser/goreleaser
Usage: $this [-b] bindir [-d] [tag]
-b sets bindir or installation directory, Defaults to ./bin
-d turns on debug logging
[tag] is a tag from
https://github.com/goreleaser/goreleaser/releases
If tag is missing, then the latest will be used.
Generated by godownloader
https://github.com/goreleaser/godownloader
EOF
exit 2
}
parse_args() {
#BINDIR is ./bin unless set be ENV
# over-ridden by flag below
BINDIR=${BINDIR:-./bin}
while getopts "b:dh?x" arg; do
case "$arg" in
b) BINDIR="$OPTARG" ;;
d) log_set_priority 10 ;;
h | \?) usage "$0" ;;
x) set -x ;;
esac
done
shift $((OPTIND - 1))
TAG=$1
}
# this function wraps all the destructive operations
# if a curl|bash cuts off the end of the script due to
# network, either nothing will happen or will syntax error
# out preventing half-done work
execute() {
tmpdir=$(mktemp -d)
log_debug "downloading files into ${tmpdir}"
http_download "${tmpdir}/${TARBALL}" "${TARBALL_URL}"
http_download "${tmpdir}/${CHECKSUM}" "${CHECKSUM_URL}"
hash_sha256_verify "${tmpdir}/${TARBALL}" "${tmpdir}/${CHECKSUM}"
srcdir="${tmpdir}"
(cd "${tmpdir}" && untar "${TARBALL}")
test ! -d "${BINDIR}" && install -d "${BINDIR}"
for binexe in $BINARIES; do
if [ "$OS" = "windows" ]; then
binexe="${binexe}.exe"
fi
install "${srcdir}/${binexe}" "${BINDIR}/"
log_info "installed ${BINDIR}/${binexe}"
done
rm -rf "${tmpdir}"
}
get_binaries() {
case "$PLATFORM" in
darwin/386) BINARIES="goreleaser" ;;
darwin/amd64) BINARIES="goreleaser" ;;
darwin/arm64) BINARIES="goreleaser" ;;
darwin/armv6) BINARIES="goreleaser" ;;
linux/386) BINARIES="goreleaser" ;;
linux/amd64) BINARIES="goreleaser" ;;
linux/arm64) BINARIES="goreleaser" ;;
linux/armv6) BINARIES="goreleaser" ;;
windows/386) BINARIES="goreleaser" ;;
windows/amd64) BINARIES="goreleaser" ;;
windows/arm64) BINARIES="goreleaser" ;;
windows/armv6) BINARIES="goreleaser" ;;
*)
log_crit "platform $PLATFORM is not supported. Make sure this script is up-to-date and file request at https://github.com/${PREFIX}/issues/new"
exit 1
;;
esac
}
tag_to_version() {
if [ -z "${TAG}" ]; then
log_info "checking GitHub for latest tag"
else
log_info "checking GitHub for tag '${TAG}'"
fi
REALTAG=$(github_release "$OWNER/$REPO" "${TAG}") && true
if test -z "$REALTAG"; then
log_crit "unable to find '${TAG}' - use 'latest' or see https://github.com/${PREFIX}/releases for details"
exit 1
fi
# if version starts with 'v', remove it
TAG="$REALTAG"
VERSION=${TAG#v}
}
adjust_format() {
# change format (tar.gz or zip) based on OS
case ${OS} in
windows) FORMAT=zip ;;
esac
true
}
adjust_os() {
# adjust archive name based on OS
case ${OS} in
386) OS=i386 ;;
amd64) OS=x86_64 ;;
darwin) OS=Darwin ;;
linux) OS=Linux ;;
windows) OS=Windows ;;
esac
true
}
adjust_arch() {
# adjust archive name based on ARCH
case ${ARCH} in
386) ARCH=i386 ;;
amd64) ARCH=x86_64 ;;
darwin) ARCH=Darwin ;;
linux) ARCH=Linux ;;
windows) ARCH=Windows ;;
esac
true
}
cat /dev/null <<EOF
------------------------------------------------------------------------
https://github.com/client9/shlib - portable posix shell functions
Public domain - http://unlicense.org
https://github.com/client9/shlib/blob/master/LICENSE.md
but credit (and pull requests) appreciated.
------------------------------------------------------------------------
EOF
is_command() {
command -v "$1" >/dev/null
}
echoerr() {
echo "$@" 1>&2
}
log_prefix() {
echo "$0"
}
_logp=6
log_set_priority() {
_logp="$1"
}
log_priority() {
if test -z "$1"; then
echo "$_logp"
return
fi
[ "$1" -le "$_logp" ]
}
log_tag() {
case $1 in
0) echo "emerg" ;;
1) echo "alert" ;;
2) echo "crit" ;;
3) echo "err" ;;
4) echo "warning" ;;
5) echo "notice" ;;
6) echo "info" ;;
7) echo "debug" ;;
*) echo "$1" ;;
esac
}
log_debug() {
log_priority 7 || return 0
echoerr "$(log_prefix)" "$(log_tag 7)" "$@"
}
log_info() {
log_priority 6 || return 0
echoerr "$(log_prefix)" "$(log_tag 6)" "$@"
}
log_err() {
log_priority 3 || return 0
echoerr "$(log_prefix)" "$(log_tag 3)" "$@"
}
log_crit() {
log_priority 2 || return 0
echoerr "$(log_prefix)" "$(log_tag 2)" "$@"
}
uname_os() {
os=$(uname -s | tr '[:upper:]' '[:lower:]')
case "$os" in
cygwin_nt*) os="windows" ;;
mingw*) os="windows" ;;
msys_nt*) os="windows" ;;
esac
echo "$os"
}
uname_arch() {
arch=$(uname -m)
case $arch in
x86_64) arch="amd64" ;;
x86) arch="386" ;;
i686) arch="386" ;;
i386) arch="386" ;;
aarch64) arch="arm64" ;;
armv5*) arch="armv5" ;;
armv6*) arch="armv6" ;;
armv7*) arch="armv7" ;;
esac
echo ${arch}
}
uname_os_check() {
os=$(uname_os)
case "$os" in
darwin) return 0 ;;
dragonfly) return 0 ;;
freebsd) return 0 ;;
linux) return 0 ;;
android) return 0 ;;
nacl) return 0 ;;
netbsd) return 0 ;;
openbsd) return 0 ;;
plan9) return 0 ;;
solaris) return 0 ;;
windows) return 0 ;;
esac
log_crit "uname_os_check '$(uname -s)' got converted to '$os' which is not a GOOS value. Please file bug at https://github.com/client9/shlib"
return 1
}
uname_arch_check() {
arch=$(uname_arch)
case "$arch" in
386) return 0 ;;
amd64) return 0 ;;
arm64) return 0 ;;
armv5) return 0 ;;
armv6) return 0 ;;
armv7) return 0 ;;
ppc64) return 0 ;;
ppc64le) return 0 ;;
mips) return 0 ;;
mipsle) return 0 ;;
mips64) return 0 ;;
mips64le) return 0 ;;
s390x) return 0 ;;
amd64p32) return 0 ;;
esac
log_crit "uname_arch_check '$(uname -m)' got converted to '$arch' which is not a GOARCH value. Please file bug report at https://github.com/client9/shlib"
return 1
}
untar() {
tarball=$1
case "${tarball}" in
*.tar.gz | *.tgz) tar --no-same-owner -xzf "${tarball}" ;;
*.tar) tar --no-same-owner -xf "${tarball}" ;;
*.zip) unzip "${tarball}" ;;
*)
log_err "untar unknown archive format for ${tarball}"
return 1
;;
esac
}
http_download_curl() {
local_file=$1
source_url=$2
header=$3
if [ -z "$header" ]; then
code=$(curl -w '%{http_code}' -sL -o "$local_file" "$source_url")
else
code=$(curl -w '%{http_code}' -sL -H "$header" -o "$local_file" "$source_url")
fi
if [ "$code" != "200" ]; then
log_debug "http_download_curl received HTTP status $code"
return 1
fi
return 0
}
http_download_wget() {
local_file=$1
source_url=$2
header=$3
if [ -z "$header" ]; then
wget -q -O "$local_file" "$source_url"
else
wget -q --header "$header" -O "$local_file" "$source_url"
fi
}
http_download() {
log_debug "http_download $2"
if is_command curl; then
http_download_curl "$@"
return
elif is_command wget; then
http_download_wget "$@"
return
fi
log_crit "http_download unable to find wget or curl"
return 1
}
http_copy() {
tmp=$(mktemp)
http_download "${tmp}" "$1" "$2" || return 1
body=$(cat "$tmp")
rm -f "${tmp}"
echo "$body"
}
github_release() {
owner_repo=$1
version=$2
test -z "$version" && version="latest"
giturl="https://github.com/${owner_repo}/releases/${version}"
json=$(http_copy "$giturl" "Accept:application/json")
test -z "$json" && return 1
version=$(echo "$json" | tr -s '\n' ' ' | sed 's/.*"tag_name":"//' | sed 's/".*//')
test -z "$version" && return 1
echo "$version"
}
hash_sha256() {
TARGET=${1:-/dev/stdin}
if is_command gsha256sum; then
hash=$(gsha256sum "$TARGET") || return 1
echo "$hash" | cut -d ' ' -f 1
elif is_command sha256sum; then
hash=$(sha256sum "$TARGET") || return 1
echo "$hash" | cut -d ' ' -f 1
elif is_command shasum; then
hash=$(shasum -a 256 "$TARGET" 2>/dev/null) || return 1
echo "$hash" | cut -d ' ' -f 1
elif is_command openssl; then
hash=$(openssl -dst openssl dgst -sha256 "$TARGET") || return 1
echo "$hash" | cut -d ' ' -f a
else
log_crit "hash_sha256 unable to find command to compute sha-256 hash"
return 1
fi
}
hash_sha256_verify() {
TARGET=$1
checksums=$2
if [ -z "$checksums" ]; then
log_err "hash_sha256_verify checksum file not specified in arg2"
return 1
fi
BASENAME=${TARGET##*/}
want=$(grep "${BASENAME}" "${checksums}" 2>/dev/null | tr '\t' ' ' | cut -d ' ' -f 1)
if [ -z "$want" ]; then
log_err "hash_sha256_verify unable to find checksum for '${TARGET}' in '${checksums}'"
return 1
fi
got=$(hash_sha256 "$TARGET")
if [ "$want" != "$got" ]; then
log_err "hash_sha256_verify checksum for '$TARGET' did not verify ${want} vs $got"
return 1
fi
}
cat /dev/null <<EOF
------------------------------------------------------------------------
End of functions from https://github.com/client9/shlib
------------------------------------------------------------------------
EOF
PROJECT_NAME="goreleaser"
OWNER=goreleaser
REPO="goreleaser"
BINARY=goreleaser
FORMAT=tar.gz
OS=$(uname_os)
ARCH=$(uname_arch)
PREFIX="$OWNER/$REPO"
# use in logging routines
log_prefix() {
echo "$PREFIX"
}
PLATFORM="${OS}/${ARCH}"
GITHUB_DOWNLOAD=https://github.com/${OWNER}/${REPO}/releases/download
uname_os_check "$OS"
uname_arch_check "$ARCH"
parse_args "$@"
get_binaries
tag_to_version
adjust_format
adjust_os
adjust_arch
log_info "found version: ${VERSION} for ${TAG}/${OS}/${ARCH}"
NAME=${PROJECT_NAME}_${OS}_${ARCH}
TARBALL=${NAME}.${FORMAT}
TARBALL_URL=${GITHUB_DOWNLOAD}/${TAG}/${TARBALL}
CHECKSUM=checksums.txt
CHECKSUM_URL=${GITHUB_DOWNLOAD}/${TAG}/${CHECKSUM}
execute
log_err "this script is deprecated, please do not use it anymore. check https://github.com/goreleaser/godownloader/issues/207"
+9
View File
@@ -31,6 +31,15 @@ jobs:
steps:
- name: Checkout repository
uses: actions/checkout@v2
with:
# We must fetch at least the immediate parents so that if this is
# a pull request then we can checkout the head.
fetch-depth: 2
# If this run was triggered by a pull request event, then checkout
# the head of the pull request instead of the merge commit.
- run: git checkout HEAD^2
if: ${{ github.event_name == 'pull_request' }}
# Initializes the CodeQL tools for scanning.
- name: Initialize CodeQL
+4
View File
@@ -96,6 +96,10 @@ jobs:
steps:
- uses: docker-practice/actions-setup-docker@v1
# note, it is important to always be auth'd into docker.io to prevent rate limiting issues
- name: Login to Docker Hub
run: echo ${{ secrets.TOOLBOX_DOCKER_PASS }} | docker login docker.io -u ${{ secrets.TOOLBOX_DOCKER_USER }} --password-stdin
- uses: actions/setup-go@v2
with:
go-version: ${{ env.GO_VERSION }}
-7
View File
@@ -232,9 +232,6 @@ jobs:
- uses: actions/checkout@v2
- name: Set up QEMU
uses: docker/setup-qemu-action@v1
- name: Restore tool cache
id: tool-cache
uses: actions/cache@v2.1.3
@@ -308,10 +305,6 @@ jobs:
needs: [Build-Snapshot-Artifacts]
runs-on: ubuntu-20.04
steps:
- uses: actions/setup-go@v2
with:
go-version: ${{ env.GO_VERSION }}
- uses: actions/checkout@v2
- name: Restore go cache
+3 -5
View File
@@ -15,7 +15,6 @@ linters:
- dogsled
- dupl
- errcheck
- exportloopref
- funlen
- gocognit
- goconst
@@ -23,6 +22,7 @@ linters:
- gocyclo
- gofmt
- goimports
- golint
- goprintffuncname
- gosec
- gosimple
@@ -31,8 +31,8 @@ linters:
- misspell
- nakedret
- nolintlint
- revive
- rowserrcheck
- scopelint
- staticcheck
- structcheck
- stylecheck
@@ -49,13 +49,11 @@ linters:
# - godot
# - godox
# - goerr113
# - golint # deprecated
# - gomnd # this is too aggressive
# - interfacer # this is a good idea, but is no longer supported and is prone to false positives
# - lll # without a way to specify per-line exception cases, this is not usable
# - maligned # this is an excellent linter, but tricky to optimize and we are not sensitive to memory layout optimizations
# - nestif
# - prealloc # following this rule isn't consistently a good idea, as it sometimes forces unnecessary allocations that result in less idiomatic code
# - scopelint # deprecated
# - testpackage
# - wsl # this doens't have an auto-fixer yet and is pretty noisy (https://github.com/bombsimon/wsl/issues/90)
# - wsl
+10 -64
View File
@@ -12,10 +12,11 @@ builds:
env:
- CGO_ENABLED=0
goos:
# windows not supported yet (due to jotframe)
# - windows
- linux
goarch:
- amd64
- arm64
# Set the modified timestamp on the output binary to the git timestamp (to ensure a reproducible build)
mod_timestamp: '{{ .CommitTimestamp }}'
ldflags: |
@@ -26,27 +27,7 @@ builds:
-X github.com/anchore/syft/internal/version.gitCommit={{.Commit}}
-X github.com/anchore/syft/internal/version.buildDate={{.Date}}
-X github.com/anchore/syft/internal/version.gitTreeState={{.Env.BUILD_GIT_TREE_STATE}}
- binary: syft
id: syft-win
env:
- CGO_ENABLED=0
goos:
- windows
goarch:
- amd64
# Set the modified timestamp on the output binary to the git timestamp (to ensure a reproducible build)
mod_timestamp: '{{ .CommitTimestamp }}'
ldflags: |
-w
-s
-extldflags '-static'
-X github.com/anchore/syft/internal/version.version={{.Version}}
-X github.com/anchore/syft/internal/version.gitCommit={{.Commit}}
-X github.com/anchore/syft/internal/version.buildDate={{.Date}}
-X github.com/anchore/syft/internal/version.gitTreeState={{.Env.BUILD_GIT_TREE_STATE}}
# For more info on this macOS build, see: https://github.com/mitchellh/gon#usage-with-goreleaser
# For more info on this macOS build, see: https://github.com/mitchellh/gon#usage-with-goreleaser
- binary: syft
id: syft-macos
env:
@@ -55,7 +36,6 @@ builds:
- darwin
goarch:
- amd64
- arm64
# Set the modified timestamp on the output binary to the git timestamp (to ensure a reproducible build)
mod_timestamp: '{{ .CommitTimestamp }}'
ldflags: |
@@ -71,11 +51,10 @@ archives:
- format: tar.gz
builds:
- syft # i.e. Linux only
- format: zip # This is a hack for syft-macos! We don't actually intend to use _this_ ZIP file, we just need goreleaser to consider the ZIP file produced by gon (which will have the same file name) to be an artifact so we can use it downstream in publishing (e.g. to a homebrew tap)
- format: zip # This is a hack! We don't actually intend to use _this_ ZIP file, we just need goreleaser to consider the ZIP file produced by gon (which will have the same file name) to be an artifact so we can use it downstream in publishing (e.g. to a homebrew tap)
id: syft-zip
builds:
- syft-macos
- syft-win
signs:
- artifacts: checksum
@@ -113,48 +92,15 @@ brews:
description: *description
dockers:
- image_templates:
- dockerfile: Dockerfile
image_templates:
- "anchore/syft:latest"
- "anchore/syft:{{ .Tag }}-amd64"
- "anchore/syft:v{{ .Major }}-amd64"
- "anchore/syft:v{{ .Major }}.{{ .Minor }}-amd64"
dockerfile: Dockerfile
use: buildx
- "anchore/syft:{{ .Tag }}"
- "anchore/syft:v{{ .Major }}"
- "anchore/syft:v{{ .Major }}.{{ .Minor }}"
build_flag_templates:
- "--platform=linux/amd64"
- "--build-arg=BUILD_DATE={{.Date}}"
- "--build-arg=BUILD_VERSION={{.Version}}"
- "--build-arg=VCS_REF={{.FullCommit}}"
- "--build-arg=VCS_URL={{.GitURL}}"
- image_templates:
- "anchore/syft:{{ .Tag }}-arm64v8"
- "anchore/syft:v{{ .Major }}-arm64v8"
- "anchore/syft:v{{ .Major }}.{{ .Minor }}-arm64v8"
goarch: arm64
dockerfile: Dockerfile
use: buildx
build_flag_templates:
- "--platform=linux/arm64/v8"
- "--build-arg=BUILD_DATE={{.Date}}"
- "--build-arg=BUILD_VERSION={{.Version}}"
- "--build-arg=VCS_REF={{.FullCommit}}"
- "--build-arg=VCS_URL={{.GitURL}}"
docker_manifests:
- name_template: anchore/syft:{{ .Tag }}
image_templates:
- anchore/syft:{{ .Tag }}-amd64
- anchore/syft:v{{ .Major }}-amd64
- anchore/syft:v{{ .Major }}.{{ .Minor }}-amd64
- anchore/syft:{{ .Tag }}-arm64v8
- anchore/syft:v{{ .Major }}-arm64v8
- anchore/syft:v{{ .Major }}.{{ .Minor }}-arm64v8
- name_template: anchore/syft:latest
image_templates:
- anchore/syft:{{ .Tag }}-amd64
- anchore/syft:v{{ .Major }}-amd64
- anchore/syft:v{{ .Major }}.{{ .Minor }}-amd64
- anchore/syft:{{ .Tag }}-arm64v8
- anchore/syft:v{{ .Major }}-arm64v8
- anchore/syft:v{{ .Major }}.{{ .Minor }}-arm64v8
use_buildx: true
+41 -27
View File
@@ -15,10 +15,10 @@ RESET := $(shell tput -T linux sgr0)
TITLE := $(BOLD)$(PURPLE)
SUCCESS := $(BOLD)$(GREEN)
# the quality gate lower threshold for unit test total % coverage (by function statements)
COVERAGE_THRESHOLD := 62
COVERAGE_THRESHOLD := 65
# CI cache busting values; change these if you want CI to not use previous stored cache
INTEGRATION_CACHE_BUSTER="88738d2f"
CLI_CACHE_BUSTER="9a2c03cf"
CLI_CACHE_BUSTER="789bacdf"
BOOTSTRAP_CACHE="c7afb99ad"
## Build variables
@@ -37,6 +37,10 @@ ifeq "$(strip $(VERSION))" ""
override VERSION = $(shell git describe --always --tags --dirty)
endif
# used to generate the changelog from the second to last tag to the current tag (used in the release pipeline when the release tag is in place)
LAST_TAG := $(shell git describe --abbrev=0 --tags $(shell git rev-list --tags --max-count=1))
SECOND_TO_LAST_TAG := $(shell git describe --abbrev=0 --tags $(shell git rev-list --tags --skip=1 --max-count=1))
## Variable assertions
ifndef TEMPDIR
@@ -97,11 +101,9 @@ $(TEMPDIR):
.PHONY: bootstrap-tools
bootstrap-tools: $(TEMPDIR)
GO111MODULE=off GOBIN=$(shell realpath $(TEMPDIR)) go get -u golang.org/x/perf/cmd/benchstat
curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(TEMPDIR)/ v1.42.1
curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(TEMPDIR)/ v1.26.0
curl -sSfL https://raw.githubusercontent.com/wagoodman/go-bouncer/master/bouncer.sh | sh -s -- -b $(TEMPDIR)/ v0.2.0
curl -sSfL https://raw.githubusercontent.com/anchore/chronicle/main/install.sh | sh -s -- -b $(TEMPDIR)/ v0.3.0
.github/scripts/goreleaser-install.sh -b $(TEMPDIR)/ v0.177.0
GOBIN="$(shell realpath $(TEMPDIR))" go install github.com/neilpa/yajsv@v1.4.0
curl -sfL https://install.goreleaser.com/github.com/goreleaser/goreleaser.sh | sh -s -- -b $(TEMPDIR)/ v0.177.0
.PHONY: bootstrap-go
bootstrap-go:
@@ -136,7 +138,7 @@ lint-fix: ## Auto-format all source code + run golangci lint fixers
go mod tidy
.PHONY: check-licenses
check-licenses: ## Ensure transitive dependencies are compliant with the current license policy
check-licenses:
$(TEMPDIR)/bouncer check
check-go-mod-tidy:
@@ -191,11 +193,6 @@ fixtures:
generate-json-schema: ## Generate a new json schema
cd schema/json && go run generate.go
.PHONY: generate-license-list
generate-license-list: ## Generate an updated spdx license list
go generate ./internal/spdxlicense/...
gofmt -s -w ./internal/spdxlicense
.PHONY: build
build: $(SNAPSHOTDIR) ## Build release snapshot binaries and packages
@@ -206,9 +203,7 @@ $(SNAPSHOTDIR): ## Build snapshot release binaries and packages
cat .goreleaser.yaml >> $(TEMPDIR)/goreleaser.yaml
# build release snapshots
# DOCKER_CLI_EXPERIMENTAL needed to support multi architecture builds for goreleaser
BUILD_GIT_TREE_STATE=$(GITTREESTATE) \
DOCKER_CLI_EXPERIMENTAL=enabled \
$(TEMPDIR)/goreleaser release --skip-publish --skip-sign --rm-dist --snapshot --config $(TEMPDIR)/goreleaser.yaml
# note: we cannot clean the snapshot directory since the pipeline builds the snapshot separately
@@ -254,19 +249,44 @@ cli: $(SNAPSHOTDIR) ## Run CLI tests
SYFT_BINARY_LOCATION='$(SNAPSHOT_CMD)' \
go test -count=1 -v ./test/cli
.PHONY: changelog
changelog: clean-changelog CHANGELOG.md
.PHONY: changlog-release
changelog-release:
@echo "Last tag: $(SECOND_TO_LAST_TAG)"
@docker run --rm \
-v "$(shell pwd)":/usr/local/src/your-app \
ferrarimarco/github-changelog-generator \
--user anchore \
--project $(BIN) \
-t ${GITHUB_TOKEN} \
--exclude-labels 'duplicate,question,invalid,wontfix,size:small,size:medium,size:large,size:x-large' \
--no-pr-wo-labels \
--no-issues-wo-labels \
--since-tag $(SECOND_TO_LAST_TAG)
@printf '\n$(BOLD)$(CYAN)Release $(VERSION) Changelog$(RESET)\n\n'
@cat CHANGELOG.md
.PHONY: changelog-unreleased
changelog-unreleased: ## show the current changelog that will be produced on the next release (note: requires GITHUB_TOKEN set)
@docker run -it --rm \
-v "$(shell pwd)":/usr/local/src/your-app \
ferrarimarco/github-changelog-generator \
--user anchore \
--project $(BIN) \
-t ${GITHUB_TOKEN} \
--exclude-labels 'duplicate,question,invalid,wontfix,size:small,size:medium,size:large,size:x-large' \
--since-tag $(LAST_TAG)
@printf '\n$(BOLD)$(CYAN)Unreleased Changes (closed PRs and issues will not be in the final changelog)$(RESET)\n'
@docker run -it --rm \
-v $(shell pwd)/CHANGELOG.md:/CHANGELOG.md \
rawkode/mdv \
-t 748.5989 \
/CHANGELOG.md
CHANGELOG.md:
$(TEMPDIR)/chronicle -vv > CHANGELOG.md
.PHONY: release
release: clean-dist CHANGELOG.md ## Build and publish final binaries and packages. Intended to be run only on macOS.
release: clean-dist changelog-release ## Build and publish final binaries and packages. Intended to be run only on macOS.
$(call title,Publishing release artifacts)
# Prepare for macOS-specific signing process
@@ -281,11 +301,9 @@ release: clean-dist CHANGELOG.md ## Build and publish final binaries and packag
cat .goreleaser.yaml >> $(TEMPDIR)/goreleaser.yaml
# release (note the version transformation from v0.7.0 --> 0.7.0)
# DOCKER_CLI_EXPERIMENTAL needed to support multi architecture builds for goreleaser
bash -c "\
BUILD_GIT_TREE_STATE=$(GITTREESTATE) \
VERSION=$(VERSION:v%=%) \
DOCKER_CLI_EXPERIMENTAL=enabled \
$(TEMPDIR)/goreleaser \
--rm-dist \
--config $(TEMPDIR)/goreleaser.yaml \
@@ -307,13 +325,9 @@ clean-snapshot:
rm -rf $(SNAPSHOTDIR) $(TEMPDIR)/goreleaser.yaml
.PHONY: clean-dist
clean-dist: clean-changelog
clean-dist:
rm -rf $(DISTDIR) $(TEMPDIR)/goreleaser.yaml
.PHONY: clean-changelog
clean-changelog:
rm -f CHANGELOG.md
clean-test-image-cache: clean-test-image-tar-cache clean-test-image-docker-cache
.PHONY: clear-test-image-tar-cache
+14 -147
View File
@@ -1,6 +1,4 @@
<p align="center">
<img src="https://user-images.githubusercontent.com/5199289/136844524-1527b09f-c5cb-4aa9-be54-5aa92a6086c1.png" width="271" alt="Cute pink owl syft logo">
</p>
# syft
[![Validations](https://github.com/anchore/syft/actions/workflows/validations.yaml/badge.svg)](https://github.com/anchore/syft/actions/workflows/validations.yaml)
[![Go Report Card](https://goreportcard.com/badge/github.com/anchore/syft)](https://goreportcard.com/report/github.com/anchore/syft)
@@ -9,17 +7,19 @@
[![License: Apache-2.0](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](https://github.com/anchore/syft/blob/main/LICENSE)
[![Slack Invite](https://img.shields.io/badge/Slack-Join-blue?logo=slack)](https://anchore.com/slack)
A CLI tool and Go library for generating a Software Bill of Materials (SBOM) from container images and filesystems. Exceptional for vulnerability detection when used with a scanner tool like [Grype](https://github.com/anchore/grype).
A CLI tool and go library for generating a Software Bill of Materials (SBOM) from container images and filesystems. Exceptional for vulnerability detection when used with a scanner tool like [Grype](https://github.com/anchore/grype).
### Join our community meetings!
## We'll be at KubeCon 2021!
- Calendar: https://calendar.google.com/calendar/u/0/r?cid=Y182OTM4dGt0MjRtajI0NnNzOThiaGtnM29qNEBncm91cC5jYWxlbmRhci5nb29nbGUuY29t
- Agenda: https://docs.google.com/document/d/1ZtSAa6fj2a6KRWviTn3WoJm09edvrNUp4Iz_dOjjyY8/edit?usp=sharing (join [this group](https://groups.google.com/g/anchore-oss-community) for write access)
- All are welcome!
Attending KubeCon 2021 in person? Join us for a meetup on **Tuesday, October 12th**.
We’ll have free swag, giveaways, snacks, and sips. Space will be limited, so make sure to [save your seat](https://get.anchore.com/2021-kubecon-na-opensource-happy-hour/)!
---
![syft-demo](https://user-images.githubusercontent.com/590471/90277200-2a253000-de33-11ea-893f-32c219eea11a.gif)
## Features
**Features**
- Catalog container images and filesystems to discover packages and libraries.
- Supports packages and libraries from various ecosystems (APK, DEB, RPM, Ruby Bundles, Python Wheel/Egg/requirements.txt, JavaScript NPM/Yarn, Java JAR/EAR/WAR, Jenkins plugins JPI/HPI, Go modules)
- Linux distribution identification (supports Alpine, BusyBox, CentOS/RedHat, Debian/Ubuntu flavored distributions)
@@ -89,136 +89,25 @@ docker-archive:path/to/yourimage.tar use a tarball from disk for archives crea
oci-archive:path/to/yourimage.tar use a tarball from disk for OCI archives (from Skopeo or otherwise)
oci-dir:path/to/yourimage read directly from a path on disk for OCI layout directories (from Skopeo or otherwise)
dir:path/to/yourproject read directly from a path on disk (any directory)
file:path/to/yourproject/file read directly from a path on disk (any single file)
registry:yourrepo/yourimage:tag pull image directly from a registry (no container runtime required)
```
### Excluding file paths
Syft can exclude files and paths from being scanned within a source by using glob expressions
with one or more `--exclude` parameters:
```
syft <source> --exclude './out/**/*.json' --exclude /etc
```
**Note:** in the case of _image scanning_, since the entire filesystem is scanned it is
possible to use absolute paths like `/etc` or `/usr/**/*.txt` whereas _directory scans_
exclude files _relative to the specified directory_. For example: scanning `/usr/foo` with
`--exclude ./package.json` would exclude `/usr/foo/package.json` and `--exclude '**/package.json'`
would exclude all `package.json` files under `/usr/foo`. For _directory scans_,
it is required to begin path expressions with `./`, `*/`, or `**/`, all of which
will be resolved _relative to the specified scan directory_. Keep in mind, your shell
may attempt to expand wildcards, so put those parameters in single quotes, like:
`'**/*.json'`.
### Output formats
The output format for Syft is configurable as well using the
`-o` (or `--output`) option:
The output format for Syft is configurable as well:
```
syft packages <image> -o <format>
```
Where the `formats` available are:
Where the `format`s available are:
- `json`: Use this to get as much information out of Syft as possible!
- `text`: A row-oriented, human-and-machine-friendly output.
- `cyclonedx`: A XML report conforming to the [CycloneDX 1.3 specification](https://cyclonedx.org/specification/overview/).
- `cyclonedx-json`: A JSON report conforming to the [CycloneDX 1.3 specification](https://cyclonedx.org/specification/overview/).
- `cyclonedx`: A XML report conforming to the [CycloneDX 1.2 specification](https://cyclonedx.org/specification/overview/).
- `spdx`: A tag-value formatted report conforming to the [SPDX 2.2 specification](https://spdx.github.io/spdx-spec/).
- `spdx-json`: A JSON report conforming to the [SPDX 2.2 JSON Schema](https://github.com/spdx/spdx-spec/blob/v2.2/schemas/spdx-schema.json).
- `table`: A columnar summary (default).
#### Multiple outputs
Syft can also output _multiple_ files in differing formats by appending
`=<file>` to the option, for example to output Syft JSON and SPDX JSON:
```shell
syft packages <image> -o json=sbom.syft.json -o spdx-json=sbom.spdx.json
```
## Private Registry Authentication
### Local Docker Credentials
When a container runtime is not present, Syft can still utilize credentials configured in common credential sources (such as `~/.docker/config.json`).
It will pull images from private registries using these credentials. The config file is where your credentials are stored when authenticating with private registries via some command like `docker login`.
For more information see the `go-containerregistry` [documentation](https://github.com/google/go-containerregistry/tree/main/pkg/authn).
An example `config.json` looks something like this:
```
// config.json
{
"auths": {
"registry.example.com": {
"username": "AzureDiamond",
"password": "hunter2"
}
}
}
```
You can run the following command as an example. It details the mount/environment configuration a container needs to access a private registry:
`docker run -v ./config.json:/config/config.json -e "DOCKER_CONFIG=/config" anchore/syft:latest <private_image>`
### Docker Credentials in Kubernetes
The below section shows a simple workflow on how to mount this config file as a secret into a container on kubernetes.
1. Create a secret. The value of `config.json` is important. It refers to the specification detailed [here](https://github.com/google/go-containerregistry/tree/main/pkg/authn#the-config-file).
Below this section is the `secret.yaml` file that the pod configuration will consume as a volume.
The key `config.json` is important. It will end up being the name of the file when mounted into the pod.
```
# secret.yaml
apiVersion: v1
kind: Secret
metadata:
name: registry-config
namespace: syft
data:
config.json: <base64 encoded config.json>
```
`kubectl apply -f secret.yaml`
2. Create your pod running syft. The env `DOCKER_CONFIG` is important because it advertises where to look for the credential file.
In the below example, setting `DOCKER_CONFIG=/config` informs syft that credentials can be found at `/config/config.json`.
This is why we used `config.json` as the key for our secret. When mounted into containers the secrets' key is used as the filename.
The `volumeMounts` section mounts our secret to `/config`. The `volumes` section names our volume and leverages the secret we created in step one.
```
# pod.yaml
apiVersion: v1
kind: Pod
spec:
containers:
- image: anchore/syft:latest
name: syft-private-registry-demo
env:
- name: DOCKER_CONFIG
value: /config
volumeMounts:
- mountPath: /config
name: registry-config
readOnly: true
args:
- <private_image>
volumes:
- name: registry-config
secret:
secretName: registry-config
```
`kubectl apply -f pod.yaml`
3. The user can now run `kubectl logs syft-private-registry-demo`. The logs should show the syft analysis for the `<private_image>` provided in the pod configuration.
Using the above information, users should be able to configure private registry access without having to do so in the `grype` or `syft` configuration files.
They will also not be dependent on a docker daemon, (or some other runtime software) for registry configuration and access.
## Configuration
Configuration search paths:
@@ -231,12 +120,8 @@ Configuration search paths:
Configuration options (example values are the default):
```yaml
# the output format(s) of the SBOM report (options: table, text, json, spdx, ...)
# same as -o, --output, and SYFT_OUTPUT env var
# to specify multiple output files in differing formats, use a list:
# output:
# - "json=<syft-json-output-file>"
# - "spdx-json=<spdx-json-output-file>"
# the output format of the SBOM report (options: table, text, json)
# same as -o ; SYFT_OUTPUT env var
output: "table"
# suppress all output (except for the SBOM report)
@@ -250,26 +135,8 @@ file: ""
# same as SYFT_CHECK_FOR_APP_UPDATE env var
check-for-app-update: true
# a list of globs to exclude from scanning. same as --exclude ; for example:
# exclude:
# - "/etc/**"
# - "./out/**/*.json"
exclude:
# cataloging packages is exposed through the packages and power-user subcommands
package:
# search within archives that do contain a file index to search against (zip)
# note: for now this only applies to the java package cataloger
# SYFT_PACKAGE_SEARCH_INDEXED_ARCHIVES env var
search-indexed-archives: true
# search within archives that do not contain a file index to search against (tar, tar.gz, tar.bz2, etc)
# note: enabling this may result in a performance impact since all discovered compressed tars will be decompressed
# note: for now this only applies to the java package cataloger
# SYFT_PACKAGE_SEARCH_UNINDEXED_ARCHIVES env var
search-unindexed-archives: false
cataloger:
# enable/disable cataloging of packages
# SYFT_PACKAGE_CATALOGER_ENABLED env var
-1
View File
@@ -11,7 +11,6 @@ import (
func checkForApplicationUpdate() {
if appConfig.CheckForAppUpdate {
log.Debugf("checking if new vesion of %s is available", internal.ApplicationName)
isAvailable, newVersion, err := version.IsUpdateAvailable()
if err != nil {
// this should never stop the application
+4 -44
View File
@@ -1,19 +1,17 @@
package cmd
import (
"encoding/json"
"fmt"
"os"
"sort"
"github.com/spf13/cobra"
"github.com/anchore/stereoscope"
"github.com/anchore/syft/internal/config"
"github.com/anchore/syft/internal/log"
"github.com/anchore/syft/internal/logger"
"github.com/anchore/syft/internal/version"
"github.com/anchore/syft/syft"
"github.com/gookit/color"
"github.com/spf13/cobra"
"github.com/spf13/viper"
"github.com/wagoodman/go-partybus"
)
@@ -30,8 +28,6 @@ func init() {
initAppConfig,
initLogging,
logAppConfig,
checkForApplicationUpdate,
logAppVersion,
initEventBus,
)
}
@@ -52,13 +48,7 @@ func initCmdAliasBindings() {
panic(err)
}
// enable all cataloger by default if power-user command is run
if activeCmd == powerUserCmd {
config.PowerUserCatalogerEnabledDefault()
}
switch activeCmd {
case packagesCmd, rootCmd:
if activeCmd == packagesCmd || activeCmd == rootCmd {
// note: we need to lazily bind config options since they are shared between both the root command
// and the packages command. Otherwise there will be global viper state that is in contention.
// See for more details: https://github.com/spf13/viper/issues/233 . Additionally, the bindings must occur BEFORE
@@ -68,7 +58,7 @@ func initCmdAliasBindings() {
if err = bindPackagesConfigOptions(activeCmd.Flags()); err != nil {
panic(err)
}
default:
} else {
// even though the root command or packages command is NOT being run, we still need default bindings
// such that application config parsing passes.
if err = bindPackagesConfigOptions(packagesCmd.Flags()); err != nil {
@@ -114,33 +104,3 @@ func initEventBus() {
stereoscope.SetBus(eventBus)
syft.SetBus(eventBus)
}
func logAppVersion() {
versionInfo := version.FromBuild()
log.Infof("syft version: %s", versionInfo.Version)
var fields map[string]interface{}
bytes, err := json.Marshal(versionInfo)
if err != nil {
return
}
err = json.Unmarshal(bytes, &fields)
if err != nil {
return
}
keys := make([]string, 0, len(fields))
for k := range fields {
keys = append(keys, k)
}
sort.Strings(keys)
for idx, field := range keys {
value := fields[field]
branch := "├──"
if idx == len(fields)-1 {
branch = "└──"
}
log.Debugf(" %s %s: %s", branch, field, value)
}
}
+3 -2
View File
@@ -14,8 +14,9 @@ import (
// completionCmd represents the completion command
var completionCmd = &cobra.Command{
Use: "completion [bash|zsh|fish]",
Short: "Generate a shell completion for Syft (listing local docker images)",
Hidden: true,
Use: "completion [bash|zsh|fish]",
Short: "Generate a shell completion for Syft (listing local docker images)",
Long: `To load completions (docker image list):
Bash:
+5 -5
View File
@@ -14,7 +14,7 @@ import (
// eventLoop listens to worker errors (from execution path), worker events (from a partybus subscription), and
// signal interrupts. Is responsible for handling each event relative to a given UI an to coordinate eventing until
// an eventual graceful exit.
// nolint:funlen
// nolint:gocognit,funlen
func eventLoop(workerErrs <-chan error, signals <-chan os.Signal, subscription *partybus.Subscription, cleanupFn func(), uxs ...ui.UI) error {
defer cleanupFn()
events := subscription.Events()
@@ -41,10 +41,9 @@ func eventLoop(workerErrs <-chan error, signals <-chan os.Signal, subscription *
if err != nil {
// capture the error from the worker and unsubscribe to complete a graceful shutdown
retErr = multierror.Append(retErr, err)
_ = subscription.Unsubscribe()
// the worker has exited, we may have been mid-handling events for the UI which should now be
// ignored, in which case forcing a teardown of the UI irregardless of the state is required.
forceTeardown = true
if err := subscription.Unsubscribe(); err != nil {
retErr = multierror.Append(retErr, err)
}
}
case e, isOpen := <-events:
if !isOpen {
@@ -54,6 +53,7 @@ func eventLoop(workerErrs <-chan error, signals <-chan os.Signal, subscription *
if err := ux.Handle(e); err != nil {
if errors.Is(err, partybus.ErrUnsubscribe) {
log.Warnf("unable to unsubscribe from the event bus")
events = nil
} else {
retErr = multierror.Append(retErr, err)
+5 -5
View File
@@ -50,7 +50,7 @@ func Test_eventLoop_gracefulExit(t *testing.T) {
t.Cleanup(testBus.Close)
finalEvent := partybus.Event{
Type: event.Exit,
Type: event.PresenterReady,
}
worker := func() <-chan error {
@@ -182,7 +182,7 @@ func Test_eventLoop_unsubscribeError(t *testing.T) {
t.Cleanup(testBus.Close)
finalEvent := partybus.Event{
Type: event.Exit,
Type: event.PresenterReady,
}
worker := func() <-chan error {
@@ -251,8 +251,8 @@ func Test_eventLoop_handlerError(t *testing.T) {
t.Cleanup(testBus.Close)
finalEvent := partybus.Event{
Type: event.Exit,
Error: fmt.Errorf("an exit error occured"),
Type: event.PresenterReady,
Error: fmt.Errorf("unable to create presenter"),
}
worker := func() <-chan error {
@@ -376,7 +376,7 @@ func Test_eventLoop_uiTeardownError(t *testing.T) {
t.Cleanup(testBus.Close)
finalEvent := partybus.Event{
Type: event.Exit,
Type: event.PresenterReady,
}
worker := func() <-chan error {
-72
View File
@@ -1,72 +0,0 @@
package cmd
import (
"fmt"
"strings"
"github.com/anchore/syft/internal/formats"
"github.com/anchore/syft/internal/output"
"github.com/anchore/syft/syft/format"
"github.com/anchore/syft/syft/sbom"
"github.com/hashicorp/go-multierror"
)
// makeWriter creates a sbom.Writer for output or returns an error. this will either return a valid writer
// or an error but neither both and if there is no error, sbom.Writer.Close() should be called
func makeWriter(outputs []string, defaultFile string) (sbom.Writer, error) {
outputOptions, err := parseOptions(outputs, defaultFile)
if err != nil {
return nil, err
}
writer, err := output.MakeWriter(outputOptions...)
if err != nil {
return nil, err
}
return writer, nil
}
// parseOptions utility to parse command-line option strings and retain the existing behavior of default format and file
func parseOptions(outputs []string, defaultFile string) (out []output.WriterOption, errs error) {
// always should have one option -- we generally get the default of "table", but just make sure
if len(outputs) == 0 {
outputs = append(outputs, string(format.TableOption))
}
for _, name := range outputs {
name = strings.TrimSpace(name)
// split to at most two parts for <format>=<file>
parts := strings.SplitN(name, "=", 2)
// the format option is the first part
name = parts[0]
// default to the --file or empty string if not specified
file := defaultFile
// If a file is specified as part of the output option, use that
if len(parts) > 1 {
file = parts[1]
}
option := format.ParseOption(name)
if option == format.UnknownFormatOption {
errs = multierror.Append(errs, fmt.Errorf("bad output format: '%s'", name))
continue
}
encoder := formats.ByOption(option)
if encoder == nil {
errs = multierror.Append(errs, fmt.Errorf("unknown format: %s", outputFormat))
continue
}
out = append(out, output.WriterOption{
Format: *encoder,
Path: file,
})
}
return out, errs
}
-78
View File
@@ -1,78 +0,0 @@
package cmd
import (
"fmt"
"strings"
"testing"
"github.com/stretchr/testify/assert"
)
func TestOutputWriterConfig(t *testing.T) {
tmp := t.TempDir() + "/"
tests := []struct {
outputs []string
file string
err bool
expected []string
}{
{
outputs: []string{},
expected: []string{""},
},
{
outputs: []string{"json"},
expected: []string{""},
},
{
file: "test-1.json",
expected: []string{"test-1.json"},
},
{
outputs: []string{"json=test-2.json"},
expected: []string{"test-2.json"},
},
{
outputs: []string{"json=test-3-1.json", "spdx-json=test-3-2.json"},
expected: []string{"test-3-1.json", "test-3-2.json"},
},
{
outputs: []string{"text", "json=test-4.json"},
expected: []string{"", "test-4.json"},
},
}
for _, test := range tests {
t.Run(fmt.Sprintf("%s/%s", test.outputs, test.file), func(t *testing.T) {
outputs := test.outputs
for i, val := range outputs {
outputs[i] = strings.Replace(val, "=", "="+tmp, 1)
}
file := test.file
if file != "" {
file = tmp + file
}
_, err := makeWriter(test.outputs, file)
if test.err {
assert.Error(t, err)
return
} else {
assert.NoError(t, err)
}
for _, expected := range test.expected {
if expected != "" {
assert.FileExists(t, tmp+expected)
} else if file != "" {
assert.FileExists(t, file)
} else {
assert.NoFileExists(t, expected)
}
}
})
}
}
+68 -104
View File
@@ -6,18 +6,19 @@ import (
"io/ioutil"
"os"
"github.com/anchore/syft/syft/format"
"github.com/anchore/stereoscope"
"github.com/anchore/syft/internal"
"github.com/anchore/syft/internal/anchore"
"github.com/anchore/syft/internal/bus"
"github.com/anchore/syft/internal/log"
"github.com/anchore/syft/internal/ui"
"github.com/anchore/syft/internal/version"
"github.com/anchore/syft/syft/artifact"
"github.com/anchore/syft/syft"
"github.com/anchore/syft/syft/distro"
"github.com/anchore/syft/syft/event"
"github.com/anchore/syft/syft/format"
"github.com/anchore/syft/syft/pkg/cataloger"
"github.com/anchore/syft/syft/sbom"
"github.com/anchore/syft/syft/pkg"
"github.com/anchore/syft/syft/presenter/packages"
"github.com/anchore/syft/syft/source"
"github.com/pkg/profile"
"github.com/spf13/cobra"
@@ -44,13 +45,14 @@ const (
{{.appName}} {{.command}} oci-archive:path/to/yourimage.tar use a tarball from disk for OCI archives (from Skopeo or otherwise)
{{.appName}} {{.command}} oci-dir:path/to/yourimage read directly from a path on disk for OCI layout directories (from Skopeo or otherwise)
{{.appName}} {{.command}} dir:path/to/yourproject read directly from a path on disk (any directory)
{{.appName}} {{.command}} file:path/to/yourproject/file read directly from a path on disk (any single file)
{{.appName}} {{.command}} registry:yourrepo/yourimage:tag pull image directly from a registry (no container runtime required)
`
)
var (
packagesCmd = &cobra.Command{
packagesPresenterOpt format.Option
packagesArgs = cobra.MaximumNArgs(1)
packagesCmd = &cobra.Command{
Use: "packages [SOURCE]",
Short: "Generate a package SBOM",
Long: "Generate a packaged-based Software Bill Of Materials (SBOM) from container images and filesystems",
@@ -58,10 +60,25 @@ var (
"appName": internal.ApplicationName,
"command": "packages",
}),
Args: validateInputArgs,
Args: packagesArgs,
SilenceUsage: true,
SilenceErrors: true,
PreRunE: func(cmd *cobra.Command, args []string) (err error) {
PreRunE: func(cmd *cobra.Command, args []string) error {
if len(args) == 0 {
err := cmd.Help()
if err != nil {
return err
}
return fmt.Errorf("an image/directory argument is required")
}
// set the presenter
presenterOption := format.ParseOption(appConfig.Output)
if presenterOption == format.UnknownOption {
return fmt.Errorf("bad --output value '%s'", appConfig.Output)
}
packagesPresenterOpt = presenterOption
if appConfig.Dev.ProfileCPU && appConfig.Dev.ProfileMem {
return fmt.Errorf("cannot profile CPU and memory simultaneously")
}
@@ -87,23 +104,23 @@ func init() {
}
func setPackageFlags(flags *pflag.FlagSet) {
// Formatting & Input options //////////////////////////////////////////////
///////// Formatting & Input options //////////////////////////////////////////////
flags.StringP(
"scope", "s", cataloger.DefaultSearchConfig().Scope.String(),
"scope", "s", source.SquashedScope.String(),
fmt.Sprintf("selection of layers to catalog, options=%v", source.AllScopes))
flags.StringArrayP(
"output", "o", []string{string(format.TableOption)},
fmt.Sprintf("report output format, options=%v", format.AllOptions),
flags.StringP(
"output", "o", string(format.TableOption),
fmt.Sprintf("report output formatter, options=%v", format.AllOptions),
)
flags.StringP(
"file", "", "",
"file to write the default report output to (default is STDOUT)",
"file to write the report output to (default is STDOUT)",
)
// Upload options //////////////////////////////////////////////////////////
///////// Upload options //////////////////////////////////////////////////////////
flags.StringP(
"host", "H", "",
"the hostname or URL of the Anchore Enterprise instance to upload to",
@@ -124,11 +141,6 @@ func setPackageFlags(flags *pflag.FlagSet) {
"include dockerfile for upload to Anchore Enterprise",
)
flags.StringArrayP(
"exclude", "", nil,
"exclude paths from being scanned using a glob expression",
)
flags.Bool(
"overwrite-existing-image", false,
"overwrite an existing image during the upload to Anchore Enterprise",
@@ -141,7 +153,7 @@ func setPackageFlags(flags *pflag.FlagSet) {
}
func bindPackagesConfigOptions(flags *pflag.FlagSet) error {
// Formatting & Input options //////////////////////////////////////////////
///////// Formatting & Input options //////////////////////////////////////////////
if err := viper.BindPFlag("package.cataloger.scope", flags.Lookup("scope")); err != nil {
return err
@@ -155,11 +167,7 @@ func bindPackagesConfigOptions(flags *pflag.FlagSet) error {
return err
}
if err := viper.BindPFlag("exclude", flags.Lookup("exclude")); err != nil {
return err
}
// Upload options //////////////////////////////////////////////////////////
///////// Upload options //////////////////////////////////////////////////////////
if err := viper.BindPFlag("anchore.host", flags.Lookup("host")); err != nil {
return err
@@ -188,117 +196,71 @@ func bindPackagesConfigOptions(flags *pflag.FlagSet) error {
return nil
}
func validateInputArgs(cmd *cobra.Command, args []string) error {
if len(args) == 0 {
// in the case that no arguments are given we want to show the help text and return with a non-0 return code.
if err := cmd.Help(); err != nil {
return fmt.Errorf("unable to display help: %w", err)
}
return fmt.Errorf("an image/directory argument is required")
}
return cobra.MaximumNArgs(1)(cmd, args)
}
func packagesExec(_ *cobra.Command, args []string) error {
writer, err := makeWriter(appConfig.Output, appConfig.File)
// could be an image or a directory, with or without a scheme
userInput := args[0]
reporter, closer, err := reportWriter()
defer func() {
if err := closer(); err != nil {
log.Warnf("unable to write to report destination: %+v", err)
}
}()
if err != nil {
return err
}
defer func() {
if err := writer.Close(); err != nil {
log.Warnf("unable to write to report destination: %w", err)
}
}()
// could be an image or a directory, with or without a scheme
userInput := args[0]
return eventLoop(
packagesExecWorker(userInput, writer),
packagesExecWorker(userInput),
setupSignals(),
eventSubscription,
stereoscope.Cleanup,
ui.Select(isVerbose(), appConfig.Quiet)...,
ui.Select(appConfig.CliOptions.Verbosity > 0, appConfig.Quiet, reporter)...,
)
}
func isVerbose() (result bool) {
isPipedInput, err := internal.IsPipedInput()
if err != nil {
// since we can't tell if there was piped input we assume that there could be to disable the ETUI
log.Warnf("unable to determine if there is piped input: %+v", err)
return true
}
// verbosity should consider if there is piped input (in which case we should not show the ETUI)
return appConfig.CliOptions.Verbosity > 0 || isPipedInput
}
func packagesExecWorker(userInput string, writer sbom.Writer) <-chan error {
func packagesExecWorker(userInput string) <-chan error {
errs := make(chan error)
go func() {
defer close(errs)
tasks, err := tasks()
checkForApplicationUpdate()
src, cleanup, err := source.New(userInput, appConfig.Registry.ToOptions())
if err != nil {
errs <- err
errs <- fmt.Errorf("failed to determine image source: %w", err)
return
}
defer cleanup()
src, cleanup, err := source.New(userInput, appConfig.Registry.ToOptions(), appConfig.Exclusions)
catalog, d, err := syft.CatalogPackages(src, appConfig.Package.Cataloger.ScopeOpt)
if err != nil {
errs <- fmt.Errorf("failed to construct source from user input %q: %w", userInput, err)
errs <- fmt.Errorf("failed to catalog input: %w", err)
return
}
if cleanup != nil {
defer cleanup()
}
s := sbom.SBOM{
Source: src.Metadata,
Descriptor: sbom.Descriptor{
Name: internal.ApplicationName,
Version: version.FromBuild().Version,
Configuration: appConfig,
},
}
var relationships []<-chan artifact.Relationship
for _, task := range tasks {
c := make(chan artifact.Relationship)
relationships = append(relationships, c)
go runTask(task, &s.Artifacts, src, c, errs)
}
s.Relationships = append(s.Relationships, mergeRelationships(relationships...)...)
if appConfig.Anchore.Host != "" {
if err := runPackageSbomUpload(src, s); err != nil {
if err := runPackageSbomUpload(src, src.Metadata, catalog, d, appConfig.Package.Cataloger.ScopeOpt); err != nil {
errs <- err
return
}
}
bus.Publish(partybus.Event{
Type: event.Exit,
Value: func() error { return writer.Write(s) },
Type: event.PresenterReady,
Value: packages.Presenter(packagesPresenterOpt, packages.Config{
SourceMetadata: src.Metadata,
Catalog: catalog,
Distro: d,
Scope: appConfig.Package.Cataloger.ScopeOpt,
}),
})
}()
return errs
}
func mergeRelationships(cs ...<-chan artifact.Relationship) (relationships []artifact.Relationship) {
for _, c := range cs {
for n := range c {
relationships = append(relationships, n)
}
}
return relationships
}
func runPackageSbomUpload(src *source.Source, s sbom.SBOM) error {
func runPackageSbomUpload(src *source.Source, s source.Metadata, catalog *pkg.Catalog, d *distro.Distro, scope source.Scope) error {
log.Infof("uploading results to %s", appConfig.Anchore.Host)
if src.Metadata.Scheme != source.ImageScheme {
@@ -333,15 +295,17 @@ func runPackageSbomUpload(src *source.Source, s sbom.SBOM) error {
importCfg := anchore.ImportConfig{
ImageMetadata: src.Image.Metadata,
SBOM: s,
SourceMetadata: s,
Catalog: catalog,
Distro: d,
Dockerfile: dockerfileContents,
OverwriteExistingUpload: appConfig.Anchore.OverwriteExistingImage,
Scope: scope,
Timeout: appConfig.Anchore.ImportTimeout,
}
if err := c.Import(context.Background(), importCfg); err != nil {
return fmt.Errorf("failed to upload results to host=%s: %+v", appConfig.Anchore.Host, err)
}
return nil
}
+43 -53
View File
@@ -2,21 +2,16 @@ package cmd
import (
"fmt"
"os"
"sync"
"github.com/anchore/stereoscope"
"github.com/anchore/syft/internal"
"github.com/anchore/syft/internal/bus"
"github.com/anchore/syft/internal/formats/syftjson"
"github.com/anchore/syft/internal/log"
"github.com/anchore/syft/internal/output"
"github.com/anchore/syft/internal/presenter/poweruser"
"github.com/anchore/syft/internal/ui"
"github.com/anchore/syft/internal/version"
"github.com/anchore/syft/syft/artifact"
"github.com/anchore/syft/syft/event"
"github.com/anchore/syft/syft/sbom"
"github.com/anchore/syft/syft/source"
"github.com/gookit/color"
"github.com/pkg/profile"
"github.com/spf13/cobra"
"github.com/wagoodman/go-partybus"
@@ -24,8 +19,6 @@ import (
const powerUserExample = ` {{.appName}} {{.command}} <image>
DEPRECATED - THIS COMMAND WILL BE REMOVED in v1.0.0
Only image sources are supported (e.g. docker: , docker-archive: , oci: , etc.), the directory source (dir:) is not supported.
All behavior is controlled via application configuration and environment variables (see https://github.com/anchore/syft#configuration)
@@ -42,11 +35,19 @@ var powerUserCmd = &cobra.Command{
"appName": internal.ApplicationName,
"command": "power-user",
}),
Args: validateInputArgs,
Args: cobra.MaximumNArgs(1),
Hidden: true,
SilenceUsage: true,
SilenceErrors: true,
PreRunE: func(cmd *cobra.Command, args []string) error {
if len(args) == 0 {
err := cmd.Help()
if err != nil {
return err
}
return fmt.Errorf("an image/directory argument is required")
}
if appConfig.Dev.ProfileCPU && appConfig.Dev.ProfileMem {
return fmt.Errorf("cannot profile CPU and memory simultaneously")
}
@@ -74,80 +75,69 @@ func powerUserExec(_ *cobra.Command, args []string) error {
// could be an image or a directory, with or without a scheme
userInput := args[0]
writer, err := output.MakeWriter(output.WriterOption{
Format: syftjson.Format(),
Path: appConfig.File,
})
reporter, closer, err := reportWriter()
defer func() {
if err := closer(); err != nil {
log.Warnf("unable to write to report destination: %+v", err)
}
}()
if err != nil {
return err
}
defer func() {
if err := writer.Close(); err != nil {
log.Warnf("unable to write to report destination: %+v", err)
}
// inform user at end of run that command will be removed
deprecated := color.Style{color.Red, color.OpBold}.Sprint("DEPRECATED: This command will be removed in v1.0.0")
fmt.Fprintln(os.Stderr, deprecated)
}()
return eventLoop(
powerUserExecWorker(userInput, writer),
powerUserExecWorker(userInput),
setupSignals(),
eventSubscription,
stereoscope.Cleanup,
ui.Select(isVerbose(), appConfig.Quiet)...,
ui.Select(appConfig.CliOptions.Verbosity > 0, appConfig.Quiet, reporter)...,
)
}
func powerUserExecWorker(userInput string, writer sbom.Writer) <-chan error {
func powerUserExecWorker(userInput string) <-chan error {
errs := make(chan error)
go func() {
defer close(errs)
appConfig.Secrets.Cataloger.Enabled = true
appConfig.FileMetadata.Cataloger.Enabled = true
appConfig.FileContents.Cataloger.Enabled = true
appConfig.FileClassification.Cataloger.Enabled = true
tasks, err := tasks()
tasks, err := powerUserTasks()
if err != nil {
errs <- err
return
}
src, cleanup, err := source.New(userInput, appConfig.Registry.ToOptions(), appConfig.Exclusions)
checkForApplicationUpdate()
src, cleanup, err := source.New(userInput, appConfig.Registry.ToOptions())
if err != nil {
errs <- err
return
}
if cleanup != nil {
defer cleanup()
defer cleanup()
analysisResults := poweruser.JSONDocumentConfig{
SourceMetadata: src.Metadata,
ApplicationConfig: *appConfig,
}
s := sbom.SBOM{
Source: src.Metadata,
Descriptor: sbom.Descriptor{
Name: internal.ApplicationName,
Version: version.FromBuild().Version,
Configuration: appConfig,
},
}
var relationships []<-chan artifact.Relationship
wg := &sync.WaitGroup{}
for _, task := range tasks {
c := make(chan artifact.Relationship)
relationships = append(relationships, c)
go runTask(task, &s.Artifacts, src, c, errs)
wg.Add(1)
go func(task powerUserTask) {
defer wg.Done()
if err = task(&analysisResults, src); err != nil {
errs <- err
return
}
}(task)
}
s.Relationships = append(s.Relationships, mergeRelationships(relationships...)...)
wg.Wait()
bus.Publish(partybus.Event{
Type: event.Exit,
Value: func() error { return writer.Write(s) },
Type: event.PresenterReady,
Value: poweruser.NewJSONPresenter(analysisResults),
})
}()
return errs
}
+41 -57
View File
@@ -4,25 +4,24 @@ import (
"crypto"
"fmt"
"github.com/anchore/syft/internal/presenter/poweruser"
"github.com/anchore/syft/syft"
"github.com/anchore/syft/syft/artifact"
"github.com/anchore/syft/syft/file"
"github.com/anchore/syft/syft/sbom"
"github.com/anchore/syft/syft/source"
)
type task func(*sbom.Artifacts, *source.Source) ([]artifact.Relationship, error)
type powerUserTask func(*poweruser.JSONDocumentConfig, *source.Source) error
func tasks() ([]task, error) {
var tasks []task
func powerUserTasks() ([]powerUserTask, error) {
var tasks []powerUserTask
generators := []func() (task, error){
generateCatalogPackagesTask,
generateCatalogFileMetadataTask,
generateCatalogFileDigestsTask,
generateCatalogSecretsTask,
generateCatalogFileClassificationsTask,
generateCatalogContentsTask,
generators := []func() (powerUserTask, error){
catalogPackagesTask,
catalogFileMetadataTask,
catalogFileDigestsTask,
catalogSecretsTask,
catalogFileClassificationsTask,
catalogContentsTask,
}
for _, generator := range generators {
@@ -30,7 +29,6 @@ func tasks() ([]task, error) {
if err != nil {
return nil, err
}
if task != nil {
tasks = append(tasks, task)
}
@@ -39,51 +37,51 @@ func tasks() ([]task, error) {
return tasks, nil
}
func generateCatalogPackagesTask() (task, error) {
func catalogPackagesTask() (powerUserTask, error) {
if !appConfig.Package.Cataloger.Enabled {
return nil, nil
}
task := func(results *sbom.Artifacts, src *source.Source) ([]artifact.Relationship, error) {
packageCatalog, relationships, theDistro, err := syft.CatalogPackages(src, appConfig.Package.ToConfig())
task := func(results *poweruser.JSONDocumentConfig, src *source.Source) error {
packageCatalog, theDistro, err := syft.CatalogPackages(src, appConfig.Package.Cataloger.ScopeOpt)
if err != nil {
return nil, err
return err
}
results.PackageCatalog = packageCatalog
results.Distro = theDistro
return relationships, nil
return nil
}
return task, nil
}
func generateCatalogFileMetadataTask() (task, error) {
func catalogFileMetadataTask() (powerUserTask, error) {
if !appConfig.FileMetadata.Cataloger.Enabled {
return nil, nil
}
metadataCataloger := file.NewMetadataCataloger()
task := func(results *sbom.Artifacts, src *source.Source) ([]artifact.Relationship, error) {
task := func(results *poweruser.JSONDocumentConfig, src *source.Source) error {
resolver, err := src.FileResolver(appConfig.FileMetadata.Cataloger.ScopeOpt)
if err != nil {
return nil, err
return err
}
result, err := metadataCataloger.Catalog(resolver)
if err != nil {
return nil, err
return err
}
results.FileMetadata = result
return nil, nil
return nil
}
return task, nil
}
func generateCatalogFileDigestsTask() (task, error) {
func catalogFileDigestsTask() (powerUserTask, error) {
if !appConfig.FileMetadata.Cataloger.Enabled {
return nil, nil
}
@@ -112,24 +110,24 @@ func generateCatalogFileDigestsTask() (task, error) {
return nil, err
}
task := func(results *sbom.Artifacts, src *source.Source) ([]artifact.Relationship, error) {
task := func(results *poweruser.JSONDocumentConfig, src *source.Source) error {
resolver, err := src.FileResolver(appConfig.FileMetadata.Cataloger.ScopeOpt)
if err != nil {
return nil, err
return err
}
result, err := digestsCataloger.Catalog(resolver)
if err != nil {
return nil, err
return err
}
results.FileDigests = result
return nil, nil
return nil
}
return task, nil
}
func generateCatalogSecretsTask() (task, error) {
func catalogSecretsTask() (powerUserTask, error) {
if !appConfig.Secrets.Cataloger.Enabled {
return nil, nil
}
@@ -144,24 +142,24 @@ func generateCatalogSecretsTask() (task, error) {
return nil, err
}
task := func(results *sbom.Artifacts, src *source.Source) ([]artifact.Relationship, error) {
task := func(results *poweruser.JSONDocumentConfig, src *source.Source) error {
resolver, err := src.FileResolver(appConfig.Secrets.Cataloger.ScopeOpt)
if err != nil {
return nil, err
return err
}
result, err := secretsCataloger.Catalog(resolver)
if err != nil {
return nil, err
return err
}
results.Secrets = result
return nil, nil
return nil
}
return task, nil
}
func generateCatalogFileClassificationsTask() (task, error) {
func catalogFileClassificationsTask() (powerUserTask, error) {
if !appConfig.FileClassification.Cataloger.Enabled {
return nil, nil
}
@@ -172,24 +170,24 @@ func generateCatalogFileClassificationsTask() (task, error) {
return nil, err
}
task := func(results *sbom.Artifacts, src *source.Source) ([]artifact.Relationship, error) {
task := func(results *poweruser.JSONDocumentConfig, src *source.Source) error {
resolver, err := src.FileResolver(appConfig.FileClassification.Cataloger.ScopeOpt)
if err != nil {
return nil, err
return err
}
result, err := classifierCataloger.Catalog(resolver)
if err != nil {
return nil, err
return err
}
results.FileClassifications = result
return nil, nil
return nil
}
return task, nil
}
func generateCatalogContentsTask() (task, error) {
func catalogContentsTask() (powerUserTask, error) {
if !appConfig.FileContents.Cataloger.Enabled {
return nil, nil
}
@@ -199,33 +197,19 @@ func generateCatalogContentsTask() (task, error) {
return nil, err
}
task := func(results *sbom.Artifacts, src *source.Source) ([]artifact.Relationship, error) {
task := func(results *poweruser.JSONDocumentConfig, src *source.Source) error {
resolver, err := src.FileResolver(appConfig.FileContents.Cataloger.ScopeOpt)
if err != nil {
return nil, err
return err
}
result, err := contentsCataloger.Catalog(resolver)
if err != nil {
return nil, err
return err
}
results.FileContents = result
return nil, nil
return nil
}
return task, nil
}
func runTask(t task, a *sbom.Artifacts, src *source.Source, c chan<- artifact.Relationship, errs chan<- error) {
defer close(c)
relationships, err := t(a, src)
if err != nil {
errs <- err
return
}
for _, relationship := range relationships {
c <- relationship
}
}
+29
View File
@@ -0,0 +1,29 @@
package cmd
import (
"fmt"
"io"
"os"
"strings"
"github.com/anchore/syft/internal/log"
)
func reportWriter() (io.Writer, func() error, error) {
nop := func() error { return nil }
path := strings.TrimSpace(appConfig.File)
switch len(path) {
case 0:
return os.Stdout, nop, nil
default:
reportFile, err := os.OpenFile(path, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0644)
if err != nil {
return nil, nop, fmt.Errorf("unable to create report file: %w", err)
}
return reportFile, func() error {
log.Infof("report written to file=%q", path)
return reportFile.Close()
}, nil
}
}
+1 -6
View File
@@ -4,9 +4,7 @@ import (
"fmt"
"os"
"github.com/anchore/syft/internal"
"github.com/anchore/syft/internal/config"
"github.com/anchore/syft/internal/version"
"github.com/spf13/cobra"
"github.com/spf13/viper"
)
@@ -24,20 +22,17 @@ var rootCmd = &cobra.Command{
PreRunE: packagesCmd.PreRunE,
RunE: packagesCmd.RunE,
ValidArgsFunction: packagesCmd.ValidArgsFunction,
Version: version.FromBuild().Version,
}
func init() {
// set universal flags
rootCmd.PersistentFlags().StringVarP(&persistentOpts.ConfigPath, "config", "c", "", "application config file")
// setting the version template to just print out the string since we already have a templatized version string
rootCmd.SetVersionTemplate(fmt.Sprintf("%s {{.Version}}\n", internal.ApplicationName))
flag := "quiet"
rootCmd.PersistentFlags().BoolP(
flag, "q", false,
"suppress all logging output",
)
if err := viper.BindPFlag(flag, rootCmd.PersistentFlags().Lookup(flag)); err != nil {
fmt.Printf("unable to bind flag '%s': %+v", flag, err)
os.Exit(1)
+15 -24
View File
@@ -3,7 +3,6 @@ module github.com/anchore/syft
go 1.16
require (
github.com/CycloneDX/cyclonedx-go v0.4.0
github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d
github.com/adrg/xdg v0.2.1
github.com/alecthomas/jsonschema v0.0.0-20210301060011-54c507b6f074
@@ -12,47 +11,39 @@ require (
github.com/anchore/go-testutils v0.0.0-20200925183923-d5f45b0d3c04
github.com/anchore/go-version v1.2.2-0.20200701162849-18adb9c92b9b
github.com/anchore/packageurl-go v0.0.0-20210922164639-b3fa992ebd29
github.com/anchore/stereoscope v0.0.0-20211222141827-6e663afeef5d
// we are hinting brotli to latest due to warning when installing archiver v3:
// go: warning: github.com/andybalholm/brotli@v1.0.1: retracted by module author: occasional panics and data corruption
github.com/andybalholm/brotli v1.0.4 // indirect
github.com/anchore/stereoscope v0.0.0-20211005213828-538011008578
github.com/antihax/optional v1.0.0
github.com/bmatcuk/doublestar/v4 v4.0.2
github.com/docker/docker v20.10.11+incompatible
github.com/bmatcuk/doublestar/v2 v2.0.4
github.com/docker/docker v17.12.0-ce-rc1.0.20200309214505-aa6a9891b09c+incompatible
github.com/dustin/go-humanize v1.0.0
github.com/facebookincubator/nvdtools v0.1.4
github.com/go-test/deep v1.0.7
github.com/google/go-cmp v0.5.6
github.com/google/uuid v1.2.0
github.com/google/uuid v1.1.1
github.com/gookit/color v1.2.7
github.com/hashicorp/go-multierror v1.1.0
github.com/hashicorp/go-version v1.2.0
github.com/jinzhu/copier v0.3.2
github.com/mgutz/ansi v0.0.0-20200706080929-d51e80ef957d // indirect
github.com/mholt/archiver/v3 v3.5.1
github.com/mitchellh/go-homedir v1.1.0
github.com/mitchellh/hashstructure/v2 v2.0.2
github.com/mitchellh/mapstructure v1.4.1
github.com/mitchellh/mapstructure v1.3.1
github.com/olekukonko/tablewriter v0.0.4
github.com/pelletier/go-toml v1.9.3
github.com/pelletier/go-toml v1.8.0
github.com/pkg/profile v1.5.0
github.com/scylladb/go-set v1.0.2
github.com/sergi/go-diff v1.1.0
github.com/sirupsen/logrus v1.8.1
github.com/sirupsen/logrus v1.6.0
github.com/spdx/tools-golang v0.1.0
github.com/spf13/afero v1.6.0
github.com/spf13/cobra v1.2.1
github.com/spf13/afero v1.2.2
github.com/spf13/cobra v1.0.1-0.20200909172742-8a63648dd905
github.com/spf13/pflag v1.0.5
github.com/spf13/viper v1.8.1
github.com/spf13/viper v1.7.0
github.com/stretchr/testify v1.7.0
github.com/vifraa/gopom v0.1.0
github.com/wagoodman/go-partybus v0.0.0-20210627031916-db1f5573bbc5
github.com/wagoodman/go-progress v0.0.0-20200731105512-1020f39e6240
github.com/wagoodman/jotframe v0.0.0-20211129225309-56b0d0a4aebb
github.com/wagoodman/jotframe v0.0.0-20200730190914-3517092dd163
github.com/x-cray/logrus-prefixed-formatter v0.5.2
github.com/xeipuuv/gojsonschema v1.2.0
golang.org/x/mod v0.4.2
golang.org/x/net v0.0.0-20211111160137-58aab5ef257a
golang.org/x/term v0.0.0-20210615171337-6886f2dfbf5b
gopkg.in/yaml.v2 v2.4.0
golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9
golang.org/x/mod v0.3.0
golang.org/x/net v0.0.0-20210813160813-60bc85c4be6d
gopkg.in/yaml.v2 v2.3.0
)
+453 -680
View File
File diff suppressed because it is too large Load Diff
+4 -7
View File
@@ -1,5 +1,7 @@
#!/bin/sh
set -e
# Code generated by godownloader on 2020-08-10T20:55:46Z.
#
usage() {
this=$1
@@ -21,7 +23,7 @@ EOF
}
parse_args() {
# BINDIR is ./bin unless set be ENV
#BINDIR is ./bin unless set be ENV
# over-ridden by flag below
BINDIR=${BINDIR:-./bin}
@@ -65,10 +67,8 @@ execute() {
}
get_binaries() {
case "$PLATFORM" in
darwin/arm64) BINARIES="syft" ;;
darwin/amd64) BINARIES="syft" ;;
linux/amd64) BINARIES="syft" ;;
linux/arm64) BINARIES="syft" ;;
windows/amd64) BINARIES="syft" ;;
*)
log_crit "platform $PLATFORM is not supported. Make sure this script is up-to-date and file request at https://github.com/${PREFIX}/issues/new"
@@ -97,9 +97,6 @@ adjust_format() {
darwin) FORMAT=dmg ;;
windows) FORMAT=zip ;;
esac
case "${PLATFORM}" in
darwin/arm64) FORMAT=zip ;;
esac
true
}
adjust_os() {
@@ -247,7 +244,7 @@ extract_from_dmg() {
dmg_file=$1
mount_point="/Volumes/tmp-dmg"
hdiutil attach -quiet -nobrowse -mountpoint "${mount_point}" "${dmg_file}"
cp -fR "${mount_point}/." ./
cp -fR "${mount_point}/" ./
hdiutil detach -quiet -force "${mount_point}"
}
http_download_curl() {
+12 -6
View File
@@ -6,21 +6,27 @@ import (
"fmt"
"time"
"github.com/antihax/optional"
"github.com/anchore/client-go/pkg/external"
"github.com/anchore/stereoscope/pkg/image"
"github.com/anchore/syft/internal/bus"
"github.com/anchore/syft/syft/distro"
"github.com/anchore/syft/syft/event"
"github.com/anchore/syft/syft/sbom"
"github.com/antihax/optional"
"github.com/anchore/syft/syft/pkg"
"github.com/anchore/syft/syft/source"
"github.com/wagoodman/go-partybus"
"github.com/wagoodman/go-progress"
)
type ImportConfig struct {
ImageMetadata image.Metadata
SBOM sbom.SBOM
SourceMetadata source.Metadata
Catalog *pkg.Catalog
Distro *distro.Distro
Dockerfile []byte
OverwriteExistingUpload bool
Scope source.Scope
Timeout uint
}
@@ -68,19 +74,19 @@ func (c *Client) Import(ctx context.Context, cfg ImportConfig) error {
prog.N++
sessionID := startOperation.Uuid
packageDigest, err := importPackageSBOM(authedCtx, c.client.ImportsApi, sessionID, cfg.SBOM, stage)
packageDigest, err := importPackageSBOM(authedCtx, c.client.ImportsApi, sessionID, cfg.SourceMetadata, cfg.Catalog, cfg.Distro, cfg.Scope, stage)
if err != nil {
return fmt.Errorf("failed to import Package SBOM: %w", err)
}
prog.N++
manifestDigest, err := importManifest(authedCtx, c.client.ImportsApi, sessionID, cfg.SBOM.Source.ImageMetadata.RawManifest, stage)
manifestDigest, err := importManifest(authedCtx, c.client.ImportsApi, sessionID, cfg.ImageMetadata.RawManifest, stage)
if err != nil {
return fmt.Errorf("failed to import Manifest: %w", err)
}
prog.N++
configDigest, err := importConfig(authedCtx, c.client.ImportsApi, sessionID, cfg.SBOM.Source.ImageMetadata.RawConfig, stage)
configDigest, err := importConfig(authedCtx, c.client.ImportsApi, sessionID, cfg.ImageMetadata.RawConfig, stage)
if err != nil {
return fmt.Errorf("failed to import Config: %w", err)
}
+12 -9
View File
@@ -8,24 +8,27 @@ import (
"fmt"
"net/http"
"github.com/anchore/syft/syft/sbom"
"github.com/anchore/syft/internal/formats/syftjson"
"github.com/anchore/syft/internal/formats"
"github.com/anchore/syft/syft/format"
"github.com/wagoodman/go-progress"
"github.com/anchore/syft/syft/distro"
"github.com/anchore/syft/syft/source"
"github.com/anchore/client-go/pkg/external"
"github.com/anchore/syft/internal/log"
"github.com/anchore/syft/syft/pkg"
)
type packageSBOMImportAPI interface {
ImportImagePackages(context.Context, string, external.ImagePackageManifest) (external.ImageImportContentResponse, *http.Response, error)
}
func packageSbomModel(s sbom.SBOM) (*external.ImagePackageManifest, error) {
func packageSbomModel(s source.Metadata, catalog *pkg.Catalog, d *distro.Distro, _ source.Scope) (*external.ImagePackageManifest, error) {
var buf bytes.Buffer
err := syftjson.Format().Encode(&buf, s)
pres := formats.ByOption(format.JSONOption).Presenter(catalog, &s, d)
err := pres.Present(&buf)
if err != nil {
return nil, fmt.Errorf("unable to serialize results: %w", err)
}
@@ -33,17 +36,17 @@ func packageSbomModel(s sbom.SBOM) (*external.ImagePackageManifest, error) {
// the model is 1:1 the JSON output of today. As the schema changes, this will need to be converted into individual mappings.
var model external.ImagePackageManifest
if err = json.Unmarshal(buf.Bytes(), &model); err != nil {
return nil, fmt.Errorf("unable to convert JSON output to import model: %w", err)
return nil, fmt.Errorf("unable to convert JSON presenter output to import model: %w", err)
}
return &model, nil
}
func importPackageSBOM(ctx context.Context, api packageSBOMImportAPI, sessionID string, s sbom.SBOM, stage *progress.Stage) (string, error) {
func importPackageSBOM(ctx context.Context, api packageSBOMImportAPI, sessionID string, s source.Metadata, catalog *pkg.Catalog, d *distro.Distro, scope source.Scope, stage *progress.Stage) (string, error) {
log.Debug("importing package SBOM")
stage.Current = "package SBOM"
model, err := packageSbomModel(s)
model, err := packageSbomModel(s, catalog, d, scope)
if err != nil {
return "", fmt.Errorf("unable to create PackageSBOM model: %w", err)
}
+12 -42
View File
@@ -1,7 +1,6 @@
package anchore
import (
"bytes"
"context"
"encoding/json"
"fmt"
@@ -9,12 +8,11 @@ import (
"strings"
"testing"
"github.com/anchore/syft/syft/sbom"
"github.com/anchore/client-go/pkg/external"
"github.com/anchore/syft/internal/formats/syftjson"
syftjsonModel "github.com/anchore/syft/internal/formats/syftjson/model"
"github.com/anchore/syft/syft"
"github.com/anchore/syft/syft/distro"
"github.com/anchore/syft/syft/format"
"github.com/anchore/syft/syft/pkg"
"github.com/anchore/syft/syft/source"
"github.com/docker/docker/pkg/ioutils"
@@ -59,10 +57,8 @@ func TestPackageSbomToModel(t *testing.T) {
FoundBy: "foundBy",
Locations: []source.Location{
{
Coordinates: source.Coordinates{
RealPath: "path",
FileSystemID: "layerID",
},
RealPath: "path",
FileSystemID: "layerID",
},
},
Licenses: []string{"license"},
@@ -76,15 +72,7 @@ func TestPackageSbomToModel(t *testing.T) {
c := pkg.NewCatalog(p)
sbomResult := sbom.SBOM{
Artifacts: sbom.Artifacts{
PackageCatalog: c,
Distro: &d,
},
Source: m,
}
model, err := packageSbomModel(sbomResult)
model, err := packageSbomModel(m, c, &d, source.AllLayersScope)
if err != nil {
t.Fatalf("unable to generate model from source material: %+v", err)
}
@@ -96,22 +84,14 @@ func TestPackageSbomToModel(t *testing.T) {
t.Fatalf("unable to marshal model: %+v", err)
}
s := sbom.SBOM{
Artifacts: sbom.Artifacts{
PackageCatalog: c,
Distro: &d,
},
Source: m,
}
var buf bytes.Buffer
if err := syftjson.Format().Encode(&buf, s); err != nil {
by, err := syft.Encode(c, &m, &d, format.JSONOption)
if err != nil {
t.Fatalf("unable to get expected json: %+v", err)
}
// unmarshal expected result
var expectedDoc syftjsonModel.Document
if err := json.Unmarshal(buf.Bytes(), &expectedDoc); err != nil {
if err := json.Unmarshal(by, &expectedDoc); err != nil {
t.Fatalf("unable to parse json doc: %+v", err)
}
@@ -158,10 +138,8 @@ func TestPackageSbomImport(t *testing.T) {
FoundBy: "foundBy",
Locations: []source.Location{
{
Coordinates: source.Coordinates{
RealPath: "path",
FileSystemID: "layerID",
},
RealPath: "path",
FileSystemID: "layerID",
},
},
Licenses: []string{"license"},
@@ -208,15 +186,7 @@ func TestPackageSbomImport(t *testing.T) {
d, _ := distro.NewDistro(distro.CentOS, "8.0", "")
sbomResult := sbom.SBOM{
Artifacts: sbom.Artifacts{
PackageCatalog: catalog,
Distro: &d,
},
Source: m,
}
theModel, err := packageSbomModel(sbomResult)
theModel, err := packageSbomModel(m, catalog, &d, source.AllLayersScope)
if err != nil {
t.Fatalf("could not get sbom model: %+v", err)
}
@@ -255,7 +225,7 @@ func TestPackageSbomImport(t *testing.T) {
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
digest, err := importPackageSBOM(context.TODO(), test.api, sessionID, sbomResult, &progress.Stage{})
digest, err := importPackageSBOM(context.TODO(), test.api, sessionID, m, catalog, &d, source.AllLayersScope, &progress.Stage{})
// validate error handling
if err != nil && !test.expectsError {
+34 -64
View File
@@ -17,8 +17,6 @@ import (
var ErrApplicationConfigNotFound = fmt.Errorf("application config not found")
var catalogerEnabledDefault = false
type defaultValueLoader interface {
loadDefaultValues(*viper.Viper)
}
@@ -30,7 +28,7 @@ type parser interface {
// Application is the main syft application configuration.
type Application struct {
ConfigPath string `yaml:",omitempty" json:"configPath"` // the location where the application config was read from (either from -c or discovered while loading)
Output []string `yaml:"output" json:"output" mapstructure:"output"` // -o, the format to use for output
Output string `yaml:"output" json:"output" mapstructure:"output"` // -o, the Presenter hint string to use for report formatting
File string `yaml:"file" json:"file" mapstructure:"file"` // --file, the file to write report output to
Quiet bool `yaml:"quiet" json:"quiet" mapstructure:"quiet"` // -q, indicates to not show any status output to stderr (ETUI or logging UI)
CheckForAppUpdate bool `yaml:"check-for-app-update" json:"check-for-app-update" mapstructure:"check-for-app-update"` // whether to check for an application update on start up or not
@@ -38,18 +36,12 @@ type Application struct {
CliOptions CliOnlyOptions `yaml:"-" json:"-"` // all options only available through the CLI (not via env vars or config)
Dev development `yaml:"dev" json:"dev" mapstructure:"dev"`
Log logging `yaml:"log" json:"log" mapstructure:"log"` // all logging-related options
Package pkg `yaml:"package" json:"package" mapstructure:"package"`
Package packages `yaml:"package" json:"package" mapstructure:"package"`
FileMetadata FileMetadata `yaml:"file-metadata" json:"file-metadata" mapstructure:"file-metadata"`
FileClassification fileClassification `yaml:"file-classification" json:"file-classification" mapstructure:"file-classification"`
FileContents fileContents `yaml:"file-contents" json:"file-contents" mapstructure:"file-contents"`
Secrets secrets `yaml:"secrets" json:"secrets" mapstructure:"secrets"`
Registry registry `yaml:"registry" json:"registry" mapstructure:"registry"`
Exclusions []string `yaml:"exclude" json:"exclude" mapstructure:"exclude"`
}
// PowerUserCatalogerEnabledDefault switches all catalogers to be enabled when running power-user command
func PowerUserCatalogerEnabledDefault() {
catalogerEnabledDefault = true
}
func newApplicationConfig(v *viper.Viper, cliOpts CliOnlyOptions) *Application {
@@ -97,18 +89,42 @@ func (cfg Application) loadDefaultValues(v *viper.Viper) {
}
}
// build inflates simple config values into syft native objects (or other complex objects) after the config is fully read in.
func (cfg *Application) parseConfigValues() error {
// parse application config options
for _, optionFn := range []func() error{
cfg.parseUploadOptions,
cfg.parseLogLevelOption,
} {
if err := optionFn(); err != nil {
return err
if cfg.Quiet {
// TODO: this is bad: quiet option trumps all other logging options
// we should be able to quiet the console logging and leave file logging alone...
// ... this will be an enhancement for later
cfg.Log.LevelOpt = logrus.PanicLevel
} else {
if cfg.Log.Level != "" {
if cfg.CliOptions.Verbosity > 0 {
return fmt.Errorf("cannot explicitly set log level (cfg file or env var) and use -v flag together")
}
lvl, err := logrus.ParseLevel(strings.ToLower(cfg.Log.Level))
if err != nil {
return fmt.Errorf("bad log level configured (%q): %w", cfg.Log.Level, err)
}
// set the log level explicitly
cfg.Log.LevelOpt = lvl
} else {
// set the log level implicitly
switch v := cfg.CliOptions.Verbosity; {
case v == 1:
cfg.Log.LevelOpt = logrus.InfoLevel
case v >= 2:
cfg.Log.LevelOpt = logrus.DebugLevel
default:
cfg.Log.LevelOpt = logrus.WarnLevel
}
}
}
// parse nested config options
if cfg.Anchore.Host == "" && cfg.Anchore.Dockerfile != "" {
return fmt.Errorf("cannot provide dockerfile option without enabling upload")
}
// for each field in the configuration struct, see if the field implements the parser interface
// note: the app config is a pointer, so we need to grab the elements explicitly (to traverse the address)
value := reflect.ValueOf(cfg).Elem()
@@ -121,52 +137,6 @@ func (cfg *Application) parseConfigValues() error {
}
}
}
return nil
}
func (cfg *Application) parseUploadOptions() error {
if cfg.Anchore.Host == "" && cfg.Anchore.Dockerfile != "" {
return fmt.Errorf("cannot provide dockerfile option without enabling upload")
}
return nil
}
func (cfg *Application) parseLogLevelOption() error {
switch {
case cfg.Quiet:
// TODO: this is bad: quiet option trumps all other logging options (such as to a file on disk)
// we should be able to quiet the console logging and leave file logging alone...
// ... this will be an enhancement for later
cfg.Log.LevelOpt = logrus.PanicLevel
case cfg.Log.Level != "":
if cfg.CliOptions.Verbosity > 0 {
return fmt.Errorf("cannot explicitly set log level (cfg file or env var) and use -v flag together")
}
lvl, err := logrus.ParseLevel(strings.ToLower(cfg.Log.Level))
if err != nil {
return fmt.Errorf("bad log level configured (%q): %w", cfg.Log.Level, err)
}
cfg.Log.LevelOpt = lvl
if cfg.Log.LevelOpt >= logrus.InfoLevel {
cfg.CliOptions.Verbosity = 1
}
default:
switch v := cfg.CliOptions.Verbosity; {
case v == 1:
cfg.Log.LevelOpt = logrus.InfoLevel
case v >= 2:
cfg.Log.LevelOpt = logrus.DebugLevel
default:
cfg.Log.LevelOpt = logrus.ErrorLevel
}
}
if cfg.Log.Level == "" {
cfg.Log.Level = cfg.Log.LevelOpt.String()
}
return nil
}
-6
View File
@@ -3,8 +3,6 @@ package config
import (
"fmt"
"github.com/spf13/viper"
"github.com/anchore/syft/syft/source"
)
@@ -14,10 +12,6 @@ type catalogerOptions struct {
ScopeOpt source.Scope `yaml:"-" json:"-"`
}
func (cfg catalogerOptions) loadDefaultValues(v *viper.Viper) {
v.SetDefault("package.cataloger.enabled", true)
}
func (cfg *catalogerOptions) parseConfigValues() error {
scopeOption := source.ParseScope(cfg.Scope)
if scopeOption == source.UnknownScope {
+1 -1
View File
@@ -10,7 +10,7 @@ type fileClassification struct {
}
func (cfg fileClassification) loadDefaultValues(v *viper.Viper) {
v.SetDefault("file-classification.cataloger.enabled", catalogerEnabledDefault)
v.SetDefault("file-classification.cataloger.enabled", true)
v.SetDefault("file-classification.cataloger.scope", source.SquashedScope)
}
+1 -1
View File
@@ -13,7 +13,7 @@ type fileContents struct {
}
func (cfg fileContents) loadDefaultValues(v *viper.Viper) {
v.SetDefault("file-contents.cataloger.enabled", catalogerEnabledDefault)
v.SetDefault("file-contents.cataloger.enabled", true)
v.SetDefault("file-contents.cataloger.scope", source.SquashedScope)
v.SetDefault("file-contents.skip-files-above-size", 1*file.MB)
v.SetDefault("file-contents.globs", []string{})
+1 -1
View File
@@ -11,7 +11,7 @@ type FileMetadata struct {
}
func (cfg FileMetadata) loadDefaultValues(v *viper.Viper) {
v.SetDefault("file-metadata.cataloger.enabled", catalogerEnabledDefault)
v.SetDefault("file-metadata.cataloger.enabled", true)
v.SetDefault("file-metadata.cataloger.scope", source.SquashedScope)
v.SetDefault("file-metadata.digests", []string{"sha256"})
}
+15
View File
@@ -0,0 +1,15 @@
package config
import "github.com/spf13/viper"
type packages struct {
Cataloger catalogerOptions `yaml:"cataloger" json:"cataloger" mapstructure:"cataloger"`
}
func (cfg packages) loadDefaultValues(v *viper.Viper) {
v.SetDefault("package.cataloger.enabled", true)
}
func (cfg *packages) parseConfigValues() error {
return cfg.Cataloger.parseConfigValues()
}
-33
View File
@@ -1,33 +0,0 @@
package config
import (
"github.com/anchore/syft/syft/pkg/cataloger"
"github.com/spf13/viper"
)
type pkg struct {
Cataloger catalogerOptions `yaml:"cataloger" json:"cataloger" mapstructure:"cataloger"`
SearchUnindexedArchives bool `yaml:"search-unindexed-archives" json:"search-unindexed-archives" mapstructure:"search-unindexed-archives"`
SearchIndexedArchives bool `yaml:"search-indexed-archives" json:"search-indexed-archives" mapstructure:"search-indexed-archives"`
}
func (cfg pkg) loadDefaultValues(v *viper.Viper) {
cfg.Cataloger.loadDefaultValues(v)
c := cataloger.DefaultSearchConfig()
v.SetDefault("package.search-unindexed-archives", c.IncludeUnindexedArchives)
v.SetDefault("package.search-indexed-archives", c.IncludeIndexedArchives)
}
func (cfg *pkg) parseConfigValues() error {
return cfg.Cataloger.parseConfigValues()
}
func (cfg pkg) ToConfig() cataloger.Config {
return cataloger.Config{
Search: cataloger.SearchConfig{
IncludeIndexedArchives: cfg.SearchIndexedArchives,
IncludeUnindexedArchives: cfg.SearchUnindexedArchives,
Scope: cfg.Cataloger.ScopeOpt,
},
}
}
+1 -2
View File
@@ -2,9 +2,8 @@ package config
import (
"fmt"
"testing"
"github.com/anchore/stereoscope/pkg/image"
"testing"
"github.com/stretchr/testify/assert"
)
+1 -1
View File
@@ -15,7 +15,7 @@ type secrets struct {
}
func (cfg secrets) loadDefaultValues(v *viper.Viper) {
v.SetDefault("secrets.cataloger.enabled", catalogerEnabledDefault)
v.SetDefault("secrets.cataloger.enabled", true)
v.SetDefault("secrets.cataloger.scope", source.AllLayersScope)
v.SetDefault("secrets.reveal-values", false)
v.SetDefault("secrets.skip-files-above-size", 1*file.MB)
+2 -2
View File
@@ -4,7 +4,7 @@ const (
// ApplicationName is the non-capitalized name of the application (do not change this)
ApplicationName = "syft"
// JSONSchemaVersion is the current schema version output by the JSON encoder
// JSONSchemaVersion is the current schema version output by the JSON presenter
// This is roughly following the "SchemaVer" guidelines for versioning the JSON schema. Please see schema/json/README.md for details on how to increment.
JSONSchemaVersion = "2.0.2"
JSONSchemaVersion = "1.1.0"
)
-19
View File
@@ -1,19 +0,0 @@
package file
import (
"errors"
"fmt"
"io"
)
const perFileReadLimit = 2 * GB
// safeCopy limits the copy from the reader. This is useful when extracting files from archives to
// protect against decompression bomb attacks.
func safeCopy(writer io.Writer, reader io.Reader) error {
numBytes, err := io.Copy(writer, io.LimitReader(reader, perFileReadLimit))
if numBytes >= perFileReadLimit || errors.Is(err, io.EOF) {
return fmt.Errorf("zip read limit hit (potential decompression bomb attack)")
}
return nil
}
-64
View File
@@ -1,64 +0,0 @@
package file
import (
"fmt"
"io/ioutil"
"path/filepath"
"github.com/bmatcuk/doublestar/v4"
"github.com/mholt/archiver/v3"
)
// ExtractGlobsFromTarToUniqueTempFile extracts paths matching the given globs within the given archive to a temporary directory, returning file openers for each file extracted.
func ExtractGlobsFromTarToUniqueTempFile(archivePath, dir string, globs ...string) (map[string]Opener, error) {
results := make(map[string]Opener)
// don't allow for full traversal, only select traversal from given paths
if len(globs) == 0 {
return results, nil
}
visitor := func(file archiver.File) error {
defer file.Close()
// ignore directories
if file.FileInfo.IsDir() {
return nil
}
// ignore any filename that doesn't match the given globs...
if !matchesAnyGlob(file.Name(), globs...) {
return nil
}
// we have a file we want to extract....
tempfilePrefix := filepath.Base(filepath.Clean(file.Name())) + "-"
tempFile, err := ioutil.TempFile(dir, tempfilePrefix)
if err != nil {
return fmt.Errorf("unable to create temp file: %w", err)
}
// we shouldn't try and keep the tempfile open as the returned result may have several files, which takes up
// resources (leading to "too many open files"). Instead we'll return a file opener to the caller which
// provides a ReadCloser. It is up to the caller to handle closing the file explicitly.
defer tempFile.Close()
if err := safeCopy(tempFile, file.ReadCloser); err != nil {
return fmt.Errorf("unable to copy source=%q for tar=%q: %w", file.Name(), archivePath, err)
}
results[file.Name()] = Opener{path: tempFile.Name()}
return nil
}
return results, archiver.Walk(archivePath, visitor)
}
func matchesAnyGlob(name string, globs ...string) bool {
for _, glob := range globs {
if matches, err := doublestar.PathMatch(glob, name); err == nil && matches {
return true
}
}
return false
}
-3
View File
@@ -1,6 +1,3 @@
//go:build !windows
// +build !windows
package file
import (
+29 -3
View File
@@ -3,7 +3,9 @@ package file
import (
"archive/zip"
"bytes"
"errors"
"fmt"
"io"
"io/ioutil"
"os"
"path/filepath"
@@ -20,6 +22,8 @@ const (
GB
)
const perFileReadLimit = 2 * GB
type errZipSlipDetected struct {
Prefix string
JoinArgs []string
@@ -106,10 +110,21 @@ func ExtractFromZipToUniqueTempFile(archivePath, dir string, paths ...string) (m
return fmt.Errorf("unable to extract directories, only files: %s", file.Name)
}
if err := safeCopy(tempFile, zippedFile); err != nil {
// limit the zip reader on each file read to prevent decompression bomb attacks
numBytes, err := io.Copy(tempFile, io.LimitReader(zippedFile, perFileReadLimit))
if numBytes >= perFileReadLimit || errors.Is(err, io.EOF) {
return fmt.Errorf("zip read limit hit (potential decompression bomb attack)")
}
if err != nil {
return fmt.Errorf("unable to copy source=%q for zip=%q: %w", file.Name, archivePath, err)
}
// the file pointer is at the end due to the copy operation, reset back to the beginning
_, err = tempFile.Seek(0, io.SeekStart)
if err != nil {
return fmt.Errorf("unable to reset file pointer (%s): %w", tempFile.Name(), err)
}
results[file.Name] = Opener{path: tempFile.Name()}
return nil
@@ -138,7 +153,13 @@ func ContentsFromZip(archivePath string, paths ...string) (map[string]string, er
}
var buffer bytes.Buffer
if err := safeCopy(&buffer, zippedFile); err != nil {
// limit the zip reader on each file read to prevent decompression bomb attacks
numBytes, err := io.Copy(&buffer, io.LimitReader(zippedFile, perFileReadLimit))
if numBytes >= perFileReadLimit || errors.Is(err, io.EOF) {
return fmt.Errorf("zip read limit hit (potential decompression bomb attack)")
}
if err != nil {
return fmt.Errorf("unable to copy source=%q for zip=%q: %w", file.Name, archivePath, err)
}
@@ -207,7 +228,12 @@ func extractSingleFile(file *zip.File, expandedFilePath, archivePath string) err
return fmt.Errorf("unable to create dest file=%q from zip=%q: %w", expandedFilePath, archivePath, err)
}
if err := safeCopy(outputFile, zippedFile); err != nil {
// limit the zip reader on each file read to prevent decompression bomb attacks
numBytes, err := io.Copy(outputFile, io.LimitReader(zippedFile, perFileReadLimit))
if numBytes >= perFileReadLimit || errors.Is(err, io.EOF) {
return fmt.Errorf("zip read limit hit (potential decompression bomb attack)")
}
if err != nil {
return fmt.Errorf("unable to copy source=%q to dest=%q for zip=%q: %w", file.Name, outputFile.Name(), archivePath, err)
}
-3
View File
@@ -1,6 +1,3 @@
//go:build !windows
// +build !windows
package file
import (
-3
View File
@@ -1,6 +1,3 @@
//go:build !windows
// +build !windows
package file
import (
@@ -1,93 +0,0 @@
package cyclonedxhelpers
import (
"time"
"github.com/CycloneDX/cyclonedx-go"
"github.com/anchore/syft/internal"
"github.com/anchore/syft/internal/version"
"github.com/anchore/syft/syft/pkg"
"github.com/anchore/syft/syft/sbom"
"github.com/anchore/syft/syft/source"
"github.com/google/uuid"
)
func ToFormatModel(s sbom.SBOM) *cyclonedx.BOM {
cdxBOM := cyclonedx.NewBOM()
versionInfo := version.FromBuild()
// NOTE(jonasagx): cycloneDX requires URN uuids (URN returns the RFC 2141 URN form of uuid):
// https://github.com/CycloneDX/specification/blob/master/schema/bom-1.3-strict.schema.json#L36
// "pattern": "^urn:uuid:[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$"
cdxBOM.SerialNumber = uuid.New().URN()
cdxBOM.Metadata = toBomDescriptor(internal.ApplicationName, versionInfo.Version, s.Source)
packages := s.Artifacts.PackageCatalog.Sorted()
components := make([]cyclonedx.Component, len(packages))
for i, p := range packages {
components[i] = toComponent(p)
}
cdxBOM.Components = &components
return cdxBOM
}
// NewBomDescriptor returns a new BomDescriptor tailored for the current time and "syft" tool details.
func toBomDescriptor(name, version string, srcMetadata source.Metadata) *cyclonedx.Metadata {
return &cyclonedx.Metadata{
Timestamp: time.Now().Format(time.RFC3339),
Tools: &[]cyclonedx.Tool{
{
Vendor: "anchore",
Name: name,
Version: version,
},
},
Component: toBomDescriptorComponent(srcMetadata),
}
}
func toComponent(p pkg.Package) cyclonedx.Component {
return cyclonedx.Component{
Type: cyclonedx.ComponentTypeLibrary,
Name: p.Name,
Version: p.Version,
PackageURL: p.PURL,
Licenses: toLicenses(p.Licenses),
}
}
func toBomDescriptorComponent(srcMetadata source.Metadata) *cyclonedx.Component {
switch srcMetadata.Scheme {
case source.ImageScheme:
return &cyclonedx.Component{
Type: cyclonedx.ComponentTypeContainer,
Name: srcMetadata.ImageMetadata.UserInput,
Version: srcMetadata.ImageMetadata.ManifestDigest,
}
case source.DirectoryScheme, source.FileScheme:
return &cyclonedx.Component{
Type: cyclonedx.ComponentTypeFile,
Name: srcMetadata.Path,
}
}
return nil
}
func toLicenses(ls []string) *cyclonedx.Licenses {
if len(ls) == 0 {
return nil
}
lc := make(cyclonedx.Licenses, len(ls))
for i, licenseName := range ls {
lc[i] = cyclonedx.LicenseChoice{
License: &cyclonedx.License{
Name: licenseName,
},
}
}
return &lc
}
@@ -2,18 +2,13 @@ package spdxhelpers
import "github.com/anchore/syft/syft/pkg"
func Description(p pkg.Package) string {
if hasMetadata(p) {
switch metadata := p.Metadata.(type) {
case pkg.ApkMetadata:
return metadata.Description
case pkg.NpmPackageJSONMetadata:
return metadata.Description
}
func Description(p *pkg.Package) string {
switch metadata := p.Metadata.(type) {
case pkg.ApkMetadata:
return metadata.Description
case pkg.NpmPackageJSONMetadata:
return metadata.Description
default:
return ""
}
return ""
}
func hasMetadata(p pkg.Package) bool {
return p.Metadata != nil
}
@@ -50,7 +50,7 @@ func Test_Description(t *testing.T) {
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
assert.Equal(t, test.expected, Description(test.input))
assert.Equal(t, test.expected, Description(&test.input))
})
}
}
@@ -1,31 +0,0 @@
package spdxhelpers
import (
"fmt"
"path"
"strings"
"github.com/anchore/syft/syft/source"
)
func DocumentName(srcMetadata source.Metadata) (string, error) {
switch srcMetadata.Scheme {
case source.ImageScheme:
return cleanName(srcMetadata.ImageMetadata.UserInput), nil
case source.DirectoryScheme, source.FileScheme:
return cleanName(srcMetadata.Path), nil
}
return "", fmt.Errorf("unable to determine document name from scheme=%q", srcMetadata.Scheme)
}
func cleanName(name string) string {
// remove # according to specification
name = strings.ReplaceAll(name, "#", "-")
// remove : for url construction
name = strings.ReplaceAll(name, ":", "-")
// clean relative pathing
return path.Clean(name)
}
@@ -1,73 +0,0 @@
package spdxhelpers
import (
"fmt"
"strings"
"testing"
"github.com/stretchr/testify/require"
"github.com/anchore/syft/syft/source"
"github.com/scylladb/go-set/strset"
"github.com/stretchr/testify/assert"
)
func Test_DocumentName(t *testing.T) {
allSchemes := strset.New()
for _, s := range source.AllSchemes {
allSchemes.Add(string(s))
}
testedSchemes := strset.New()
tests := []struct {
name string
inputName string
srcMetadata source.Metadata
expected string
}{
{
name: "image",
inputName: "my-name",
srcMetadata: source.Metadata{
Scheme: source.ImageScheme,
ImageMetadata: source.ImageMetadata{
UserInput: "image-repo/name:tag",
ID: "id",
ManifestDigest: "digest",
},
},
expected: "image-repo/name-tag",
},
{
name: "directory",
inputName: "my-name",
srcMetadata: source.Metadata{
Scheme: source.DirectoryScheme,
Path: "some/path/to/place",
},
expected: "some/path/to/place",
},
{
name: "file",
inputName: "my-name",
srcMetadata: source.Metadata{
Scheme: source.FileScheme,
Path: "some/path/to/place",
},
expected: "some/path/to/place",
},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
actual, err := DocumentName(test.srcMetadata)
require.NoError(t, err)
assert.True(t, strings.HasPrefix(actual, test.expected), fmt.Sprintf("actual name %q", actual))
// track each scheme tested (passed or not)
testedSchemes.Add(string(test.srcMetadata.Scheme))
})
}
// assert all possible schemes were under test
assert.ElementsMatch(t, allSchemes.List(), testedSchemes.List(), "not all source.Schemes are under test")
}
@@ -1,45 +0,0 @@
package spdxhelpers
import (
"fmt"
"net/url"
"path"
"github.com/anchore/syft/internal"
"github.com/anchore/syft/syft/source"
"github.com/google/uuid"
)
func DocumentNameAndNamespace(srcMetadata source.Metadata) (string, string, error) {
name, err := DocumentName(srcMetadata)
if err != nil {
return "", "", err
}
return name, DocumentNamespace(name, srcMetadata), nil
}
func DocumentNamespace(name string, srcMetadata source.Metadata) string {
input := "unknown-source-type"
switch srcMetadata.Scheme {
case source.ImageScheme:
input = "image"
case source.DirectoryScheme:
input = "dir"
case source.FileScheme:
input = "file"
}
uniqueID := uuid.Must(uuid.NewRandom())
identifier := path.Join(input, uniqueID.String())
if name != "." {
identifier = path.Join(input, fmt.Sprintf("%s-%s", name, uniqueID.String()))
}
u := url.URL{
Scheme: "https",
Host: "anchore.com",
Path: path.Join(internal.ApplicationName, identifier),
}
return u.String()
}
@@ -1,71 +0,0 @@
package spdxhelpers
import (
"fmt"
"strings"
"testing"
"github.com/anchore/syft/syft/source"
"github.com/scylladb/go-set/strset"
"github.com/stretchr/testify/assert"
)
func Test_documentNamespace(t *testing.T) {
allSchemes := strset.New()
for _, s := range source.AllSchemes {
allSchemes.Add(string(s))
}
testedSchemes := strset.New()
tests := []struct {
name string
inputName string
srcMetadata source.Metadata
expected string
}{
{
name: "image",
inputName: "my-name",
srcMetadata: source.Metadata{
Scheme: source.ImageScheme,
ImageMetadata: source.ImageMetadata{
UserInput: "image-repo/name:tag",
ID: "id",
ManifestDigest: "digest",
},
},
expected: "https://anchore.com/syft/image/my-name-",
},
{
name: "directory",
inputName: "my-name",
srcMetadata: source.Metadata{
Scheme: source.DirectoryScheme,
Path: "some/path/to/place",
},
expected: "https://anchore.com/syft/dir/my-name-",
},
{
name: "file",
inputName: "my-name",
srcMetadata: source.Metadata{
Scheme: source.FileScheme,
Path: "some/path/to/place",
},
expected: "https://anchore.com/syft/file/my-name-",
},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
actual := DocumentNamespace(test.inputName, test.srcMetadata)
// note: since the namespace ends with a UUID we check the prefix
assert.True(t, strings.HasPrefix(actual, test.expected), fmt.Sprintf("actual namespace %q", actual))
// track each scheme tested (passed or not)
testedSchemes.Add(string(test.srcMetadata.Scheme))
})
}
// assert all possible schemes were under test
assert.ElementsMatch(t, allSchemes.List(), testedSchemes.List(), "not all source.Schemes are under test")
}
@@ -2,7 +2,7 @@ package spdxhelpers
import "github.com/anchore/syft/syft/pkg"
func DownloadLocation(p pkg.Package) string {
func DownloadLocation(p *pkg.Package) string {
// 3.7: Package Download Location
// Cardinality: mandatory, one
// NONE if there is no download location whatsoever.
@@ -11,13 +11,12 @@ func DownloadLocation(p pkg.Package) string {
// (ii) the SPDX file creator has made no attempt to determine this field; or
// (iii) the SPDX file creator has intentionally provided no information (no meaning should be implied by doing so).
if hasMetadata(p) {
switch metadata := p.Metadata.(type) {
case pkg.ApkMetadata:
return NoneIfEmpty(metadata.URL)
case pkg.NpmPackageJSONMetadata:
return NoneIfEmpty(metadata.URL)
}
switch metadata := p.Metadata.(type) {
case pkg.ApkMetadata:
return NoneIfEmpty(metadata.URL)
case pkg.NpmPackageJSONMetadata:
return NoneIfEmpty(metadata.URL)
default:
return "NOASSERTION"
}
return "NOASSERTION"
}
@@ -48,7 +48,7 @@ func Test_DownloadLocation(t *testing.T) {
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
assert.Equal(t, test.expected, DownloadLocation(test.input))
assert.Equal(t, test.expected, DownloadLocation(&test.input))
})
}
}
@@ -6,13 +6,12 @@ import (
"github.com/anchore/syft/syft/pkg"
)
func ExternalRefs(p pkg.Package) (externalRefs []model.ExternalRef) {
func ExternalRefs(p *pkg.Package) (externalRefs []model.ExternalRef) {
externalRefs = make([]model.ExternalRef, 0)
for _, c := range p.CPEs {
externalRefs = append(externalRefs, model.ExternalRef{
ReferenceCategory: model.SecurityReferenceCategory,
ReferenceLocator: pkg.CPEString(c),
ReferenceLocator: c.BindToFmtString(),
ReferenceType: model.Cpe23ExternalRefType,
})
}
@@ -3,13 +3,14 @@ package spdxhelpers
import (
"testing"
"github.com/anchore/syft/internal/formats/common/testutils"
"github.com/anchore/syft/internal/formats/spdx22json/model"
"github.com/anchore/syft/syft/pkg"
"github.com/stretchr/testify/assert"
)
func Test_ExternalRefs(t *testing.T) {
testCPE := pkg.MustCPE("cpe:2.3:a:name:name:3.2:*:*:*:*:*:*:*")
testCPE := testutils.MustCPE(pkg.NewCPE("cpe:2.3:a:name:name:3.2:*:*:*:*:*:*:*"))
tests := []struct {
name string
input pkg.Package
@@ -26,7 +27,7 @@ func Test_ExternalRefs(t *testing.T) {
expected: []model.ExternalRef{
{
ReferenceCategory: model.SecurityReferenceCategory,
ReferenceLocator: pkg.CPEString(testCPE),
ReferenceLocator: testCPE.BindToFmtString(),
ReferenceType: model.Cpe23ExternalRefType,
},
{
@@ -39,7 +40,7 @@ func Test_ExternalRefs(t *testing.T) {
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
assert.ElementsMatch(t, test.expected, ExternalRefs(test.input))
assert.ElementsMatch(t, test.expected, ExternalRefs(&test.input))
})
}
}
@@ -0,0 +1,47 @@
package spdxhelpers
import (
"crypto/sha256"
"fmt"
"path/filepath"
"github.com/anchore/syft/internal/formats/spdx22json/model"
"github.com/anchore/syft/syft/pkg"
)
func Files(packageSpdxID string, p *pkg.Package) (files []model.File, fileIDs []string, relationships []model.Relationship) {
files = make([]model.File, 0)
fileIDs = make([]string, 0)
relationships = make([]model.Relationship, 0)
pkgFileOwner, ok := p.Metadata.(pkg.FileOwner)
if !ok {
return files, fileIDs, relationships
}
for _, ownedFilePath := range pkgFileOwner.OwnedFiles() {
baseFileName := filepath.Base(ownedFilePath)
pathHash := sha256.Sum256([]byte(ownedFilePath))
fileSpdxID := model.ElementID(fmt.Sprintf("File-%s-%x", p.Name, pathHash)).String()
fileIDs = append(fileIDs, fileSpdxID)
files = append(files, model.File{
FileName: ownedFilePath,
Item: model.Item{
Element: model.Element{
SPDXID: fileSpdxID,
Name: baseFileName,
},
},
})
relationships = append(relationships, model.Relationship{
SpdxElementID: packageSpdxID,
RelationshipType: model.ContainsRelationship,
RelatedSpdxElement: fileSpdxID,
})
}
return files, fileIDs, relationships
}
@@ -2,14 +2,13 @@ package spdxhelpers
import "github.com/anchore/syft/syft/pkg"
func Homepage(p pkg.Package) string {
if hasMetadata(p) {
switch metadata := p.Metadata.(type) {
case pkg.GemMetadata:
return metadata.Homepage
case pkg.NpmPackageJSONMetadata:
return metadata.Homepage
}
func Homepage(p *pkg.Package) string {
switch metadata := p.Metadata.(type) {
case pkg.GemMetadata:
return metadata.Homepage
case pkg.NpmPackageJSONMetadata:
return metadata.Homepage
default:
return ""
}
return ""
}
@@ -50,7 +50,7 @@ func Test_Homepage(t *testing.T) {
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
assert.Equal(t, test.expected, Homepage(test.input))
assert.Equal(t, test.expected, Homepage(&test.input))
})
}
}
@@ -7,7 +7,7 @@ import (
"github.com/anchore/syft/syft/pkg"
)
func License(p pkg.Package) string {
func License(p *pkg.Package) string {
// source: https://spdx.github.io/spdx-spec/3-package-information/#313-concluded-license
// The options to populate this field are limited to:
// A valid SPDX License Expression as defined in Appendix IV;
@@ -67,7 +67,7 @@ func Test_License(t *testing.T) {
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
assert.Equal(t, test.expected, License(test.input))
assert.Equal(t, test.expected, License(&test.input))
})
}
}
@@ -108,7 +108,7 @@ func Test_Originator(t *testing.T) {
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
assert.Equal(t, test.expected, Originator(test.input))
assert.Equal(t, test.expected, Originator(&test.input))
})
}
}
@@ -6,32 +6,31 @@ import (
"github.com/anchore/syft/syft/pkg"
)
func Originator(p pkg.Package) string {
if hasMetadata(p) {
switch metadata := p.Metadata.(type) {
case pkg.ApkMetadata:
return metadata.Maintainer
case pkg.NpmPackageJSONMetadata:
return metadata.Author
case pkg.PythonPackageMetadata:
author := metadata.Author
if author == "" {
return metadata.AuthorEmail
}
if metadata.AuthorEmail != "" {
author += fmt.Sprintf(" <%s>", metadata.AuthorEmail)
}
return author
case pkg.GemMetadata:
if len(metadata.Authors) > 0 {
return metadata.Authors[0]
}
return ""
case pkg.RpmdbMetadata:
return metadata.Vendor
case pkg.DpkgMetadata:
return metadata.Maintainer
func Originator(p *pkg.Package) string {
switch metadata := p.Metadata.(type) {
case pkg.ApkMetadata:
return metadata.Maintainer
case pkg.NpmPackageJSONMetadata:
return metadata.Author
case pkg.PythonPackageMetadata:
author := metadata.Author
if author == "" {
return metadata.AuthorEmail
}
if metadata.AuthorEmail != "" {
author += fmt.Sprintf(" <%s>", metadata.AuthorEmail)
}
return author
case pkg.GemMetadata:
if len(metadata.Authors) > 0 {
return metadata.Authors[0]
}
return ""
case pkg.RpmdbMetadata:
return metadata.Vendor
case pkg.DpkgMetadata:
return metadata.Maintainer
default:
return ""
}
return ""
}
@@ -6,7 +6,7 @@ import (
"github.com/anchore/syft/syft/pkg"
)
func SourceInfo(p pkg.Package) string {
func SourceInfo(p *pkg.Package) string {
answer := ""
switch p.Type {
case pkg.RpmPkg:
@@ -27,8 +27,6 @@ func SourceInfo(p pkg.Package) string {
answer = "acquired package info from go module information"
case pkg.RustPkg:
answer = "acquired package info from rust cargo manifest"
case pkg.PhpComposerPkg:
answer = "acquired package info from PHP composer manifest"
default:
answer = "acquired package info from the following paths"
}
@@ -19,8 +19,14 @@ func Test_SourceInfo(t *testing.T) {
input: pkg.Package{
// note: no type given
Locations: []source.Location{
source.NewVirtualLocation("/a-place", "/b-place"),
source.NewVirtualLocation("/c-place", "/d-place"),
{
RealPath: "/a-place",
VirtualPath: "/b-place",
},
{
RealPath: "/c-place",
VirtualPath: "/d-place",
},
},
},
expected: []string{
@@ -118,14 +124,6 @@ func Test_SourceInfo(t *testing.T) {
"from rust cargo manifest",
},
},
{
input: pkg.Package{
Type: pkg.PhpComposerPkg,
},
expected: []string{
"from PHP composer manifest",
},
},
}
var pkgTypes []pkg.Type
for _, test := range tests {
@@ -133,7 +131,7 @@ func Test_SourceInfo(t *testing.T) {
if test.input.Type != "" {
pkgTypes = append(pkgTypes, test.input.Type)
}
actual := SourceInfo(test.input)
actual := SourceInfo(&test.input)
for _, expected := range test.expected {
assert.Contains(t, actual, expected)
}
+38 -101
View File
@@ -2,17 +2,15 @@ package testutils
import (
"bytes"
"strings"
"testing"
"github.com/anchore/syft/syft/presenter"
"github.com/anchore/go-testutils"
"github.com/anchore/stereoscope/pkg/filetree"
"github.com/anchore/stereoscope/pkg/image"
"github.com/anchore/stereoscope/pkg/imagetest"
"github.com/anchore/syft/syft/distro"
"github.com/anchore/syft/syft/format"
"github.com/anchore/syft/syft/pkg"
"github.com/anchore/syft/syft/sbom"
"github.com/anchore/syft/syft/source"
"github.com/sergi/go-diff/diffmatchpatch"
"github.com/stretchr/testify/assert"
@@ -20,19 +18,14 @@ import (
type redactor func(s []byte) []byte
type imageCfg struct {
fromSnapshot bool
}
type ImageOption func(cfg *imageCfg)
func FromSnapshot() ImageOption {
return func(cfg *imageCfg) {
cfg.fromSnapshot = true
func MustCPE(c pkg.CPE, e error) pkg.CPE {
if e != nil {
panic(e)
}
return c
}
func AssertEncoderAgainstGoldenImageSnapshot(t *testing.T, format format.Format, sbom sbom.SBOM, testImage string, updateSnapshot bool, redactors ...redactor) {
func AssertPresenterAgainstGoldenImageSnapshot(t *testing.T, pres presenter.Presenter, testImage string, updateSnapshot bool, redactors ...redactor) {
var buffer bytes.Buffer
// grab the latest image contents and persist
@@ -40,11 +33,11 @@ func AssertEncoderAgainstGoldenImageSnapshot(t *testing.T, format format.Format,
imagetest.UpdateGoldenFixtureImage(t, testImage)
}
err := format.Encode(&buffer, sbom)
err := pres.Present(&buffer)
assert.NoError(t, err)
actual := buffer.Bytes()
// replace the expected snapshot contents with the current encoder contents
// replace the expected snapshot contents with the current presenter contents
if updateSnapshot {
testutils.UpdateGoldenFileContents(t, actual)
}
@@ -52,7 +45,6 @@ func AssertEncoderAgainstGoldenImageSnapshot(t *testing.T, format format.Format,
var expected = testutils.GetGoldenFileContents(t)
// remove dynamic values, which should be tested independently
redactors = append(redactors, carriageRedactor)
for _, r := range redactors {
actual = r(actual)
expected = r(expected)
@@ -66,14 +58,14 @@ func AssertEncoderAgainstGoldenImageSnapshot(t *testing.T, format format.Format,
}
}
func AssertEncoderAgainstGoldenSnapshot(t *testing.T, format format.Format, sbom sbom.SBOM, updateSnapshot bool, redactors ...redactor) {
func AssertPresenterAgainstGoldenSnapshot(t *testing.T, pres presenter.Presenter, updateSnapshot bool, redactors ...redactor) {
var buffer bytes.Buffer
err := format.Encode(&buffer, sbom)
err := pres.Present(&buffer)
assert.NoError(t, err)
actual := buffer.Bytes()
// replace the expected snapshot contents with the current encoder contents
// replace the expected snapshot contents with the current presenter contents
if updateSnapshot {
testutils.UpdateGoldenFileContents(t, actual)
}
@@ -81,7 +73,6 @@ func AssertEncoderAgainstGoldenSnapshot(t *testing.T, format format.Format, sbom
var expected = testutils.GetGoldenFileContents(t)
// remove dynamic values, which should be tested independently
redactors = append(redactors, carriageRedactor)
for _, r := range redactors {
actual = r(actual)
expected = r(expected)
@@ -90,68 +81,21 @@ func AssertEncoderAgainstGoldenSnapshot(t *testing.T, format format.Format, sbom
if !bytes.Equal(expected, actual) {
dmp := diffmatchpatch.New()
diffs := dmp.DiffMain(string(expected), string(actual), true)
t.Logf("len: %d\nexpected: %s", len(expected), expected)
t.Logf("len: %d\nactual: %s", len(actual), actual)
t.Errorf("mismatched output:\n%s", dmp.DiffPrettyText(diffs))
}
}
func ImageInput(t testing.TB, testImage string, options ...ImageOption) sbom.SBOM {
func ImageInput(t testing.TB, testImage string) (*pkg.Catalog, source.Metadata, *distro.Distro) {
t.Helper()
catalog := pkg.NewCatalog()
var cfg imageCfg
var img *image.Image
for _, opt := range options {
opt(&cfg)
}
img := imagetest.GetGoldenFixtureImage(t, testImage)
switch cfg.fromSnapshot {
case true:
img = imagetest.GetGoldenFixtureImage(t, testImage)
default:
img = imagetest.GetFixtureImage(t, "docker-archive", testImage)
}
populateImageCatalog(catalog, img)
// this is a hard coded value that is not given by the fixture helper and must be provided manually
img.Metadata.ManifestDigest = "sha256:2731251dc34951c0e50fcc643b4c5f74922dad1a5d98f302b504cf46cd5d9368"
src, err := source.NewFromImage(img, "user-image-input")
assert.NoError(t, err)
dist, err := distro.NewDistro(distro.Debian, "1.2.3", "like!")
assert.NoError(t, err)
return sbom.SBOM{
Artifacts: sbom.Artifacts{
PackageCatalog: catalog,
Distro: &dist,
},
Source: src.Metadata,
Descriptor: sbom.Descriptor{
Name: "syft",
Version: "v0.42.0-bogus",
// the application configuration should be persisted here, however, we do not want to import
// the application configuration in this package (it's reserved only for ingestion by the cmd package)
Configuration: map[string]string{
"config-key": "config-value",
},
},
}
}
func carriageRedactor(s []byte) []byte {
msg := strings.ReplaceAll(string(s), "\r\n", "\n")
return []byte(msg)
}
func populateImageCatalog(catalog *pkg.Catalog, img *image.Image) {
_, ref1, _ := img.SquashedTree().File("/somefile-1.txt", filetree.FollowBasenameLinks)
_, ref2, _ := img.SquashedTree().File("/somefile-2.txt", filetree.FollowBasenameLinks)
// populate catalog with test data
catalog.Add(pkg.Package{
ID: "package-1-id",
Name: "package-1",
Version: "1.0.1",
Locations: []source.Location{
@@ -168,10 +112,11 @@ func populateImageCatalog(catalog *pkg.Catalog, img *image.Image) {
},
PURL: "a-purl-1",
CPEs: []pkg.CPE{
pkg.MustCPE("cpe:2.3:*:some:package:1:*:*:*:*:*:*:*"),
MustCPE(pkg.NewCPE("cpe:2.3:*:some:package:1:*:*:*:*:*:*:*")),
},
})
catalog.Add(pkg.Package{
ID: "package-2-id",
Name: "package-2",
Version: "2.0.1",
Locations: []source.Location{
@@ -186,49 +131,34 @@ func populateImageCatalog(catalog *pkg.Catalog, img *image.Image) {
},
PURL: "a-purl-2",
CPEs: []pkg.CPE{
pkg.MustCPE("cpe:2.3:*:some:package:2:*:*:*:*:*:*:*"),
MustCPE(pkg.NewCPE("cpe:2.3:*:some:package:2:*:*:*:*:*:*:*")),
},
})
}
func DirectoryInput(t testing.TB) sbom.SBOM {
catalog := newDirectoryCatalog()
// this is a hard coded value that is not given by the fixture helper and must be provided manually
img.Metadata.ManifestDigest = "sha256:2731251dc34951c0e50fcc643b4c5f74922dad1a5d98f302b504cf46cd5d9368"
src, err := source.NewFromImage(img, "user-image-input")
assert.NoError(t, err)
dist, err := distro.NewDistro(distro.Debian, "1.2.3", "like!")
assert.NoError(t, err)
src, err := source.NewFromDirectory("/some/path")
assert.NoError(t, err)
return sbom.SBOM{
Artifacts: sbom.Artifacts{
PackageCatalog: catalog,
Distro: &dist,
},
Source: src.Metadata,
Descriptor: sbom.Descriptor{
Name: "syft",
Version: "v0.42.0-bogus",
// the application configuration should be persisted here, however, we do not want to import
// the application configuration in this package (it's reserved only for ingestion by the cmd package)
Configuration: map[string]string{
"config-key": "config-value",
},
},
}
return catalog, src.Metadata, &dist
}
func newDirectoryCatalog() *pkg.Catalog {
func DirectoryInput(t testing.TB) (*pkg.Catalog, source.Metadata, *distro.Distro) {
catalog := pkg.NewCatalog()
// populate catalog with test data
catalog.Add(pkg.Package{
ID: "package-1-id",
Name: "package-1",
Version: "1.0.1",
Type: pkg.PythonPkg,
FoundBy: "the-cataloger-1",
Locations: []source.Location{
source.NewLocation("/some/path/pkg1"),
{RealPath: "/some/path/pkg1"},
},
Language: pkg.Python,
MetadataType: pkg.PythonPackageMetadataType,
@@ -244,16 +174,17 @@ func newDirectoryCatalog() *pkg.Catalog {
},
PURL: "a-purl-2",
CPEs: []pkg.CPE{
pkg.MustCPE("cpe:2.3:*:some:package:2:*:*:*:*:*:*:*"),
MustCPE(pkg.NewCPE("cpe:2.3:*:some:package:2:*:*:*:*:*:*:*")),
},
})
catalog.Add(pkg.Package{
ID: "package-2-id",
Name: "package-2",
Version: "2.0.1",
Type: pkg.DebPkg,
FoundBy: "the-cataloger-2",
Locations: []source.Location{
source.NewLocation("/some/path/pkg1"),
{RealPath: "/some/path/pkg1"},
},
MetadataType: pkg.DpkgMetadataType,
Metadata: pkg.DpkgMetadata{
@@ -262,9 +193,15 @@ func newDirectoryCatalog() *pkg.Catalog {
},
PURL: "a-purl-2",
CPEs: []pkg.CPE{
pkg.MustCPE("cpe:2.3:*:some:package:2:*:*:*:*:*:*:*"),
MustCPE(pkg.NewCPE("cpe:2.3:*:some:package:2:*:*:*:*:*:*:*")),
},
})
return catalog
dist, err := distro.NewDistro(distro.Debian, "1.2.3", "like!")
assert.NoError(t, err)
src, err := source.NewFromDirectory("/some/path")
assert.NoError(t, err)
return catalog, src.Metadata, &dist
}
@@ -1,18 +0,0 @@
package cyclonedx13json
import (
"io"
"github.com/CycloneDX/cyclonedx-go"
"github.com/anchore/syft/internal/formats/common/cyclonedxhelpers"
"github.com/anchore/syft/syft/sbom"
)
func encoder(output io.Writer, s sbom.SBOM) error {
bom := cyclonedxhelpers.ToFormatModel(s)
enc := cyclonedx.NewBOMEncoder(output, cyclonedx.BOMFileFormatJSON)
enc.SetPretty(true)
err := enc.Encode(bom)
return err
}
@@ -1,41 +0,0 @@
package cyclonedx13json
import (
"flag"
"regexp"
"testing"
"github.com/anchore/syft/internal/formats/common/testutils"
)
var updateCycloneDx = flag.Bool("update-cyclonedx", false, "update the *.golden files for cyclone-dx encoders")
func TestCycloneDxDirectoryEncoder(t *testing.T) {
testutils.AssertEncoderAgainstGoldenSnapshot(t,
Format(),
testutils.DirectoryInput(t),
*updateCycloneDx,
cycloneDxRedactor,
)
}
func TestCycloneDxImageEncoder(t *testing.T) {
testImage := "image-simple"
testutils.AssertEncoderAgainstGoldenImageSnapshot(t,
Format(),
testutils.ImageInput(t, testImage),
testImage,
*updateCycloneDx,
cycloneDxRedactor,
)
}
func cycloneDxRedactor(s []byte) []byte {
serialPattern := regexp.MustCompile(`urn:uuid:[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}`)
rfc3339Pattern := regexp.MustCompile(`([0-9]+)-(0[1-9]|1[012])-(0[1-9]|[12][0-9]|3[01])[Tt]([01][0-9]|2[0-3]):([0-5][0-9]):([0-5][0-9]|60)(\.[0-9]+)?(([Zz])|([\+|\-]([01][0-9]|2[0-3]):[0-5][0-9]))`)
for _, pattern := range []*regexp.Regexp{serialPattern, rfc3339Pattern} {
s = pattern.ReplaceAll(s, []byte("redacted"))
}
return s
}
@@ -1,12 +0,0 @@
package cyclonedx13json
import "github.com/anchore/syft/syft/format"
func Format() format.Format {
return format.NewFormat(
format.CycloneDxJSONOption,
encoder,
nil,
nil,
)
}
@@ -1,42 +0,0 @@
{
"bomFormat": "CycloneDX",
"specVersion": "1.3",
"serialNumber": "urn:uuid:a81dc685-cf22-48e0-bda5-65ea1a8bca5b",
"version": 1,
"metadata": {
"timestamp": "2021-12-03T13:17:26-08:00",
"tools": [
{
"vendor": "anchore",
"name": "syft",
"version": "[not provided]"
}
],
"component": {
"type": "file",
"name": "/some/path",
"version": ""
}
},
"components": [
{
"type": "library",
"name": "package-1",
"version": "1.0.1",
"licenses": [
{
"license": {
"name": "MIT"
}
}
],
"purl": "a-purl-2"
},
{
"type": "library",
"name": "package-2",
"version": "2.0.1",
"purl": "a-purl-2"
}
]
}
@@ -1,42 +0,0 @@
{
"bomFormat": "CycloneDX",
"specVersion": "1.3",
"serialNumber": "urn:uuid:2156ac1f-c838-4e93-8dc5-a3874ffeb967",
"version": 1,
"metadata": {
"timestamp": "2021-12-03T13:17:26-08:00",
"tools": [
{
"vendor": "anchore",
"name": "syft",
"version": "[not provided]"
}
],
"component": {
"type": "container",
"name": "user-image-input",
"version": "sha256:2731251dc34951c0e50fcc643b4c5f74922dad1a5d98f302b504cf46cd5d9368"
}
},
"components": [
{
"type": "library",
"name": "package-1",
"version": "1.0.1",
"licenses": [
{
"license": {
"name": "MIT"
}
}
],
"purl": "a-purl-1"
},
{
"type": "library",
"name": "package-2",
"version": "2.0.1",
"purl": "a-purl-2"
}
]
}
@@ -1,18 +0,0 @@
package cyclonedx13xml
import (
"io"
"github.com/CycloneDX/cyclonedx-go"
"github.com/anchore/syft/internal/formats/common/cyclonedxhelpers"
"github.com/anchore/syft/syft/sbom"
)
func encoder(output io.Writer, s sbom.SBOM) error {
bom := cyclonedxhelpers.ToFormatModel(s)
enc := cyclonedx.NewBOMEncoder(output, cyclonedx.BOMFileFormatXML)
enc.SetPretty(true)
err := enc.Encode(bom)
return err
}
-12
View File
@@ -1,12 +0,0 @@
package cyclonedx13xml
import "github.com/anchore/syft/syft/format"
func Format() format.Format {
return format.NewFormat(
format.CycloneDxXMLOption,
encoder,
nil,
nil,
)
}
@@ -1,4 +0,0 @@
# Note: changes to this file will result in updating several test values. Consider making a new image fixture instead of editing this one.
FROM scratch
ADD file-1.txt /somefile-1.txt
ADD file-2.txt /somefile-2.txt
@@ -1 +0,0 @@
this file has contents
@@ -1 +0,0 @@
file-2 contents!
+3 -16
View File
@@ -1,37 +1,24 @@
package formats
import (
"bytes"
"github.com/anchore/syft/internal/formats/cyclonedx13json"
"github.com/anchore/syft/internal/formats/cyclonedx13xml"
"github.com/anchore/syft/internal/formats/spdx22json"
"github.com/anchore/syft/internal/formats/spdx22tagvalue"
"github.com/anchore/syft/internal/formats/syftjson"
"github.com/anchore/syft/internal/formats/table"
"github.com/anchore/syft/internal/formats/text"
"github.com/anchore/syft/syft/format"
)
// TODO: eventually this is the source of truth for all formatters
func All() []format.Format {
return []format.Format{
syftjson.Format(),
table.Format(),
cyclonedx13xml.Format(),
cyclonedx13json.Format(),
spdx22json.Format(),
spdx22tagvalue.Format(),
text.Format(),
syftjson.Format(),
}
}
func Identify(by []byte) (*format.Format, error) {
for _, f := range All() {
if err := f.Validate(bytes.NewReader(by)); err != nil {
continue
if f.Detect(by) {
return &f, nil
}
return &f, nil
}
return nil, nil
}
+5
View File
@@ -10,6 +10,7 @@ import (
)
func TestIdentify(t *testing.T) {
tests := []struct {
fixture string
expected format.Option
@@ -18,6 +19,10 @@ func TestIdentify(t *testing.T) {
fixture: "test-fixtures/alpine-syft.json",
expected: format.JSONOption,
},
{
fixture: "test-fixtures/alpine-syft-spdx.json",
expected: format.SPDXJSONOption,
},
}
for _, test := range tests {
t.Run(test.fixture, func(t *testing.T) {
+24
View File
@@ -0,0 +1,24 @@
package spdx22json
import (
"encoding/json"
"fmt"
"io"
"github.com/anchore/syft/internal/formats/spdx22json/model"
"github.com/anchore/syft/syft/distro"
"github.com/anchore/syft/syft/pkg"
"github.com/anchore/syft/syft/source"
)
func decoder(reader io.Reader) (*pkg.Catalog, *source.Metadata, *distro.Distro, error) {
dec := json.NewDecoder(reader)
var doc model.Document
err := dec.Decode(&doc)
if err != nil {
return nil, nil, nil, fmt.Errorf("unable to decode spdx-json: %w", err)
}
return toSyftModel(doc)
}
@@ -0,0 +1,50 @@
package spdx22json
import (
"bytes"
"strings"
"testing"
"github.com/anchore/syft/internal/formats/common/testutils"
"github.com/go-test/deep"
"github.com/stretchr/testify/assert"
)
func TestEncodeDecodeCycle(t *testing.T) {
testImage := "image-simple"
originalCatalog, originalMetadata, _ := testutils.ImageInput(t, testImage)
var buf bytes.Buffer
assert.NoError(t, encoder(&buf, originalCatalog, &originalMetadata, nil))
actualCatalog, actualMetadata, _, err := decoder(bytes.NewReader(buf.Bytes()))
assert.NoError(t, err)
for _, d := range deep.Equal(originalMetadata, *actualMetadata) {
t.Errorf("metadata difference: %+v", d)
}
actualPackages := actualCatalog.Sorted()
for idx, p := range originalCatalog.Sorted() {
if !assert.Equal(t, p.Name, actualPackages[idx].Name) {
t.Errorf("different package at idx=%d: %s vs %s", idx, p.Name, actualPackages[idx].Name)
continue
}
// ids will never be equal
p.ID = ""
actualPackages[idx].ID = ""
for _, d := range deep.Equal(*p, *actualPackages[idx]) {
if strings.Contains(d, ".VirtualPath: ") {
// location.Virtual path is not exposed in the json output
continue
}
if strings.HasSuffix(d, "<nil slice> != []") {
// semantically the same
continue
}
t.Errorf("package difference (%s): %+v", p.Name, d)
}
}
}
+9 -7
View File
@@ -4,19 +4,21 @@ import (
"encoding/json"
"io"
"github.com/anchore/syft/syft/sbom"
"github.com/anchore/syft/syft/distro"
"github.com/anchore/syft/syft/pkg"
"github.com/anchore/syft/syft/source"
)
func encoder(output io.Writer, s sbom.SBOM) error {
doc, err := toFormatModel(s)
if err != nil {
return err
}
const anchoreNamespace = "https://anchore.com/syft"
func encoder(output io.Writer, catalog *pkg.Catalog, srcMetadata *source.Metadata, d *distro.Distro) error {
doc := toFormatModel(catalog, srcMetadata, d)
enc := json.NewEncoder(output)
// prevent > and < from being escaped in the payload
enc.SetEscapeHTML(false)
enc.SetIndent("", " ")
return enc.Encode(doc)
return enc.Encode(&doc)
}
+10 -9
View File
@@ -6,24 +6,25 @@ import (
"testing"
"github.com/anchore/syft/internal/formats/common/testutils"
"github.com/anchore/syft/syft/format"
)
var updateSpdxJson = flag.Bool("update-spdx-json", false, "update the *.golden files for spdx-json encoders")
var updateSpdxJson = flag.Bool("update-spdx-json", false, "update the *.golden files for spdx-json presenters")
func TestSPDXJSONDirectoryEncoder(t *testing.T) {
testutils.AssertEncoderAgainstGoldenSnapshot(t,
Format(),
testutils.DirectoryInput(t),
func TestSPDXJSONDirectoryPresenter(t *testing.T) {
catalog, metadata, distro := testutils.DirectoryInput(t)
testutils.AssertPresenterAgainstGoldenSnapshot(t,
format.NewPresenter(encoder, catalog, &metadata, distro),
*updateSpdxJson,
spdxJsonRedactor,
)
}
func TestSPDXJSONImageEncoder(t *testing.T) {
func TestSPDXJSONImagePresenter(t *testing.T) {
testImage := "image-simple"
testutils.AssertEncoderAgainstGoldenImageSnapshot(t,
Format(),
testutils.ImageInput(t, testImage, testutils.FromSnapshot()),
catalog, metadata, distro := testutils.ImageInput(t, testImage)
testutils.AssertPresenterAgainstGoldenImageSnapshot(t,
format.NewPresenter(encoder, catalog, &metadata, distro),
testImage,
*updateSpdxJson,
spdxJsonRedactor,
+2 -3
View File
@@ -2,12 +2,11 @@ package spdx22json
import "github.com/anchore/syft/syft/format"
// note: this format is LOSSY relative to the syftjson formation, which means that decoding and validation is not supported at this time
func Format() format.Format {
return format.NewFormat(
format.SPDXJSONOption,
encoder,
nil,
nil,
decoder,
validator,
)
}
@@ -10,7 +10,7 @@ type CreationInfo struct {
Created time.Time `json:"created"`
// Identify who (or what, in the case of a tool) created the SPDX file. If the SPDX file was created by an
// individual, indicate the person's name. If the SPDX file was created on behalf of a company or organization,
// indicate the entity name. If the SPDX file was created using a software tool, indicate the name and version
//indicate the entity name. If the SPDX file was created using a software tool, indicate the name and version
// for that tool. If multiple participants or tools were involved, use multiple instances of this field. Person
// name or organization name may be designated as “anonymous” if appropriate.
Creators []string `json:"creators"`
@@ -1,5 +1,7 @@
package model
import "github.com/anchore/syft/syft/source"
// derived from:
// - https://spdx.github.io/spdx-spec/appendix-III-RDF-data-model-implementation-and-identifier-syntax/
// - https://github.com/spdx/spdx-spec/blob/v2.2/schemas/spdx-schema.json
@@ -11,6 +13,10 @@ type Document struct {
// One instance is required for each SPDX file produced. It provides the necessary information for forward
// and backward compatibility for processing tools.
CreationInfo CreationInfo `json:"creationInfo"`
// SyftSourceData contains information about what is being described in this SPDX document (e.g. a container image, a directory, etc)
SyftSourceData *source.Metadata `json:"syftSourceData,omitempty"`
// SyftDistroData contains information about the linux distribution discovered
SyftDistroData *SyftDistroData `json:"syftDistroData,omitempty"`
// 2.2: Data License; should be "CC0-1.0"
// Cardinality: mandatory, one
// License expression for dataLicense. Compliance with the SPDX specification includes populating the SPDX
+12 -12
View File
@@ -3,17 +3,17 @@ package model
type FileType string
const (
DocumentationFileType FileType = "DOCUMENTATION" // if the file serves as documentation
ImageFileType FileType = "IMAGE" // if the file is associated with a picture image file (MIME type of image/*, e.g., .jpg, .gif)
VideoFileType FileType = "VIDEO" // if the file is associated with a video file type (MIME type of video/*)
ArchiveFileType FileType = "ARCHIVE" // if the file represents an archive (.tar, .jar, etc.)
SpdxFileType FileType = "SPDX" // if the file is an SPDX document
ApplicationFileType FileType = "APPLICATION" // if the file is associated with a specific application type (MIME type of application/*)
SourceFileType FileType = "SOURCE" // if the file is human readable source code (.c, .html, etc.)
BinaryFileType FileType = "BINARY" // if the file is a compiled object, target image or binary executable (.o, .a, etc.)
TextFileType FileType = "TEXT" // if the file is human readable text file (MIME type of text/*)
AudioFileType FileType = "AUDIO" // if the file is associated with an audio file (MIME type of audio/* , e.g. .mp3)
OtherFileType FileType = "OTHER" // if the file doesn't fit into the above categories (generated artifacts, data files, etc.)
DocumentationFileType FileType = "DOCUMENTATION"
ImageFileType FileType = "IMAGE"
VideoFileType FileType = "VIDEO"
ArchiveFileType FileType = "ARCHIVE"
SpdxFileType FileType = "SPDX"
ApplicationFileType FileType = "APPLICATION"
SourceFileType FileType = "SOURCE"
BinaryFileType FileType = "BINARY"
TextFileType FileType = "TEXT"
AudioFileType FileType = "AUDIO"
OtherFileType FileType = "OTHER"
)
type File struct {
@@ -36,6 +36,6 @@ type File struct {
// Indicates the project in which the SpdxElement originated. Tools must preserve doap:homepage and doap:name
// properties and the URI (if one is known) of doap:Project resources that are values of this property. All other
// properties of doap:Projects are not directly supported by SPDX and may be dropped when translating to or
// from some SPDX formats (deprecated).
// from some SPDX formats(deprecated).
ArtifactOf []string `json:"artifactOf,omitempty"`
}
@@ -47,4 +47,7 @@ type Package struct {
Supplier string `json:"supplier,omitempty"`
// Provides an indication of the version of the package that is described by this SpdxDocument.
VersionInfo string `json:"versionInfo,omitempty"`
// SyftPackageData provides a spot to add syft-specific data that is not available in-spec or is not easily decodable from
// other SPDX fields.
SyftPackageData *SyftPackageData `json:"syftPackageData,omitempty"`
}
@@ -0,0 +1,7 @@
package model
type SyftDistroData struct {
Name string `json:"name"` // Name of the Linux distribution
Version string `json:"version"` // Version of the Linux distribution (major or major.minor version)
IDLike string `json:"idLike"` // the ID_LIKE field found within the /etc/os-release file
}
@@ -0,0 +1,115 @@
package model
import (
"encoding/json"
"fmt"
"github.com/anchore/syft/internal/log"
"github.com/anchore/syft/syft/pkg"
"github.com/anchore/syft/syft/source"
)
type SyftPackageData struct {
SyftPackageBasicData
SyftPackageCustomData
}
type SyftPackageCustomData struct {
MetadataType pkg.MetadataType `json:"metadataType,omitempty"`
Metadata interface{} `json:"metadata,omitempty"`
}
type SyftPackageBasicData struct {
PackageType pkg.Type `json:"type,omitempty"`
FoundBy string `json:"foundBy,omitempty"`
Locations []source.Location `json:"locations,omitempty"`
Licenses []string `json:"licenses,omitempty"`
Language pkg.Language `json:"language,omitempty"`
}
// syftPackageMetadataUnpacker is all values needed from Package to disambiguate ambiguous fields during json unmarshaling.
type syftPackageMetadataUnpacker struct {
MetadataType pkg.MetadataType `json:"metadataType"`
Metadata json.RawMessage `json:"metadata"`
}
func (p *syftPackageMetadataUnpacker) String() string {
return fmt.Sprintf("metadataType: %s, metadata: %s", p.MetadataType, string(p.Metadata))
}
// UnmarshalJSON is a custom unmarshaller for handling basic values and values with ambiguous types.
// nolint:funlen
func (p *SyftPackageData) UnmarshalJSON(b []byte) error {
var basic SyftPackageBasicData
if err := json.Unmarshal(b, &basic); err != nil {
return err
}
p.SyftPackageBasicData = basic
var unpacker syftPackageMetadataUnpacker
if err := json.Unmarshal(b, &unpacker); err != nil {
log.Warnf("failed to unmarshall into syftPackageMetadataUnpacker: %v", err)
return err
}
p.MetadataType = unpacker.MetadataType
switch p.MetadataType {
case pkg.ApkMetadataType:
var payload pkg.ApkMetadata
if err := json.Unmarshal(unpacker.Metadata, &payload); err != nil {
return err
}
p.Metadata = payload
case pkg.RpmdbMetadataType:
var payload pkg.RpmdbMetadata
if err := json.Unmarshal(unpacker.Metadata, &payload); err != nil {
return err
}
p.Metadata = payload
case pkg.DpkgMetadataType:
var payload pkg.DpkgMetadata
if err := json.Unmarshal(unpacker.Metadata, &payload); err != nil {
return err
}
p.Metadata = payload
case pkg.JavaMetadataType:
var payload pkg.JavaMetadata
if err := json.Unmarshal(unpacker.Metadata, &payload); err != nil {
return err
}
p.Metadata = payload
case pkg.RustCargoPackageMetadataType:
var payload pkg.CargoPackageMetadata
if err := json.Unmarshal(unpacker.Metadata, &payload); err != nil {
return err
}
p.Metadata = payload
case pkg.GemMetadataType:
var payload pkg.GemMetadata
if err := json.Unmarshal(unpacker.Metadata, &payload); err != nil {
return err
}
p.Metadata = payload
case pkg.KbPackageMetadataType:
var payload pkg.KbPackageMetadata
if err := json.Unmarshal(unpacker.Metadata, &payload); err != nil {
return err
}
p.Metadata = payload
case pkg.PythonPackageMetadataType:
var payload pkg.PythonPackageMetadata
if err := json.Unmarshal(unpacker.Metadata, &payload); err != nil {
return err
}
p.Metadata = payload
case pkg.NpmPackageJSONMetadataType:
var payload pkg.NpmPackageJSONMetadata
if err := json.Unmarshal(unpacker.Metadata, &payload); err != nil {
return err
}
p.Metadata = payload
}
return nil
}
@@ -1,61 +0,0 @@
{
"SPDXID": "SPDXRef-DOCUMENT",
"name": "/some/path",
"spdxVersion": "SPDX-2.2",
"creationInfo": {
"created": "2021-12-20T19:12:47.869816Z",
"creators": [
"Organization: Anchore, Inc",
"Tool: syft-[not provided]"
],
"licenseListVersion": "3.15"
},
"dataLicense": "CC0-1.0",
"documentNamespace": "https://anchore.com/syft/dir/some/path-4b896ded-7852-4e31-b764-136b53bdf346",
"packages": [
{
"SPDXID": "SPDXRef-1d97af55efe9512f",
"name": "package-1",
"licenseConcluded": "MIT",
"downloadLocation": "NOASSERTION",
"externalRefs": [
{
"referenceCategory": "SECURITY",
"referenceLocator": "cpe:2.3:*:some:package:2:*:*:*:*:*:*:*",
"referenceType": "cpe23Type"
},
{
"referenceCategory": "PACKAGE_MANAGER",
"referenceLocator": "a-purl-2",
"referenceType": "purl"
}
],
"filesAnalyzed": false,
"licenseDeclared": "MIT",
"sourceInfo": "acquired package info from installed python package manifest file: /some/path/pkg1",
"versionInfo": "1.0.1"
},
{
"SPDXID": "SPDXRef-ad3d1c4abd84bf75",
"name": "package-2",
"licenseConcluded": "NONE",
"downloadLocation": "NOASSERTION",
"externalRefs": [
{
"referenceCategory": "SECURITY",
"referenceLocator": "cpe:2.3:*:some:package:2:*:*:*:*:*:*:*",
"referenceType": "cpe23Type"
},
{
"referenceCategory": "PACKAGE_MANAGER",
"referenceLocator": "a-purl-2",
"referenceType": "purl"
}
],
"filesAnalyzed": false,
"licenseDeclared": "NONE",
"sourceInfo": "acquired package info from DPKG DB: /some/path/pkg1",
"versionInfo": "2.0.1"
}
]
}
@@ -0,0 +1,148 @@
{
"SPDXID": "SPDXRef-DOCUMENT",
"name": "/some/path",
"spdxVersion": "SPDX-2.2",
"creationInfo": {
"created": "2021-10-11T01:54:40.609143Z",
"creators": [
"Organization: Anchore, Inc",
"Tool: syft-[not provided]"
],
"licenseListVersion": "3.14"
},
"syftSourceData": {
"Scheme": "DirectoryScheme",
"ImageMetadata": {
"userInput": "",
"imageID": "",
"manifestDigest": "",
"mediaType": "",
"tags": null,
"imageSize": 0,
"layers": null,
"manifest": null,
"config": null,
"repoDigests": null
},
"Path": "/some/path"
},
"syftDistroData": {
"name": "debian",
"version": "1.2.3",
"idLike": "like!"
},
"dataLicense": "CC0-1.0",
"documentNamespace": "https:/anchore.com/syft/dir/some/path-e94d6e43-5de5-4849-8e21-983ada9103c5",
"packages": [
{
"SPDXID": "SPDXRef-Package-python-package-1-1.0.1",
"name": "package-1",
"licenseConcluded": "MIT",
"downloadLocation": "NOASSERTION",
"externalRefs": [
{
"referenceCategory": "SECURITY",
"referenceLocator": "cpe:2.3:*:some:package:2:*:*:*:*:*:*:*",
"referenceType": "cpe23Type"
},
{
"referenceCategory": "PACKAGE_MANAGER",
"referenceLocator": "a-purl-2",
"referenceType": "purl"
}
],
"filesAnalyzed": false,
"hasFiles": [
"SPDXRef-File-package-1-04cd22424378dcd6c77fce08beb52493b5494a60ea5e1f9bdf9b16dc0cacffe9"
],
"licenseDeclared": "MIT",
"sourceInfo": "acquired package info from installed python package manifest file: /some/path/pkg1",
"versionInfo": "1.0.1",
"syftPackageData": {
"type": "python",
"foundBy": "the-cataloger-1",
"locations": [
{
"path": "/some/path/pkg1"
}
],
"licenses": [
"MIT"
],
"language": "python",
"metadataType": "PythonPackageMetadata",
"metadata": {
"name": "package-1",
"version": "1.0.1",
"license": "",
"author": "",
"authorEmail": "",
"platform": "",
"files": [
{
"path": "/some/path/pkg1/depedencies/foo"
}
],
"sitePackagesRootPath": ""
}
}
},
{
"SPDXID": "SPDXRef-Package-deb-package-2-2.0.1",
"name": "package-2",
"licenseConcluded": "NONE",
"downloadLocation": "NOASSERTION",
"externalRefs": [
{
"referenceCategory": "SECURITY",
"referenceLocator": "cpe:2.3:*:some:package:2:*:*:*:*:*:*:*",
"referenceType": "cpe23Type"
},
{
"referenceCategory": "PACKAGE_MANAGER",
"referenceLocator": "a-purl-2",
"referenceType": "purl"
}
],
"filesAnalyzed": false,
"licenseDeclared": "NONE",
"sourceInfo": "acquired package info from DPKG DB: /some/path/pkg1",
"versionInfo": "2.0.1",
"syftPackageData": {
"type": "deb",
"foundBy": "the-cataloger-2",
"locations": [
{
"path": "/some/path/pkg1"
}
],
"metadataType": "DpkgMetadata",
"metadata": {
"package": "package-2",
"source": "",
"version": "2.0.1",
"sourceVersion": "",
"architecture": "",
"maintainer": "",
"installedSize": 0,
"files": null
}
}
}
],
"files": [
{
"SPDXID": "SPDXRef-File-package-1-04cd22424378dcd6c77fce08beb52493b5494a60ea5e1f9bdf9b16dc0cacffe9",
"name": "foo",
"licenseConcluded": "",
"fileName": "/some/path/pkg1/depedencies/foo"
}
],
"relationships": [
{
"spdxElementId": "SPDXRef-Package-python-package-1-1.0.1",
"relationshipType": "CONTAINS",
"relatedSpdxElement": "SPDXRef-File-package-1-04cd22424378dcd6c77fce08beb52493b5494a60ea5e1f9bdf9b16dc0cacffe9"
}
]
}
@@ -1,61 +0,0 @@
{
"SPDXID": "SPDXRef-DOCUMENT",
"name": "user-image-input",
"spdxVersion": "SPDX-2.2",
"creationInfo": {
"created": "2021-12-20T19:13:07.647486Z",
"creators": [
"Organization: Anchore, Inc",
"Tool: syft-[not provided]"
],
"licenseListVersion": "3.15"
},
"dataLicense": "CC0-1.0",
"documentNamespace": "https://anchore.com/syft/image/user-image-input-174da656-1824-4bd3-8604-28919f8a65bc",
"packages": [
{
"SPDXID": "SPDXRef-d16127444133b5c1",
"name": "package-1",
"licenseConcluded": "MIT",
"downloadLocation": "NOASSERTION",
"externalRefs": [
{
"referenceCategory": "SECURITY",
"referenceLocator": "cpe:2.3:*:some:package:1:*:*:*:*:*:*:*",
"referenceType": "cpe23Type"
},
{
"referenceCategory": "PACKAGE_MANAGER",
"referenceLocator": "a-purl-1",
"referenceType": "purl"
}
],
"filesAnalyzed": false,
"licenseDeclared": "MIT",
"sourceInfo": "acquired package info from installed python package manifest file: /somefile-1.txt",
"versionInfo": "1.0.1"
},
{
"SPDXID": "SPDXRef-24907357f3705420",
"name": "package-2",
"licenseConcluded": "NONE",
"downloadLocation": "NOASSERTION",
"externalRefs": [
{
"referenceCategory": "SECURITY",
"referenceLocator": "cpe:2.3:*:some:package:2:*:*:*:*:*:*:*",
"referenceType": "cpe23Type"
},
{
"referenceCategory": "PACKAGE_MANAGER",
"referenceLocator": "a-purl-2",
"referenceType": "purl"
}
],
"filesAnalyzed": false,
"licenseDeclared": "NONE",
"sourceInfo": "acquired package info from DPKG DB: /somefile-2.txt",
"versionInfo": "2.0.1"
}
]
}
@@ -0,0 +1,140 @@
{
"SPDXID": "SPDXRef-DOCUMENT",
"name": "user-image-input",
"spdxVersion": "SPDX-2.2",
"creationInfo": {
"created": "2021-10-11T01:54:40.61361Z",
"creators": [
"Organization: Anchore, Inc",
"Tool: syft-[not provided]"
],
"licenseListVersion": "3.14"
},
"syftSourceData": {
"Scheme": "ImageScheme",
"ImageMetadata": {
"userInput": "user-image-input",
"imageID": "sha256:2480160b55bec40c44d3b145c7b2c1c47160db8575c3dcae086d76b9370ae7ca",
"manifestDigest": "sha256:2731251dc34951c0e50fcc643b4c5f74922dad1a5d98f302b504cf46cd5d9368",
"mediaType": "application/vnd.docker.distribution.manifest.v2+json",
"tags": [
"stereoscope-fixture-image-simple:85066c51088bdd274f7a89e99e00490f666c49e72ffc955707cd6e18f0e22c5b"
],
"imageSize": 38,
"layers": [
{
"mediaType": "application/vnd.docker.image.rootfs.diff.tar.gzip",
"digest": "sha256:fb6beecb75b39f4bb813dbf177e501edd5ddb3e69bb45cedeb78c676ee1b7a59",
"size": 22
},
{
"mediaType": "application/vnd.docker.image.rootfs.diff.tar.gzip",
"digest": "sha256:319b588ce64253a87b533c8ed01cf0025e0eac98e7b516e12532957e1244fdec",
"size": 16
}
],
"manifest": "eyJzY2hlbWFWZXJzaW9uIjoyLCJtZWRpYVR5cGUiOiJhcHBsaWNhdGlvbi92bmQuZG9ja2VyLmRpc3RyaWJ1dGlvbi5tYW5pZmVzdC52Mitqc29uIiwiY29uZmlnIjp7Im1lZGlhVHlwZSI6ImFwcGxpY2F0aW9uL3ZuZC5kb2NrZXIuY29udGFpbmVyLmltYWdlLnYxK2pzb24iLCJzaXplIjo2NjcsImRpZ2VzdCI6InNoYTI1NjoyNDgwMTYwYjU1YmVjNDBjNDRkM2IxNDVjN2IyYzFjNDcxNjBkYjg1NzVjM2RjYWUwODZkNzZiOTM3MGFlN2NhIn0sImxheWVycyI6W3sibWVkaWFUeXBlIjoiYXBwbGljYXRpb24vdm5kLmRvY2tlci5pbWFnZS5yb290ZnMuZGlmZi50YXIuZ3ppcCIsInNpemUiOjIwNDgsImRpZ2VzdCI6InNoYTI1NjpmYjZiZWVjYjc1YjM5ZjRiYjgxM2RiZjE3N2U1MDFlZGQ1ZGRiM2U2OWJiNDVjZWRlYjc4YzY3NmVlMWI3YTU5In0seyJtZWRpYVR5cGUiOiJhcHBsaWNhdGlvbi92bmQuZG9ja2VyLmltYWdlLnJvb3Rmcy5kaWZmLnRhci5nemlwIiwic2l6ZSI6MjA0OCwiZGlnZXN0Ijoic2hhMjU2OjMxOWI1ODhjZTY0MjUzYTg3YjUzM2M4ZWQwMWNmMDAyNWUwZWFjOThlN2I1MTZlMTI1MzI5NTdlMTI0NGZkZWMifV19",
"config": "eyJhcmNoaXRlY3R1cmUiOiJhbWQ2NCIsImNvbmZpZyI6eyJFbnYiOlsiUEFUSD0vdXNyL2xvY2FsL3NiaW46L3Vzci9sb2NhbC9iaW46L3Vzci9zYmluOi91c3IvYmluOi9zYmluOi9iaW4iXSwiV29ya2luZ0RpciI6Ii8iLCJPbkJ1aWxkIjpudWxsfSwiY3JlYXRlZCI6IjIwMjEtMTAtMDRUMTE6NDA6MDAuNjM4Mzk0NVoiLCJoaXN0b3J5IjpbeyJjcmVhdGVkIjoiMjAyMS0xMC0wNFQxMTo0MDowMC41OTA3MzE2WiIsImNyZWF0ZWRfYnkiOiJBREQgZmlsZS0xLnR4dCAvc29tZWZpbGUtMS50eHQgIyBidWlsZGtpdCIsImNvbW1lbnQiOiJidWlsZGtpdC5kb2NrZXJmaWxlLnYwIn0seyJjcmVhdGVkIjoiMjAyMS0xMC0wNFQxMTo0MDowMC42MzgzOTQ1WiIsImNyZWF0ZWRfYnkiOiJBREQgZmlsZS0yLnR4dCAvc29tZWZpbGUtMi50eHQgIyBidWlsZGtpdCIsImNvbW1lbnQiOiJidWlsZGtpdC5kb2NrZXJmaWxlLnYwIn1dLCJvcyI6ImxpbnV4Iiwicm9vdGZzIjp7InR5cGUiOiJsYXllcnMiLCJkaWZmX2lkcyI6WyJzaGEyNTY6ZmI2YmVlY2I3NWIzOWY0YmI4MTNkYmYxNzdlNTAxZWRkNWRkYjNlNjliYjQ1Y2VkZWI3OGM2NzZlZTFiN2E1OSIsInNoYTI1NjozMTliNTg4Y2U2NDI1M2E4N2I1MzNjOGVkMDFjZjAwMjVlMGVhYzk4ZTdiNTE2ZTEyNTMyOTU3ZTEyNDRmZGVjIl19fQ==",
"repoDigests": []
},
"Path": ""
},
"syftDistroData": {
"name": "debian",
"version": "1.2.3",
"idLike": "like!"
},
"dataLicense": "CC0-1.0",
"documentNamespace": "https:/anchore.com/syft/image/user-image-input-9badd1fb-a067-4207-addf-243c5b7705fe",
"packages": [
{
"SPDXID": "SPDXRef-Package-python-package-1-1.0.1",
"name": "package-1",
"licenseConcluded": "MIT",
"downloadLocation": "NOASSERTION",
"externalRefs": [
{
"referenceCategory": "SECURITY",
"referenceLocator": "cpe:2.3:*:some:package:1:*:*:*:*:*:*:*",
"referenceType": "cpe23Type"
},
{
"referenceCategory": "PACKAGE_MANAGER",
"referenceLocator": "a-purl-1",
"referenceType": "purl"
}
],
"filesAnalyzed": false,
"licenseDeclared": "MIT",
"sourceInfo": "acquired package info from installed python package manifest file: /somefile-1.txt",
"versionInfo": "1.0.1",
"syftPackageData": {
"type": "python",
"foundBy": "the-cataloger-1",
"locations": [
{
"path": "/somefile-1.txt",
"layerID": "sha256:fb6beecb75b39f4bb813dbf177e501edd5ddb3e69bb45cedeb78c676ee1b7a59"
}
],
"licenses": [
"MIT"
],
"language": "python",
"metadataType": "PythonPackageMetadata",
"metadata": {
"name": "package-1",
"version": "1.0.1",
"license": "",
"author": "",
"authorEmail": "",
"platform": "",
"sitePackagesRootPath": ""
}
}
},
{
"SPDXID": "SPDXRef-Package-deb-package-2-2.0.1",
"name": "package-2",
"licenseConcluded": "NONE",
"downloadLocation": "NOASSERTION",
"externalRefs": [
{
"referenceCategory": "SECURITY",
"referenceLocator": "cpe:2.3:*:some:package:2:*:*:*:*:*:*:*",
"referenceType": "cpe23Type"
},
{
"referenceCategory": "PACKAGE_MANAGER",
"referenceLocator": "a-purl-2",
"referenceType": "purl"
}
],
"filesAnalyzed": false,
"licenseDeclared": "NONE",
"sourceInfo": "acquired package info from DPKG DB: /somefile-2.txt",
"versionInfo": "2.0.1",
"syftPackageData": {
"type": "deb",
"foundBy": "the-cataloger-2",
"locations": [
{
"path": "/somefile-2.txt",
"layerID": "sha256:319b588ce64253a87b533c8ed01cf0025e0eac98e7b516e12532957e1244fdec"
}
],
"metadataType": "DpkgMetadata",
"metadata": {
"package": "package-2",
"source": "",
"version": "2.0.1",
"sourceVersion": "",
"architecture": "",
"maintainer": "",
"installedSize": 0,
"files": null
}
}
}
]
}
+97 -159
View File
@@ -2,32 +2,28 @@ package spdx22json
import (
"fmt"
"path/filepath"
"sort"
"path"
"strings"
"time"
"github.com/anchore/syft/syft/distro"
"github.com/anchore/syft/internal"
"github.com/anchore/syft/internal/formats/common/spdxhelpers"
"github.com/anchore/syft/internal/formats/spdx22json/model"
"github.com/anchore/syft/internal/log"
"github.com/anchore/syft/internal/spdxlicense"
"github.com/anchore/syft/internal/version"
"github.com/anchore/syft/syft/artifact"
"github.com/anchore/syft/syft/file"
"github.com/anchore/syft/syft/pkg"
"github.com/anchore/syft/syft/sbom"
"github.com/anchore/syft/syft/source"
"github.com/google/uuid"
)
// toFormatModel creates and populates a new JSON document struct that follows the SPDX 2.2 spec from the given cataloging results.
func toFormatModel(s sbom.SBOM) (*model.Document, error) {
name, namespace, err := spdxhelpers.DocumentNameAndNamespace(s.Source)
if err != nil {
return nil, err
}
func toFormatModel(catalog *pkg.Catalog, srcMetadata *source.Metadata, d *distro.Distro) model.Document {
name := documentName(srcMetadata)
packages, files, relationships := extractFromCatalog(catalog)
return &model.Document{
return model.Document{
Element: model.Element{
SPDXID: model.ElementID("DOCUMENT").String(),
Name: name,
@@ -43,19 +39,72 @@ func toFormatModel(s sbom.SBOM) (*model.Document, error) {
LicenseListVersion: spdxlicense.Version,
},
DataLicense: "CC0-1.0",
DocumentNamespace: namespace,
Packages: toPackages(s.Artifacts.PackageCatalog, s.Relationships),
Files: toFiles(s),
Relationships: toRelationships(s.Relationships),
}, nil
DocumentNamespace: documentNamespace(name, srcMetadata),
Packages: packages,
Files: files,
Relationships: relationships,
SyftSourceData: srcMetadata,
SyftDistroData: toSyftDistroData(d),
}
}
func toPackages(catalog *pkg.Catalog, relationships []artifact.Relationship) []model.Package {
func toSyftDistroData(d *distro.Distro) *model.SyftDistroData {
if d == nil {
return nil
}
return &model.SyftDistroData{
Name: d.Name(),
Version: d.FullVersion(),
IDLike: d.IDLike,
}
}
func documentName(srcMetadata *source.Metadata) string {
if srcMetadata != nil {
switch srcMetadata.Scheme {
case source.ImageScheme:
return cleanSPDXName(srcMetadata.ImageMetadata.UserInput)
case source.DirectoryScheme:
return cleanSPDXName(srcMetadata.Path)
}
}
// TODO: is this alright?
return uuid.Must(uuid.NewRandom()).String()
}
func documentNamespace(name string, srcMetadata *source.Metadata) string {
input := "unknown-source-type"
if srcMetadata != nil {
switch srcMetadata.Scheme {
case source.ImageScheme:
input = "image"
case source.DirectoryScheme:
input = "dir"
}
}
uniqueID := uuid.Must(uuid.NewRandom())
identifier := path.Join(input, uniqueID.String())
if name != "." {
identifier = path.Join(input, fmt.Sprintf("%s-%s", name, uniqueID.String()))
}
return path.Join(anchoreNamespace, identifier)
}
func extractFromCatalog(catalog *pkg.Catalog) ([]model.Package, []model.File, []model.Relationship) {
packages := make([]model.Package, 0)
relationships := make([]model.Relationship, 0)
files := make([]model.File, 0)
for _, p := range catalog.Sorted() {
license := spdxhelpers.License(p)
packageSpdxID := model.ElementID(p.ID()).String()
packageSpdxID := model.ElementID(fmt.Sprintf("Package-%+v-%s-%s", p.Type, p.Name, p.Version)).String()
packageFiles, fileIDs, packageFileRelationships := spdxhelpers.Files(packageSpdxID, p)
files = append(files, packageFiles...)
relationships = append(relationships, packageFileRelationships...)
// note: the license concluded and declared should be the same since we are collecting license information
// from the project data itself (the installed package files).
@@ -64,163 +113,52 @@ func toPackages(catalog *pkg.Catalog, relationships []artifact.Relationship) []m
DownloadLocation: spdxhelpers.DownloadLocation(p),
ExternalRefs: spdxhelpers.ExternalRefs(p),
FilesAnalyzed: false,
HasFiles: fileIDsForPackage(packageSpdxID, relationships),
HasFiles: fileIDs,
Homepage: spdxhelpers.Homepage(p),
// The Declared License is what the authors of a project believe govern the package
LicenseDeclared: license,
Originator: spdxhelpers.Originator(p),
SourceInfo: spdxhelpers.SourceInfo(p),
VersionInfo: p.Version,
LicenseDeclared: license, // The Declared License is what the authors of a project believe govern the package
Originator: spdxhelpers.Originator(p),
SourceInfo: spdxhelpers.SourceInfo(p),
VersionInfo: p.Version,
Item: model.Item{
// The Concluded License field is the license the SPDX file creator believes governs the package
LicenseConcluded: license,
LicenseConcluded: license, // The Concluded License field is the license the SPDX file creator believes governs the package
Element: model.Element{
SPDXID: packageSpdxID,
Name: p.Name,
},
},
SyftPackageData: toSyftPackageData(p),
})
}
return packages
return packages, files, relationships
}
func fileIDsForPackage(packageSpdxID string, relationships []artifact.Relationship) (fileIDs []string) {
for _, relationship := range relationships {
if relationship.Type != artifact.ContainsRelationship {
continue
}
if _, ok := relationship.From.(pkg.Package); !ok {
continue
}
if _, ok := relationship.To.(source.Coordinates); !ok {
continue
}
if string(relationship.From.ID()) == packageSpdxID {
fileIDs = append(fileIDs, string(relationship.To.ID()))
}
}
return fileIDs
}
func toFiles(s sbom.SBOM) []model.File {
results := make([]model.File, 0)
artifacts := s.Artifacts
for _, coordinates := range sbom.AllCoordinates(s) {
var metadata *source.FileMetadata
if metadataForLocation, exists := artifacts.FileMetadata[coordinates]; exists {
metadata = &metadataForLocation
}
var digests []file.Digest
if digestsForLocation, exists := artifacts.FileDigests[coordinates]; exists {
digests = digestsForLocation
}
// TODO: add file classifications (?) and content as a snippet
var comment string
if coordinates.FileSystemID != "" {
comment = fmt.Sprintf("layerID: %s", coordinates.FileSystemID)
}
results = append(results, model.File{
Item: model.Item{
Element: model.Element{
SPDXID: string(coordinates.ID()),
Name: filepath.Base(coordinates.RealPath),
Comment: comment,
},
// required, no attempt made to determine license information
LicenseConcluded: "NOASSERTION",
},
Checksums: toFileChecksums(digests),
FileName: coordinates.RealPath,
FileTypes: toFileTypes(metadata),
})
}
// sort by real path then virtual path to ensure the result is stable across multiple runs
sort.SliceStable(results, func(i, j int) bool {
return results[i].FileName < results[j].FileName
})
return results
}
func toFileChecksums(digests []file.Digest) (checksums []model.Checksum) {
for _, digest := range digests {
checksums = append(checksums, model.Checksum{
Algorithm: digest.Algorithm,
ChecksumValue: digest.Value,
})
}
return checksums
}
func toFileTypes(metadata *source.FileMetadata) (ty []string) {
if metadata == nil {
func toSyftPackageData(p *pkg.Package) *model.SyftPackageData {
if p == nil {
return nil
}
mimeTypePrefix := strings.Split(metadata.MIMEType, "/")[0]
switch mimeTypePrefix {
case "image":
ty = append(ty, string(model.ImageFileType))
case "video":
ty = append(ty, string(model.VideoFileType))
case "application":
ty = append(ty, string(model.ApplicationFileType))
case "text":
ty = append(ty, string(model.TextFileType))
case "audio":
ty = append(ty, string(model.AudioFileType))
return &model.SyftPackageData{
SyftPackageBasicData: model.SyftPackageBasicData{
PackageType: p.Type,
FoundBy: p.FoundBy,
Locations: p.Locations,
Language: p.Language,
Licenses: p.Licenses,
},
SyftPackageCustomData: model.SyftPackageCustomData{
MetadataType: p.MetadataType,
Metadata: p.Metadata,
},
}
if internal.IsExecutable(metadata.MIMEType) {
ty = append(ty, string(model.BinaryFileType))
}
if internal.IsArchive(metadata.MIMEType) {
ty = append(ty, string(model.ArchiveFileType))
}
// TODO: add support for source, spdx, and documentation file types
if len(ty) == 0 {
ty = append(ty, string(model.OtherFileType))
}
return ty
}
func toRelationships(relationships []artifact.Relationship) (result []model.Relationship) {
for _, r := range relationships {
exists, relationshipType, comment := lookupRelationship(r.Type)
func cleanSPDXName(name string) string {
// remove # according to specification
name = strings.ReplaceAll(name, "#", "-")
if !exists {
log.Warnf("unable to convert relationship from SPDX 2.2 JSON, dropping: %+v", r)
continue
}
// remove : for url construction
name = strings.ReplaceAll(name, ":", "-")
result = append(result, model.Relationship{
SpdxElementID: string(r.From.ID()),
RelationshipType: relationshipType,
RelatedSpdxElement: string(r.To.ID()),
Comment: comment,
})
}
return result
}
func lookupRelationship(ty artifact.RelationshipType) (bool, model.RelationshipType, string) {
switch ty {
case artifact.ContainsRelationship:
return true, model.ContainsRelationship, ""
case artifact.OwnershipByFileOverlapRelationship:
return true, model.OtherRelationship, fmt.Sprintf("%s: indicates that the parent package claims ownership of a child package since the parent metadata indicates overlap with a location that a cataloger found the child package by", ty)
}
return false, "", ""
// clean relative pathing
return path.Clean(name)
}
@@ -1,256 +0,0 @@
package spdx22json
import (
"testing"
"github.com/anchore/syft/syft/pkg"
"github.com/anchore/syft/syft/file"
"github.com/anchore/syft/syft/artifact"
"github.com/anchore/syft/internal/formats/spdx22json/model"
"github.com/anchore/syft/syft/source"
"github.com/stretchr/testify/assert"
)
func Test_toFileTypes(t *testing.T) {
tests := []struct {
name string
metadata source.FileMetadata
expected []string
}{
{
name: "application",
metadata: source.FileMetadata{
MIMEType: "application/vnd.unknown",
},
expected: []string{
string(model.ApplicationFileType),
},
},
{
name: "archive",
metadata: source.FileMetadata{
MIMEType: "application/zip",
},
expected: []string{
string(model.ApplicationFileType),
string(model.ArchiveFileType),
},
},
{
name: "audio",
metadata: source.FileMetadata{
MIMEType: "audio/ogg",
},
expected: []string{
string(model.AudioFileType),
},
},
{
name: "video",
metadata: source.FileMetadata{
MIMEType: "video/3gpp",
},
expected: []string{
string(model.VideoFileType),
},
},
{
name: "text",
metadata: source.FileMetadata{
MIMEType: "text/html",
},
expected: []string{
string(model.TextFileType),
},
},
{
name: "image",
metadata: source.FileMetadata{
MIMEType: "image/png",
},
expected: []string{
string(model.ImageFileType),
},
},
{
name: "binary",
metadata: source.FileMetadata{
MIMEType: "application/x-sharedlib",
},
expected: []string{
string(model.ApplicationFileType),
string(model.BinaryFileType),
},
},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
assert.ElementsMatch(t, test.expected, toFileTypes(&test.metadata))
})
}
}
func Test_lookupRelationship(t *testing.T) {
tests := []struct {
input artifact.RelationshipType
exists bool
ty model.RelationshipType
comment string
}{
{
input: artifact.ContainsRelationship,
exists: true,
ty: model.ContainsRelationship,
},
{
input: artifact.OwnershipByFileOverlapRelationship,
exists: true,
ty: model.OtherRelationship,
comment: "ownership-by-file-overlap: indicates that the parent package claims ownership of a child package since the parent metadata indicates overlap with a location that a cataloger found the child package by",
},
{
input: "made-up",
exists: false,
},
}
for _, test := range tests {
t.Run(string(test.input), func(t *testing.T) {
exists, ty, comment := lookupRelationship(test.input)
assert.Equal(t, exists, test.exists)
assert.Equal(t, ty, test.ty)
assert.Equal(t, comment, test.comment)
})
}
}
func Test_toFileChecksums(t *testing.T) {
tests := []struct {
name string
digests []file.Digest
expected []model.Checksum
}{
{
name: "empty",
},
{
name: "has digests",
digests: []file.Digest{
{
Algorithm: "sha256",
Value: "deadbeefcafe",
},
{
Algorithm: "md5",
Value: "meh",
},
},
expected: []model.Checksum{
{
Algorithm: "sha256",
ChecksumValue: "deadbeefcafe",
},
{
Algorithm: "md5",
ChecksumValue: "meh",
},
},
},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
assert.ElementsMatch(t, test.expected, toFileChecksums(test.digests))
})
}
}
func Test_fileIDsForPackage(t *testing.T) {
p := pkg.Package{
Name: "bogus",
}
c := source.Coordinates{
RealPath: "/path",
FileSystemID: "nowhere",
}
tests := []struct {
name string
id string
relationships []artifact.Relationship
expected []string
}{
{
name: "find file IDs for packages with package-file relationships",
id: string(p.ID()),
relationships: []artifact.Relationship{
{
From: p,
To: c,
Type: artifact.ContainsRelationship,
},
},
expected: []string{
string(c.ID()),
},
},
{
name: "ignore package-to-package",
id: string(p.ID()),
relationships: []artifact.Relationship{
{
From: p,
To: p,
Type: artifact.ContainsRelationship,
},
},
expected: []string{},
},
{
name: "ignore file-to-file",
id: string(p.ID()),
relationships: []artifact.Relationship{
{
From: c,
To: c,
Type: artifact.ContainsRelationship,
},
},
expected: []string{},
},
{
name: "ignore file-to-package",
id: string(p.ID()),
relationships: []artifact.Relationship{
{
From: c,
To: p,
Type: artifact.ContainsRelationship,
},
},
expected: []string{},
},
{
name: "filter by relationship type",
id: string(p.ID()),
relationships: []artifact.Relationship{
{
From: p,
To: c,
Type: artifact.OwnershipByFileOverlapRelationship,
},
},
expected: []string{},
},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
assert.ElementsMatch(t, test.expected, fileIDsForPackage(test.id, test.relationships))
})
}
}
@@ -0,0 +1,64 @@
package spdx22json
import (
"github.com/anchore/syft/internal/formats/common/spdxhelpers"
"github.com/anchore/syft/internal/formats/spdx22json/model"
"github.com/anchore/syft/internal/log"
"github.com/anchore/syft/syft/distro"
"github.com/anchore/syft/syft/pkg"
"github.com/anchore/syft/syft/source"
)
func toSyftModel(doc model.Document) (*pkg.Catalog, *source.Metadata, *distro.Distro, error) {
d, err := toSyftDistro(doc.SyftDistroData)
if err != nil {
log.Warnf("unable to parse distro info=%+v: %+v", d, err)
d = nil
}
return toSyftCatalog(doc.Packages), doc.SyftSourceData, d, nil
}
func toSyftCatalog(pkgs []model.Package) *pkg.Catalog {
catalog := pkg.NewCatalog()
for _, p := range pkgs {
catalog.Add(toSyftPackage(p))
}
return catalog
}
func toSyftPackage(p model.Package) pkg.Package {
syftPkg := pkg.Package{
Name: p.Name,
Version: p.VersionInfo,
CPEs: spdxhelpers.ExtractCPEs(p.ExternalRefs),
PURL: spdxhelpers.ExtractPURL(p.ExternalRefs),
}
if extra := p.SyftPackageData; extra != nil {
syftPkg.Type = extra.PackageType
syftPkg.FoundBy = extra.FoundBy
syftPkg.Locations = extra.Locations
syftPkg.Language = extra.Language
syftPkg.Licenses = extra.Licenses
syftPkg.MetadataType = extra.MetadataType
syftPkg.Metadata = extra.Metadata
}
//if syftPkg.Type == "" && syftPkg.PURL != "" {
// // TODO: extract package type from purl --this is useful for ingesting from tools other than syft and is important for grype
//}
return syftPkg
}
func toSyftDistro(d *model.SyftDistroData) (*distro.Distro, error) {
if d == nil {
return nil, nil
}
newDistro, err := distro.NewDistro(distro.Type(d.Name), d.Version, d.IDLike)
if err != nil {
return nil, err
}
return &newDistro, nil
}
+26
View File
@@ -0,0 +1,26 @@
package spdx22json
import (
"encoding/json"
"fmt"
"io"
)
func validator(reader io.Reader) error {
type Document struct {
SPDXID string `json:"SPDXID"`
}
dec := json.NewDecoder(reader)
var doc Document
err := dec.Decode(&doc)
if err != nil {
return fmt.Errorf("unable to decode: %w", err)
}
if doc.SPDXID != "" {
return nil
}
return fmt.Errorf("could not extract document SPDXID")
}
@@ -1,16 +0,0 @@
package spdx22tagvalue
import (
"io"
"github.com/anchore/syft/syft/sbom"
"github.com/spdx/tools-golang/tvsaver"
)
func encoder(output io.Writer, s sbom.SBOM) error {
model, err := toFormatModel(s)
if err != nil {
return err
}
return tvsaver.Save2_2(model, output)
}
-13
View File
@@ -1,13 +0,0 @@
package spdx22tagvalue
import "github.com/anchore/syft/syft/format"
// note: this format is LOSSY relative to the syftjson formation, which means that decoding and validation is not supported at this time
func Format() format.Format {
return format.NewFormat(
format.SPDXTagValueOption,
encoder,
nil,
nil,
)
}
@@ -1,4 +0,0 @@
# Note: changes to this file will result in updating several test values. Consider making a new image fixture instead of editing this one.
FROM scratch
ADD file-1.txt /somefile-1.txt
ADD file-2.txt /somefile-2.txt

Some files were not shown because too many files have changed in this diff Show More