Compare commits

...
30 Commits
Author SHA1 Message Date
Alex Goodman dad1d9691c Merge pull request #263 from anchore/add-source-to-rpmdb-cataloger
Add source to packages found by RPMdb cataloger
2020-11-12 09:36:20 -05:00
Alex Goodman 7964c27bd8 add source to packages found by rpmdb cataloger
Signed-off-by: Alex Goodman <alex.goodman@anchore.com>
2020-11-12 09:27:08 -05:00
Alex Goodman 9581a0309b Merge pull request #262 from anchore/fix-release
Fix release process
2020-11-11 15:15:12 -05:00
Alex Goodman 7e270bf76c restore the checksum file during release
Signed-off-by: Alex Goodman <alex.goodman@anchore.com>
2020-11-11 15:11:51 -05:00
Alex Goodman fde37b4b56 provide goreleaser asset version via env var
Signed-off-by: Alex Goodman <alex.goodman@anchore.com>
2020-11-11 14:36:24 -05:00
Dan Luhring 8627ea88ce Fix usage of goreleaser's artifact pipeline
Signed-off-by: Dan Luhring <dan.luhring@anchore.com>
2020-11-11 14:29:55 -05:00
Alex Goodman 2c82c7c0f0 Merge pull request #257 from anchore/enhance-deb-support
Enhance dpkg support
2020-11-11 13:12:29 -05:00
Dan Luhring 4c38ea595d Merge pull request #261 from anchore/fix-brew
Add signed and notarized ZIP release asset
2020-11-11 13:09:45 -05:00
Dan Luhring 6fe5ad0739 Avoid Finder window flash during install
Signed-off-by: Dan Luhring <dan.luhring@anchore.com>
2020-11-11 12:38:11 -05:00
Dan Luhring 7bced775c4 Add zip to gon outputs
Signed-off-by: Dan Luhring <dan.luhring@anchore.com>
2020-11-11 12:30:29 -05:00
Alex Goodman db98fba3b4 Merge pull request #260 from anchore/add-warning-levels
Add warning log level
2020-11-11 11:13:51 -05:00
Alfredo Deza 29d464c38a Merge pull request #258 from anchore/distro-like
Report the ID_LIKE value in the JSON presenter
2020-11-11 11:11:13 -05:00
Alex Goodman 2a329002b8 enhance dpkg support by parsing md5sum and copyright file sources
Signed-off-by: Alex Goodman <alex.goodman@anchore.com>
2020-11-11 11:06:34 -05:00
Alex Goodman a6365886ad add warning log level
Signed-off-by: Alex Goodman <alex.goodman@anchore.com>
2020-11-10 15:09:41 -05:00
Alfredo Deza 6ae3b47959 tests: update all tests to pass the IDLike value
Signed-off-by: Alfredo Deza <adeza@anchore.com>
2020-11-10 12:40:50 -05:00
Alfredo Deza 10b55311df presenter: provide ID_LIKE information in json
Signed-off-by: Alfredo Deza <adeza@anchore.com>
2020-11-10 12:40:17 -05:00
Alfredo Deza 64d5554144 include ID_LIKE in distro detection
Signed-off-by: Alfredo Deza <adeza@anchore.com>
2020-11-10 12:37:35 -05:00
Alfredo Deza b3098f3423 Merge pull request #250 from anchore/issue-169
Add distro information to JSON presenter
2020-11-09 14:18:48 -05:00
Alfredo Deza c2cf4eb7b0 update tests for the new distro information
Signed-off-by: Alfredo Deza <adeza@anchore.com>
2020-11-09 12:16:00 -05:00
Alfredo Deza 1e79986188 json: update the document to include distro information
Signed-off-by: Alfredo Deza <adeza@anchore.com>
2020-11-09 12:16:00 -05:00
Dan Luhring 3699a917fd Merge pull request #254 from anchore/updates-to-mac-releases
Updates for macOS release process
2020-11-06 13:49:54 -05:00
Dan Luhring 501870f89f Update README for secure macOS releases
Signed-off-by: Dan Luhring <dan.luhring@anchore.com>
2020-11-06 13:40:31 -05:00
Dan Luhring c878cc361c Update install.sh to handle new macOS releases
Signed-off-by: Dan Luhring <dan.luhring@anchore.com>
2020-11-06 13:39:26 -05:00
Dan Luhring 8fb5b17dbd Use .Version in place of .Tag for release asset name
Signed-off-by: Dan Luhring <dan.luhring@anchore.com>
2020-11-06 13:36:02 -05:00
Alex Goodman b6eb589b78 Merge pull request #251 from anchore/add-rpm-file-info
Add RPM file info sourced from the RPM DB
2020-11-06 11:06:34 -05:00
Alex Goodman 6aba2f48d4 split out rpmdb file processing to helper + lint fixes
Signed-off-by: Alex Goodman <alex.goodman@anchore.com>
2020-11-06 10:44:27 -05:00
Alex Goodman 0205e72be9 ensure resolvers ignore directories for "FilesBy*" methods
Signed-off-by: Alex Goodman <alex.goodman@anchore.com>
2020-11-06 07:21:38 -05:00
Alex Goodman 8095cd9980 add rpmdb file info to cataloger
Signed-off-by: Alex Goodman <alex.goodman@anchore.com>
2020-11-06 07:21:23 -05:00
Dan Luhring ecfc471ce5 Resolve security warning for macOS users (#249)
* Add support for macOS signing and notarization

Signed-off-by: Dan Luhring <dan.luhring@anchore.com>

* Use Docker to run the changelog generator locally

Signed-off-by: Dan Luhring <dan.luhring@anchore.com>
2020-11-04 15:47:55 -05:00
Alex Goodman a52750bdd3 enhance alpine file discovery (#248)
Signed-off-by: Alex Goodman <alex.goodman@anchore.com>
2020-11-04 09:56:52 -05:00
76 changed files with 2392 additions and 304 deletions
+31
View File
@@ -0,0 +1,31 @@
#!/usr/bin/env bash
set -eu
CI_HOME="/Users/runner"
if [[ "${HOME}" != "${CI_HOME}" ]]; then
printf "WARNING! It looks like this isn't the CI environment. This script modifies the macOS Keychain setup in ways you probably wouldn't want for your own machine. It also requires an Apple Developer ID Certificate that you shouldn't have outside of the CI environment.\n\nExiting early to make sure nothing bad happens.\n"
exit 1
fi
# Install gon (see https://github.com/mitchellh/gon for details).
brew tap mitchellh/gon
brew install mitchellh/gon/gon
# Write signing certificate to disk from environment variable.
CERT_FILE="$HOME/developer_id_certificate.p12"
echo -n "$APPLE_DEVELOPER_ID_CERT" | base64 --decode > "$CERT_FILE"
# In order to have all keychain interactions avoid an interactive user prompt, we need to control the password for the keychain in question, which means we need to create a new keychain into which we'll import the signing certificate and from which we'll later access this certificate during code signing.
EPHEMERAL_KEYCHAIN="ci-ephemeral-keychain"
EPHEMERAL_KEYCHAIN_PASSWORD="$(openssl rand -base64 100)"
security create-keychain -p "${EPHEMERAL_KEYCHAIN_PASSWORD}" "${EPHEMERAL_KEYCHAIN}"
# Import signing certificate into the keychain. (This is a pre-requisite for gon, which is invoked via goreleaser.)
EPHEMERAL_KEYCHAIN_FULL_PATH="$HOME/Library/Keychains/${EPHEMERAL_KEYCHAIN}-db"
security import "${CERT_FILE}" -k "${EPHEMERAL_KEYCHAIN_FULL_PATH}" -P "${APPLE_DEVELOPER_ID_CERT_PASS}" -T "$(command -v codesign)"
# Setting the partition list for this certificate's private key to include "apple-tool:" and "apple:" allows the codesign command to access this keychain item without an interactive user prompt. (codesign is invoked by gon.)
security set-key-partition-list -S "apple-tool:,apple:" -s -k "${EPHEMERAL_KEYCHAIN_PASSWORD}" "${EPHEMERAL_KEYCHAIN_FULL_PATH}"
# Make this new keychain the user's default keychain, so that codesign will be able to find this certificate when we specify it during signing.
security default-keychain -d "user" -s "${EPHEMERAL_KEYCHAIN_FULL_PATH}"
+19
View File
@@ -0,0 +1,19 @@
#!/usr/bin/env bash
set -eu
IS_SNAPSHOT="$1" # e.g. "true", "false"
if [[ "${IS_SNAPSHOT}" == "true" ]]; then
# This is a snapshot build —— skipping signing and notarization...
exit 0
fi
GON_CONFIG="$2" # e.g. "gon.hcl"
NEW_NAME_WITHOUT_EXTENSION="$3" # e.g. "./dist/syft-0.1.0"
ORIGINAL_NAME_WITHOUT_EXTENSION="./dist/output" # This should match dmg and zip output_path in the gon config file, without the extension.
gon "${GON_CONFIG}"
# Rename outputs with specified desired name
mv -v "${ORIGINAL_NAME_WITHOUT_EXTENSION}.dmg" "${NEW_NAME_WITHOUT_EXTENSION}.dmg"
mv -v "${ORIGINAL_NAME_WITHOUT_EXTENSION}.zip" "${NEW_NAME_WITHOUT_EXTENSION}.zip"
+4 -9
View File
@@ -15,14 +15,9 @@ fi
echo "creating and publishing version file"
# create a version file for version-update checks
echo "${VERSION}" | tee ${DISTDIR}/VERSION
VERSION_FILE="${DISTDIR}/VERSION"
echo "${VERSION}" | tee "${VERSION_FILE}"
# upload the version file that supports the application version update check
docker run --rm \
-i \
-e AWS_DEFAULT_REGION=us-west-2 \
-e AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID} \
-e AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY} \
-v $(pwd)/${DISTDIR}/:/distmount \
amazon/aws-cli \
s3 cp /distmount/VERSION s3://toolbox-data.anchore.io/${BIN}/releases/latest/VERSION
export AWS_DEFAULT_REGION=us-west-2
aws s3 cp "${VERSION_FILE}" s3://toolbox-data.anchore.io/${BIN}/releases/latest/VERSION
+2 -6
View File
@@ -14,8 +14,7 @@ env:
jobs:
Build-Snapshot-Artifacts:
# come Nov 30 2020 ubuntu-latest will be ubuntu-20.04, until then it needs to be explicitly referenced due to python 3.7 specific features being used
runs-on: ubuntu-20.04
runs-on: macos-latest # We're creating these snapshot builds on macOS to be consistent with our release workflow's build process, which also takes place on macOS (due to code signing requirements).
steps:
- uses: actions/setup-go@v2
@@ -40,9 +39,6 @@ jobs:
if: steps.bootstrap-cache.outputs.cache-hit != 'true'
run: make bootstrap
- name: Bootstrap CI dependencies
run: make ci-bootstrap
- name: Import GPG key
id: import_gpg
uses: crazy-max/ghaction-import-gpg@v2
@@ -162,4 +158,4 @@ jobs:
text: The syft acceptance tests have failed tragically!
env:
SLACK_WEBHOOK_URL: ${{ secrets.SLACK_TOOLBOX_WEBHOOK_URL }}
if: ${{ failure() }}
if: ${{ failure() }}
+8 -7
View File
@@ -13,7 +13,7 @@ env:
jobs:
wait-for-checks:
runs-on: ubuntu-latest
runs-on: ubuntu-latest # This OS choice is arbitrary. None of the steps in this job are specific to either Linux or macOS.
steps:
- uses: actions/checkout@v2
@@ -82,7 +82,7 @@ jobs:
release:
needs: [ wait-for-checks ]
runs-on: ubuntu-latest
runs-on: macos-latest # Due to our code signing process, it's vital that we run our release steps on macOS.
steps:
- uses: actions/setup-go@v2
@@ -93,6 +93,7 @@ jobs:
with:
fetch-depth: 0
# We are expecting this cache to have been created during the "Build-Snapshot-Artifacts" job in the "Acceptance" workflow.
- name: Restore bootstrap cache
id: cache
uses: actions/cache@v2
@@ -105,10 +106,6 @@ jobs:
${{ runner.os }}-go-${{ env.GO_VERSION }}-${{ hashFiles('**/go.sum') }}-
${{ runner.os }}-go-${{ env.GO_VERSION }}-
- name: Bootstrap dependencies
if: steps.cache.outputs.cache-hit != 'true'
run: make ci-bootstrap
- name: Import GPG key
id: import_gpg
uses: crazy-max/ghaction-import-gpg@v2
@@ -132,6 +129,10 @@ jobs:
SIGNING_FINGERPRINT: ${{ steps.import_gpg.outputs.fingerprint }}
AWS_ACCESS_KEY_ID: ${{ secrets.TOOLBOX_AWS_ACCESS_KEY_ID }}
AWS_SECRET_ACCESS_KEY: ${{ secrets.TOOLBOX_AWS_SECRET_ACCESS_KEY }}
APPLE_DEVELOPER_ID_CERT: ${{ secrets.APPLE_DEVELOPER_ID_CERT }} # Used during macOS code signing.
APPLE_DEVELOPER_ID_CERT_PASS: ${{ secrets.APPLE_DEVELOPER_ID_CERT_PASS }} # Used during macOS code signing.
AC_USERNAME: ${{ secrets.ENG_CI_APPLE_ID }} # Used during macOS notarization.
AC_PASSWORD: ${{ secrets.ENG_CI_APPLE_ID_PASS }} # Used during macOS notarization.
- uses: 8398a7/action-slack@v3
with:
@@ -145,4 +146,4 @@ jobs:
- uses: actions/upload-artifact@v2
with:
name: artifacts
path: dist/**/*
path: dist/**/*
+47 -11
View File
@@ -6,18 +6,33 @@ release:
# If set to true, will not auto-publish the release. This is done to allow us to review the changelog before publishing.
draft: true
signs:
- artifacts: checksum
args: ["--output", "${signature}", "--detach-sign", "${artifact}"]
builds:
- binary: syft
id: syft
env:
- CGO_ENABLED=0
goos:
# windows not supported yet (due to jotframe)
# - windows
- linux
goarch:
- amd64
# Set the modified timestamp on the output binary to the git timestamp (to ensure a reproducible build)
mod_timestamp: '{{ .CommitTimestamp }}'
ldflags: |
-w
-s
-extldflags '-static'
-X github.com/anchore/syft/internal/version.version={{.Version}}
-X github.com/anchore/syft/internal/version.gitCommit={{.Commit}}
-X github.com/anchore/syft/internal/version.buildDate={{.Date}}
-X github.com/anchore/syft/internal/version.gitTreeState={{.Env.BUILD_GIT_TREE_STATE}}
# For more info on this macOS build, see: https://github.com/mitchellh/gon#usage-with-goreleaser
- binary: syft
id: syft-macos
env:
- CGO_ENABLED=0
goos:
- darwin
goarch:
- amd64
@@ -32,6 +47,34 @@ builds:
-X github.com/anchore/syft/internal/version.buildDate={{.Date}}
-X github.com/anchore/syft/internal/version.gitTreeState={{.Env.BUILD_GIT_TREE_STATE}}
archives:
- format: tar.gz
builds:
- syft # i.e. Linux only
- format: zip # This is a hack! We don't actually intend to use _this_ ZIP file, we just need goreleaser to consider the ZIP file produced by gon (which will have the same file name) to be an artifact so we can use it downstream in publishing (e.g. to a homebrew tap)
id: syft-zip
builds:
- syft-macos
signs:
- artifacts: checksum
cmd: sh
args:
- '-c'
# we should not include the zip artifact, as the artifact is mutated throughout the next macOS notarization step
# note: sed -i is not portable
- 'sed "/.*\.zip/d" ${artifact} > tmpfile && mv tmpfile ${artifact} && gpg --output ${signature} --detach-sign ${artifact}'
- id: syft-macos-signing
ids:
- syft-macos
cmd: ./.github/scripts/mac-sign-and-notarize.sh
signature: "syft_${VERSION}_darwin_amd64.dmg" # This is somewhat unintuitive. This gets the DMG file recognized as an artifact. In fact, both a DMG and a ZIP file are being produced by this signing step.
args:
- "{{ .IsSnapshot }}"
- "gon.hcl"
- "./dist/syft_{{ .Version }}_darwin_amd64"
artifacts: all
nfpms:
- license: "Apache 2.0"
maintainer: "Anchore, Inc"
@@ -47,10 +90,3 @@ brews:
name: homebrew-syft
homepage: *website
description: *description
archives:
- format: tar.gz
format_overrides:
- goos: windows
format: zip
+22 -11
View File
@@ -74,6 +74,10 @@ help:
ci-bootstrap:
DEBIAN_FRONTEND=noninteractive sudo apt update && sudo -E apt install -y bc jq libxml2-utils
.PHONY:
ci-bootstrap-mac:
github_changelog_generator --version || sudo gem install github_changelog_generator
.PHONY: bootstrap
bootstrap: ## Download and install all go dependencies (+ prep tooling in the ./tmp dir)
$(call title,Bootstrapping dependencies)
@@ -226,8 +230,7 @@ acceptance-test-rpm-package-install: $(SNAPSHOTDIR)
changelog-release:
@echo "Last tag: $(SECOND_TO_LAST_TAG)"
@echo "Current tag: $(VERSION)"
@docker run -i --rm \
-v "$(shell pwd)":/usr/local/src/your-app ferrarimarco/github-changelog-generator \
@github_changelog_generator \
--user anchore \
--project $(BIN) \
-t ${GITHUB_TOKEN} \
@@ -241,8 +244,9 @@ changelog-release:
.PHONY: changelog-unreleased
changelog-unreleased: ## show the current changelog that will be produced on the next release (note: requires GITHUB_TOKEN set)
@docker run -it --rm \
-v "$(shell pwd)":/usr/local/src/your-app ferrarimarco/github-changelog-generator \
@docker run -it --rm \
-v "$(shell pwd)":/usr/local/src/your-app \
ferrarimarco/github-changelog-generator \
--user anchore \
--project $(BIN) \
-t ${GITHUB_TOKEN} \
@@ -258,17 +262,24 @@ changelog-unreleased: ## show the current changelog that will be produced on the
/CHANGELOG.md
.PHONY: release
release: clean-dist changelog-release ## Build and publish final binaries and packages
release: clean-dist ci-bootstrap-mac changelog-release ## Build and publish final binaries and packages. Intended to be run only on macOS.
$(call title,Publishing release artifacts)
# Prepare for macOS-specific signing process
.github/scripts/mac-prepare-for-signing.sh
# create a config with the dist dir overridden
echo "dist: $(DISTDIR)" > $(TEMPDIR)/goreleaser.yaml
cat .goreleaser.yaml >> $(TEMPDIR)/goreleaser.yaml
# release
bash -c "BUILD_GIT_TREE_STATE=$(GITTREESTATE) $(TEMPDIR)/goreleaser \
--rm-dist \
--config $(TEMPDIR)/goreleaser.yaml \
--release-notes <(cat CHANGELOG.md)"
# release (note the version transformation from v0.7.0 --> 0.7.0)
bash -c "\
BUILD_GIT_TREE_STATE=$(GITTREESTATE) \
VERSION=$(VERSION:v%=%) \
$(TEMPDIR)/goreleaser \
--rm-dist \
--config $(TEMPDIR)/goreleaser.yaml \
--release-notes <(cat CHANGELOG.md)"
# verify checksum signatures
.github/scripts/verify-signature.sh "$(DISTDIR)"
@@ -290,4 +301,4 @@ clean-dist:
.PHONY: clean-json-schema-examples
clean-json-schema-examples:
rm -f schema/json/examples/*
rm -f schema/json/examples/*
+2 -7
View File
@@ -54,7 +54,7 @@ Where the `format`s available are:
## Installation
**Recommended**
**Recommended (macOS and Linux)**
```bash
# install the latest version to /usr/local/bin
curl -sSfL https://raw.githubusercontent.com/anchore/syft/main/install.sh | sh -s -- -b /usr/local/bin
@@ -63,17 +63,12 @@ curl -sSfL https://raw.githubusercontent.com/anchore/syft/main/install.sh | sh -
curl -sSfL https://raw.githubusercontent.com/anchore/syft/main/install.sh | sh -s -- -b <SOME_BIN_PATH> <RELEASE_VERSION>
```
**macOS**
**Homebrew (macOS)**
```bash
brew tap anchore/syft
brew install syft
```
You may experience a "macOS cannot verify app is free from malware" error upon running Syft because it is not yet signed and notarized. You can override this using `xattr`.
```bash
xattr -rd com.apple.quarantine syft
```
## Configuration
Configuration search paths:
+3 -3
View File
@@ -4,14 +4,14 @@ go 1.14
require (
github.com/adrg/xdg v0.2.1
github.com/anchore/go-rpmdb v0.0.0-20200811175839-cbc751c28e8e
github.com/anchore/go-rpmdb v0.0.0-20201106153645-0043963c2e12
github.com/anchore/go-testutils v0.0.0-20200925183923-d5f45b0d3c04
github.com/anchore/go-version v1.2.2-0.20200701162849-18adb9c92b9b
github.com/anchore/stereoscope v0.0.0-20200925184903-c82da54e98fe
github.com/anchore/stereoscope v0.0.0-20201106140100-12e75c48f409
github.com/bmatcuk/doublestar v1.3.3
github.com/docker/docker v17.12.0-ce-rc1.0.20200309214505-aa6a9891b09c+incompatible
github.com/dustin/go-humanize v1.0.0
github.com/go-test/deep v1.0.6
github.com/go-test/deep v1.0.7
github.com/google/uuid v1.1.1
github.com/gookit/color v1.2.7
github.com/hashicorp/go-multierror v1.1.0
+6
View File
@@ -126,12 +126,16 @@ github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRF
github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho=
github.com/anchore/go-rpmdb v0.0.0-20200811175839-cbc751c28e8e h1:kty6r0R2JeaNPeWKSYDC+HW3hkqwFh4PP5TQ8pUPYFw=
github.com/anchore/go-rpmdb v0.0.0-20200811175839-cbc751c28e8e/go.mod h1:iYuIG0Nai8dR0ri3LhZQKUyO1loxUWAGvoWhXDmjy1A=
github.com/anchore/go-rpmdb v0.0.0-20201106153645-0043963c2e12 h1:xbeIbn5F52JVx3RUIajxCj8b0y+9lywspql4sFhcxWQ=
github.com/anchore/go-rpmdb v0.0.0-20201106153645-0043963c2e12/go.mod h1:juoyWXIj7sJ1IDl4E/KIfyLtovbs5XQVSIdaQifFQT8=
github.com/anchore/go-testutils v0.0.0-20200925183923-d5f45b0d3c04 h1:VzprUTpc0vW0nnNKJfJieyH/TZ9UYAnTZs5/gHTdAe8=
github.com/anchore/go-testutils v0.0.0-20200925183923-d5f45b0d3c04/go.mod h1:6dK64g27Qi1qGQZ67gFmBFvEHScy0/C8qhQhNe5B5pQ=
github.com/anchore/go-version v1.2.2-0.20200701162849-18adb9c92b9b h1:e1bmaoJfZVsCYMrIZBpFxwV26CbsuoEh5muXD5I1Ods=
github.com/anchore/go-version v1.2.2-0.20200701162849-18adb9c92b9b/go.mod h1:Bkc+JYWjMCF8OyZ340IMSIi2Ebf3uwByOk6ho4wne1E=
github.com/anchore/stereoscope v0.0.0-20200925184903-c82da54e98fe h1:m4NSyTo2fVUoUHAV/ZVqE/PFMr/y8oz9HRrhWLk9It0=
github.com/anchore/stereoscope v0.0.0-20200925184903-c82da54e98fe/go.mod h1:2Jja/4l0zYggW52og+nn0rut4i+OYjCf9vTyrM8RT4E=
github.com/anchore/stereoscope v0.0.0-20201106140100-12e75c48f409 h1:xKSpDRjmYrEFrdMeDh4AuSUAFc99pdro6YFBKxy2um0=
github.com/anchore/stereoscope v0.0.0-20201106140100-12e75c48f409/go.mod h1:2Jja/4l0zYggW52og+nn0rut4i+OYjCf9vTyrM8RT4E=
github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883/go.mod h1:rCTlJbsFo29Kk6CurOXKm700vrz8f0KW0JNfpkRJY/8=
github.com/apex/log v1.1.4/go.mod h1:AlpoD9aScyQfJDVHmLMEcx4oU6LqzkWp4Mg9GdAcEvQ=
github.com/apex/log v1.3.0 h1:1fyfbPvUwD10nMoh3hY6MXzvZShJQn9/ck7ATgAt5pA=
@@ -299,6 +303,8 @@ github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LB
github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
github.com/go-test/deep v1.0.6 h1:UHSEyLZUwX9Qoi99vVwvewiMC8mM2bf7XEM2nqvzEn8=
github.com/go-test/deep v1.0.6/go.mod h1:QV8Hv/iy04NyLBxAdO9njL0iVPN1S4d/A3NVv1V36o8=
github.com/go-test/deep v1.0.7 h1:/VSMRlnY/JSyqxQUzQLKVMAskpY/NZKFA5j2P+0pP2M=
github.com/go-test/deep v1.0.7/go.mod h1:QV8Hv/iy04NyLBxAdO9njL0iVPN1S4d/A3NVv1V36o8=
github.com/go-toolsmith/astcast v1.0.0 h1:JojxlmI6STnFVG9yOImLeGREv8W2ocNUM+iOhR6jE7g=
github.com/go-toolsmith/astcast v1.0.0/go.mod h1:mt2OdQTeAQcY4DQgPSArJjHCcOwlX+Wl/kwN+LbLGQ4=
github.com/go-toolsmith/astcopy v1.0.0 h1:OMgl1b1MEpjFQ1m5ztEO06rz5CUd3oBv9RF7+DyvdG8=
+15
View File
@@ -0,0 +1,15 @@
source = ["./dist/syft-macos_darwin_amd64/syft"] # The 'dist' directory path should ideally reference an env var, where the source of truth is the Makefile. I wasn't able to figure out how to solve this.
bundle_id = "com.anchore.toolbox.syft"
sign {
application_identity = "Developer ID Application: ANCHORE, INC. (9MJHKYX5AT)"
}
dmg {
output_path = "./dist/output.dmg"
volume_name = "Syft"
}
zip {
output_path = "./dist/output.zip"
}
+27 -13
View File
@@ -1,6 +1,6 @@
#!/bin/sh
set -e
# Code generated by godownloader on 2020-08-10T20:55:46Z. DO NOT EDIT.
# Code generated by godownloader on 2020-08-10T20:55:46Z.
#
usage() {
@@ -45,11 +45,16 @@ parse_args() {
execute() {
tmpdir=$(mktemp -d)
log_debug "downloading files into ${tmpdir}"
http_download "${tmpdir}/${TARBALL}" "${TARBALL_URL}"
http_download "${tmpdir}/${ARCHIVE}" "${ARCHIVE_URL}"
http_download "${tmpdir}/${CHECKSUM}" "${CHECKSUM_URL}"
hash_sha256_verify "${tmpdir}/${TARBALL}" "${tmpdir}/${CHECKSUM}"
# macOS has its own secure verification mechanism, and checksums.txt is not used.
if [ "$OS" != "darwin" ]; then
hash_sha256_verify "${tmpdir}/${ARCHIVE}" "${tmpdir}/${CHECKSUM}"
fi
srcdir="${tmpdir}"
(cd "${tmpdir}" && untar "${TARBALL}")
(cd "${tmpdir}" && unpack "${ARCHIVE}")
test ! -d "${BINDIR}" && install -d "${BINDIR}"
for binexe in $BINARIES; do
if [ "$OS" = "windows" ]; then
@@ -89,6 +94,7 @@ tag_to_version() {
adjust_format() {
# change format (tar.gz or zip) based on OS
case ${OS} in
darwin) FORMAT=dmg ;;
windows) FORMAT=zip ;;
esac
true
@@ -221,18 +227,26 @@ uname_arch_check() {
log_crit "uname_arch_check '$(uname -m)' got converted to '$arch' which is not a GOARCH value. Please file bug report at https://github.com/client9/shlib"
return 1
}
untar() {
tarball=$1
case "${tarball}" in
*.tar.gz | *.tgz) tar --no-same-owner -xzf "${tarball}" ;;
*.tar) tar --no-same-owner -xf "${tarball}" ;;
*.zip) unzip "${tarball}" ;;
unpack() {
archive=$1
case "${archive}" in
*.tar.gz | *.tgz) tar --no-same-owner -xzf "${archive}" ;;
*.tar) tar --no-same-owner -xf "${archive}" ;;
*.zip) unzip "${archive}" ;;
*.dmg) extract_from_dmg "${archive}" ;;
*)
log_err "untar unknown archive format for ${tarball}"
log_err "unpack unknown archive format for ${archive}"
return 1
;;
esac
}
extract_from_dmg() {
dmg_file=$1
mount_point="/Volumes/tmp-dmg"
hdiutil attach -quiet -nobrowse -mountpoint "${mount_point}" "${dmg_file}"
cp -fR "${mount_point}/" ./
hdiutil detach -quiet -force "${mount_point}"
}
http_download_curl() {
local_file=$1
source_url=$2
@@ -366,8 +380,8 @@ adjust_arch
log_info "found version: ${VERSION} for ${TAG}/${OS}/${ARCH}"
NAME=${PROJECT_NAME}_${VERSION}_${OS}_${ARCH}
TARBALL=${NAME}.${FORMAT}
TARBALL_URL=${GITHUB_DOWNLOAD}/${TAG}/${TARBALL}
ARCHIVE=${NAME}.${FORMAT}
ARCHIVE_URL=${GITHUB_DOWNLOAD}/${TAG}/${ARCHIVE}
CHECKSUM=${PROJECT_NAME}_${VERSION}_checksums.txt
CHECKSUM_URL=${GITHUB_DOWNLOAD}/${TAG}/${CHECKSUM}
+1 -1
View File
@@ -111,7 +111,7 @@ func (cfg *Application) Build() error {
case v >= 2:
cfg.Log.LevelOpt = logrus.DebugLevel
default:
cfg.Log.LevelOpt = logrus.ErrorLevel
cfg.Log.LevelOpt = logrus.WarnLevel
}
}
}
+12
View File
@@ -8,6 +8,18 @@ func Errorf(format string, args ...interface{}) {
Log.Errorf(format, args...)
}
func Error(args ...interface{}) {
Log.Error(args...)
}
func Warnf(format string, args ...interface{}) {
Log.Warnf(format, args...)
}
func Warn(args ...interface{}) {
Log.Warn(args...)
}
func Infof(format string, args ...interface{}) {
Log.Infof(format, args...)
}
+3
View File
@@ -3,6 +3,9 @@ package log
type nopLogger struct{}
func (l *nopLogger) Errorf(format string, args ...interface{}) {}
func (l *nopLogger) Error(args ...interface{}) {}
func (l *nopLogger) Warnf(format string, args ...interface{}) {}
func (l *nopLogger) Warn(args ...interface{}) {}
func (l *nopLogger) Infof(format string, args ...interface{}) {}
func (l *nopLogger) Info(args ...interface{}) {}
func (l *nopLogger) Debugf(format string, args ...interface{}) {}
+28 -4
View File
@@ -84,6 +84,14 @@ func (l *LogrusLogger) Infof(format string, args ...interface{}) {
l.Logger.Infof(format, args...)
}
func (l *LogrusLogger) Warnf(format string, args ...interface{}) {
l.Logger.Warnf(format, args...)
}
func (l *LogrusLogger) Errorf(format string, args ...interface{}) {
l.Logger.Errorf(format, args...)
}
func (l *LogrusLogger) Debug(args ...interface{}) {
l.Logger.Debug(args...)
}
@@ -92,8 +100,12 @@ func (l *LogrusLogger) Info(args ...interface{}) {
l.Logger.Info(args...)
}
func (l *LogrusLogger) Errorf(format string, args ...interface{}) {
l.Logger.Errorf(format, args...)
func (l *LogrusLogger) Warn(args ...interface{}) {
l.Logger.Warn(args...)
}
func (l *LogrusLogger) Error(args ...interface{}) {
l.Logger.Error(args...)
}
func (l *LogrusNestedLogger) Debugf(format string, args ...interface{}) {
@@ -104,6 +116,14 @@ func (l *LogrusNestedLogger) Infof(format string, args ...interface{}) {
l.Logger.Infof(format, args...)
}
func (l *LogrusNestedLogger) Warnf(format string, args ...interface{}) {
l.Logger.Warnf(format, args...)
}
func (l *LogrusNestedLogger) Errorf(format string, args ...interface{}) {
l.Logger.Errorf(format, args...)
}
func (l *LogrusNestedLogger) Debug(args ...interface{}) {
l.Logger.Debug(args...)
}
@@ -112,6 +132,10 @@ func (l *LogrusNestedLogger) Info(args ...interface{}) {
l.Logger.Info(args...)
}
func (l *LogrusNestedLogger) Errorf(format string, args ...interface{}) {
l.Logger.Errorf(format, args...)
func (l *LogrusNestedLogger) Warn(args ...interface{}) {
l.Logger.Warn(args...)
}
func (l *LogrusNestedLogger) Error(args ...interface{}) {
l.Logger.Error(args...)
}
+58 -37
View File
@@ -10,6 +10,19 @@
},
"type": "array"
},
"licenses": {
"anyOf": [
{
"type": "null"
},
{
"items": {
"type": "string"
},
"type": "array"
}
]
},
"locations": {
"items": {
"anyOf": [
@@ -53,55 +66,62 @@
"type": "integer"
},
"files": {
"items": {
"anyOf": [
{
"type": "string"
},
{
"properties": {
"checksum": {
"anyOf": [
{
"type": "null"
},
{
"items": {
"anyOf": [
{
"type": "string"
},
"digest": {
{
"properties": {
"algorithm": {
"checksum": {
"type": "string"
},
"value": {
"digest": {
"properties": {
"algorithm": {
"type": "string"
},
"value": {
"type": "string"
}
},
"required": [
"algorithm",
"value"
],
"type": "object"
},
"ownerGid": {
"type": "string"
},
"ownerUid": {
"type": "string"
},
"path": {
"type": "string"
},
"permissions": {
"type": "string"
},
"size": {
"type": "string"
}
},
"required": [
"algorithm",
"value"
"path"
],
"type": "object"
},
"ownerGid": {
"type": "string"
},
"ownerUid": {
"type": "string"
},
"path": {
"type": "string"
},
"permissions": {
"type": "string"
},
"size": {
"type": "string"
}
},
"required": [
"path"
],
"type": "object"
}
]
},
"type": "array"
]
},
"type": "array"
}
]
},
"gitCommitOfApkPort": {
"type": "string"
@@ -307,6 +327,7 @@
},
"required": [
"foundBy",
"licenses",
"locations",
"name",
"type",
+15 -6
View File
@@ -89,7 +89,16 @@ func parseApkDBEntry(reader io.Reader) (*pkg.ApkMetadata, error) {
switch key {
case "F":
lastFile = "/" + value
currentFile := "/" + value
newFileRecord := pkg.ApkFileRecord{
Path: currentFile,
}
files = append(files, newFileRecord)
fileRecord = &files[len(files)-1]
// future aux references are relative to previous "F" records
lastFile = currentFile
continue
case "R":
newFileRecord := pkg.ApkFileRecord{
@@ -97,24 +106,24 @@ func parseApkDBEntry(reader io.Reader) (*pkg.ApkMetadata, error) {
}
files = append(files, newFileRecord)
fileRecord = &files[len(files)-1]
case "a":
case "a", "M":
ownershipFields := strings.Split(value, ":")
if len(ownershipFields) < 3 {
log.Errorf("unexpected APK ownership field: %q", value)
log.Warnf("unexpected APK ownership field: %q", value)
continue
}
if fileRecord == nil {
log.Errorf("ownership field with no parent record: %q", value)
log.Warnf("ownership field with no parent record: %q", value)
continue
}
fileRecord.OwnerUID = ownershipFields[0]
fileRecord.OwnerGUI = ownershipFields[1]
fileRecord.OwnerGID = ownershipFields[1]
fileRecord.Permissions = ownershipFields[2]
// note: there are more optional fields available that we are not capturing, e.g.:
// "0:0:755:Q1JaDEHQHBbizhEzoWK1YxuraNU/4="
case "Z":
if fileRecord == nil {
log.Errorf("checksum field with no parent record: %q", value)
log.Warnf("checksum field with no parent record: %q", value)
continue
}
fileRecord.Checksum = value
+418 -17
View File
@@ -19,10 +19,25 @@ func TestExtraFileAttributes(t *testing.T) {
name: "test extra file attributes (checksum) are ignored",
expected: pkg.ApkMetadata{
Files: []pkg.ApkFileRecord{
{
Path: "/usr",
},
{
Path: "/usr/lib",
},
{
Path: "/usr/lib/jvm",
},
{
Path: "/usr/lib/jvm/java-1.8-openjdk",
},
{
Path: "/usr/lib/jvm/java-1.8-openjdk/bin",
},
{
Path: "/usr/lib/jvm/java-1.8-openjdk/bin/policytool",
OwnerUID: "0",
OwnerGUI: "0",
OwnerGID: "0",
Permissions: "755",
Checksum: "Q1M0C9qfC/+kdRiOodeihG2GMRtkE=",
},
@@ -60,13 +75,13 @@ func TestExtraFileAttributes(t *testing.T) {
}
}
func TestSinglePackage(t *testing.T) {
func TestSinglePackageDetails(t *testing.T) {
tests := []struct {
name string
fixture string
expected pkg.ApkMetadata
}{
{
name: "Test Single Package",
fixture: "test-fixtures/single",
expected: pkg.ApkMetadata{
Package: "musl-utils",
OriginPackage: "musl",
@@ -82,51 +97,428 @@ func TestSinglePackage(t *testing.T) {
PullChecksum: "Q1bTtF5526tETKfL+lnigzIDvm+2o=",
GitCommitOfAport: "4024cc3b29ad4c65544ad068b8f59172b5494306",
Files: []pkg.ApkFileRecord{
{
Path: "/sbin",
},
{
Path: "/sbin/ldconfig",
OwnerUID: "0",
OwnerGUI: "0",
OwnerGID: "0",
Permissions: "755",
Checksum: "Q1Kja2+POZKxEkUOZqwSjC6kmaED4=",
},
{
Path: "/usr",
},
{
Path: "/usr/bin",
},
{
Path: "/usr/bin/iconv",
OwnerUID: "0",
OwnerGUI: "0",
OwnerGID: "0",
Permissions: "755",
Checksum: "Q1CVmFbdY+Hv6/jAHl1gec2Kbx1EY=",
},
{
Path: "/usr/bin/ldd",
OwnerUID: "0",
OwnerGUI: "0",
OwnerGID: "0",
Permissions: "755",
Checksum: "Q1yFAhGggmL7ERgbIA7KQxyTzf3ks=",
},
{
Path: "/usr/bin/getconf",
OwnerUID: "0",
OwnerGUI: "0",
OwnerGID: "0",
Permissions: "755",
Checksum: "Q1dAdYK8M/INibRQF5B3Rw7cmNDDA=",
},
{
Path: "/usr/bin/getent",
OwnerUID: "0",
OwnerGUI: "0",
OwnerGID: "0",
Permissions: "755",
Checksum: "Q1eR2Dz/WylabgbWMTkd2+hGmEya4=",
},
},
},
},
{
fixture: "test-fixtures/base",
expected: pkg.ApkMetadata{
Package: "alpine-baselayout",
OriginPackage: "alpine-baselayout",
Version: "3.2.0-r6",
Description: "Alpine base dir structure and init scripts",
Maintainer: "Natanael Copa <ncopa@alpinelinux.org>",
License: "GPL-2.0-only",
Architecture: "x86_64",
URL: "https://git.alpinelinux.org/cgit/aports/tree/main/alpine-baselayout",
Size: 19917,
InstalledSize: 409600,
PullDependencies: "/bin/sh so:libc.musl-x86_64.so.1",
PullChecksum: "Q1myMNfd7u5v5UTgNHeq1e31qTjZU=",
GitCommitOfAport: "e1c51734fa96fa4bac92e9f14a474324c67916fc",
Files: []pkg.ApkFileRecord{
{
Path: "/dev",
},
{
Path: "/dev/pts",
},
{
Path: "/dev/shm",
},
{
Path: "/etc",
},
{
Path: "/etc/fstab",
Checksum: "Q11Q7hNe8QpDS531guqCdrXBzoA/o=",
},
{
Path: "/etc/group",
Checksum: "Q1oJ16xWudgKOrXIEquEDzlF2Lsm4=",
},
{
Path: "/etc/hostname",
Checksum: "Q16nVwYVXP/tChvUPdukVD2ifXOmc=",
},
{
Path: "/etc/hosts",
Checksum: "Q1BD6zJKZTRWyqGnPi4tSfd3krsMU=",
},
{
Path: "/etc/inittab",
Checksum: "Q1TsthbhW7QzWRe1E/NKwTOuD4pHc=",
},
{
Path: "/etc/modules",
Checksum: "Q1toogjUipHGcMgECgPJX64SwUT1M=",
},
{
Path: "/etc/motd",
Checksum: "Q1XmduVVNURHQ27TvYp1Lr5TMtFcA=",
},
{
Path: "/etc/mtab",
OwnerUID: "0",
OwnerGID: "0",
Permissions: "777",
Checksum: "Q1kiljhXXH1LlQroHsEJIkPZg2eiw=",
},
{
Path: "/etc/passwd",
Checksum: "Q1TchuuLUfur0izvfZQZxgN/LJhB8=",
},
{
Path: "/etc/profile",
Checksum: "Q1KpFb8kl5LvwXWlY3e58FNsjrI34=",
},
{
Path: "/etc/protocols",
Checksum: "Q13FqXUnvuOpMDrH/6rehxuYAEE34=",
},
{
Path: "/etc/services",
Checksum: "Q1C6HJNgQvLWqt5VY+n7MZJ1rsDuY=",
},
{
Path: "/etc/shadow",
OwnerUID: "0",
OwnerGID: "42",
Permissions: "640",
Checksum: "Q1ltrPIAW2zHeDiajsex2Bdmq3uqA=",
},
{
Path: "/etc/shells",
Checksum: "Q1ojm2YdpCJ6B/apGDaZ/Sdb2xJkA=",
},
{
Path: "/etc/sysctl.conf",
Checksum: "Q14upz3tfnNxZkIEsUhWn7Xoiw96g=",
},
{
Path: "/etc/apk",
},
{
Path: "/etc/conf.d",
},
{
Path: "/etc/crontabs",
},
{
Path: "/etc/crontabs/root",
OwnerUID: "0",
OwnerGID: "0",
Permissions: "600",
Checksum: "Q1vfk1apUWI4yLJGhhNRd0kJixfvY=",
},
{
Path: "/etc/init.d",
},
{
Path: "/etc/modprobe.d",
},
{
Path: "/etc/modprobe.d/aliases.conf",
Checksum: "Q1WUbh6TBYNVK7e4Y+uUvLs/7viqk=",
},
{
Path: "/etc/modprobe.d/blacklist.conf",
Checksum: "Q1xxYGU6S6TLQvb7ervPrWWwAWqMg=",
},
{
Path: "/etc/modprobe.d/i386.conf",
Checksum: "Q1pnay/njn6ol9cCssL7KiZZ8etlc=",
},
{
Path: "/etc/modprobe.d/kms.conf",
Checksum: "Q1ynbLn3GYDpvajba/ldp1niayeog=",
},
{
Path: "/etc/modules-load.d",
},
{
Path: "/etc/network",
},
{
Path: "/etc/network/if-down.d",
},
{
Path: "/etc/network/if-post-down.d",
},
{
Path: "/etc/network/if-pre-up.d",
},
{
Path: "/etc/network/if-up.d",
},
{
Path: "/etc/opt",
},
{
Path: "/etc/periodic",
},
{
Path: "/etc/periodic/15min",
},
{
Path: "/etc/periodic/daily",
},
{
Path: "/etc/periodic/hourly",
},
{
Path: "/etc/periodic/monthly",
},
{
Path: "/etc/periodic/weekly",
},
{
Path: "/etc/profile.d",
},
{
Path: "/etc/profile.d/color_prompt",
Checksum: "Q10wL23GuSCVfumMRgakabUI6EsSk=",
},
{
Path: "/etc/profile.d/locale",
Checksum: "Q1R4bIEpnKxxOSrlnZy9AoawqZ5DU=",
},
{
Path: "/etc/sysctl.d",
},
{
Path: "/home",
},
{
Path: "/lib",
},
{
Path: "/lib/firmware",
},
{
Path: "/lib/mdev",
},
{
Path: "/lib/modules-load.d",
},
{
Path: "/lib/sysctl.d",
},
{
Path: "/lib/sysctl.d/00-alpine.conf",
Checksum: "Q1HpElzW1xEgmKfERtTy7oommnq6c=",
},
{
Path: "/media",
},
{
Path: "/media/cdrom",
},
{
Path: "/media/floppy",
},
{
Path: "/media/usb",
},
{
Path: "/mnt",
},
{
Path: "/opt",
},
{
Path: "/proc",
},
{
Path: "/root",
OwnerUID: "0",
OwnerGID: "0",
Permissions: "700",
},
{
Path: "/run",
},
{
Path: "/sbin",
},
{
Path: "/sbin/mkmntdirs",
OwnerUID: "0",
OwnerGID: "0",
Permissions: "755",
Checksum: "Q1YeuSmC7iDbEWrusPzA/zUQF6YSg=",
},
{
Path: "/srv",
},
{
Path: "/sys",
},
{
Path: "/tmp",
OwnerUID: "0",
OwnerGID: "0",
Permissions: "1777",
},
{
Path: "/usr",
},
{
Path: "/usr/lib",
},
{
Path: "/usr/lib/modules-load.d",
},
{
Path: "/usr/local",
},
{
Path: "/usr/local/bin",
},
{
Path: "/usr/local/lib",
},
{
Path: "/usr/local/share",
},
{
Path: "/usr/sbin",
},
{
Path: "/usr/share",
},
{
Path: "/usr/share/man",
},
{
Path: "/usr/share/misc",
},
{
Path: "/var",
},
{
Path: "/var/run",
OwnerUID: "0",
OwnerGID: "0",
Permissions: "777",
Checksum: "Q11/SNZz/8cK2dSKK+cJpVrZIuF4Q=",
},
{
Path: "/var/cache",
},
{
Path: "/var/cache/misc",
},
{
Path: "/var/empty",
OwnerUID: "0",
OwnerGID: "0",
Permissions: "555",
},
{
Path: "/var/lib",
},
{
Path: "/var/lib/misc",
},
{
Path: "/var/local",
},
{
Path: "/var/lock",
},
{
Path: "/var/lock/subsys",
},
{
Path: "/var/log",
},
{
Path: "/var/mail",
},
{
Path: "/var/opt",
},
{
Path: "/var/spool",
},
{
Path: "/var/spool/mail",
OwnerUID: "0",
OwnerGID: "0",
Permissions: "777",
Checksum: "Q1dzbdazYZA2nTzSIG3YyNw7d4Juc=",
},
{
Path: "/var/spool/cron",
},
{
Path: "/var/spool/cron/crontabs",
OwnerUID: "0",
OwnerGID: "0",
Permissions: "777",
Checksum: "Q1OFZt+ZMp7j0Gny0rqSKuWJyqYmA=",
},
{
Path: "/var/tmp",
OwnerUID: "0",
OwnerGID: "0",
Permissions: "1777",
},
},
},
},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
file, err := os.Open("test-fixtures/single")
t.Run(test.fixture, func(t *testing.T) {
file, err := os.Open(test.fixture)
if err != nil {
t.Fatal("Unable to read test_fixtures/single: ", err)
t.Fatal("Unable to read fixture: ", err)
}
defer func() {
err := file.Close()
@@ -203,38 +595,47 @@ func TestMultiplePackages(t *testing.T) {
PullChecksum: "Q1bTtF5526tETKfL+lnigzIDvm+2o=",
GitCommitOfAport: "4024cc3b29ad4c65544ad068b8f59172b5494306",
Files: []pkg.ApkFileRecord{
{
Path: "/sbin",
},
{
Path: "/sbin/ldconfig",
OwnerUID: "0",
OwnerGUI: "0",
OwnerGID: "0",
Permissions: "755",
Checksum: "Q1Kja2+POZKxEkUOZqwSjC6kmaED4=",
},
{
Path: "/usr",
},
{
Path: "/usr/bin",
},
{
Path: "/usr/bin/iconv",
OwnerUID: "0",
OwnerGUI: "0",
OwnerGID: "0",
Permissions: "755",
Checksum: "Q1CVmFbdY+Hv6/jAHl1gec2Kbx1EY=",
},
{
Path: "/usr/bin/ldd",
OwnerUID: "0",
OwnerGUI: "0",
OwnerGID: "0",
Permissions: "755",
Checksum: "Q1yFAhGggmL7ERgbIA7KQxyTzf3ks=",
},
{
Path: "/usr/bin/getconf",
OwnerUID: "0",
OwnerGUI: "0",
OwnerGID: "0",
Permissions: "755",
Checksum: "Q1dAdYK8M/INibRQF5B3Rw7cmNDDA=",
},
{
Path: "/usr/bin/getent",
OwnerUID: "0",
OwnerGUI: "0",
OwnerGID: "0",
Permissions: "755",
Checksum: "Q1eR2Dz/WylabgbWMTkd2+hGmEya4=",
},
+149
View File
@@ -0,0 +1,149 @@
C:Q1myMNfd7u5v5UTgNHeq1e31qTjZU=
P:alpine-baselayout
V:3.2.0-r6
A:x86_64
S:19917
I:409600
T:Alpine base dir structure and init scripts
U:https://git.alpinelinux.org/cgit/aports/tree/main/alpine-baselayout
L:GPL-2.0-only
o:alpine-baselayout
m:Natanael Copa <ncopa@alpinelinux.org>
t:1590679557
c:e1c51734fa96fa4bac92e9f14a474324c67916fc
D:/bin/sh so:libc.musl-x86_64.so.1
p:cmd:mkmntdirs
F:dev
F:dev/pts
F:dev/shm
F:etc
R:fstab
Z:Q11Q7hNe8QpDS531guqCdrXBzoA/o=
R:group
Z:Q1oJ16xWudgKOrXIEquEDzlF2Lsm4=
R:hostname
Z:Q16nVwYVXP/tChvUPdukVD2ifXOmc=
R:hosts
Z:Q1BD6zJKZTRWyqGnPi4tSfd3krsMU=
R:inittab
Z:Q1TsthbhW7QzWRe1E/NKwTOuD4pHc=
R:modules
Z:Q1toogjUipHGcMgECgPJX64SwUT1M=
R:motd
Z:Q1XmduVVNURHQ27TvYp1Lr5TMtFcA=
R:mtab
a:0:0:777
Z:Q1kiljhXXH1LlQroHsEJIkPZg2eiw=
R:passwd
Z:Q1TchuuLUfur0izvfZQZxgN/LJhB8=
R:profile
Z:Q1KpFb8kl5LvwXWlY3e58FNsjrI34=
R:protocols
Z:Q13FqXUnvuOpMDrH/6rehxuYAEE34=
R:services
Z:Q1C6HJNgQvLWqt5VY+n7MZJ1rsDuY=
R:shadow
a:0:42:640
Z:Q1ltrPIAW2zHeDiajsex2Bdmq3uqA=
R:shells
Z:Q1ojm2YdpCJ6B/apGDaZ/Sdb2xJkA=
R:sysctl.conf
Z:Q14upz3tfnNxZkIEsUhWn7Xoiw96g=
F:etc/apk
F:etc/conf.d
F:etc/crontabs
R:root
a:0:0:600
Z:Q1vfk1apUWI4yLJGhhNRd0kJixfvY=
F:etc/init.d
F:etc/modprobe.d
R:aliases.conf
Z:Q1WUbh6TBYNVK7e4Y+uUvLs/7viqk=
R:blacklist.conf
Z:Q1xxYGU6S6TLQvb7ervPrWWwAWqMg=
R:i386.conf
Z:Q1pnay/njn6ol9cCssL7KiZZ8etlc=
R:kms.conf
Z:Q1ynbLn3GYDpvajba/ldp1niayeog=
F:etc/modules-load.d
F:etc/network
F:etc/network/if-down.d
F:etc/network/if-post-down.d
F:etc/network/if-pre-up.d
F:etc/network/if-up.d
F:etc/opt
F:etc/periodic
F:etc/periodic/15min
F:etc/periodic/daily
F:etc/periodic/hourly
F:etc/periodic/monthly
F:etc/periodic/weekly
F:etc/profile.d
R:color_prompt
Z:Q10wL23GuSCVfumMRgakabUI6EsSk=
R:locale
Z:Q1R4bIEpnKxxOSrlnZy9AoawqZ5DU=
F:etc/sysctl.d
F:home
F:lib
F:lib/firmware
F:lib/mdev
F:lib/modules-load.d
F:lib/sysctl.d
R:00-alpine.conf
Z:Q1HpElzW1xEgmKfERtTy7oommnq6c=
F:media
F:media/cdrom
F:media/floppy
F:media/usb
F:mnt
F:opt
F:proc
F:root
M:0:0:700
F:run
F:sbin
R:mkmntdirs
a:0:0:755
Z:Q1YeuSmC7iDbEWrusPzA/zUQF6YSg=
F:srv
F:sys
F:tmp
M:0:0:1777
F:usr
F:usr/lib
F:usr/lib/modules-load.d
F:usr/local
F:usr/local/bin
F:usr/local/lib
F:usr/local/share
F:usr/sbin
F:usr/share
F:usr/share/man
F:usr/share/misc
F:var
R:run
a:0:0:777
Z:Q11/SNZz/8cK2dSKK+cJpVrZIuF4Q=
F:var/cache
F:var/cache/misc
F:var/empty
M:0:0:555
F:var/lib
F:var/lib/misc
F:var/local
F:var/lock
F:var/lock/subsys
F:var/log
F:var/mail
F:var/opt
F:var/spool
R:mail
a:0:0:777
Z:Q1dzbdazYZA2nTzSIG3YyNw7d4Juc=
F:var/spool/cron
R:crontabs
a:0:0:777
Z:Q1OFZt+ZMp7j0Gny0rqSKuWJyqYmA=
F:var/tmp
M:0:0:1777
+4 -4
View File
@@ -68,7 +68,7 @@ func (c *GenericCataloger) selectFiles(resolver scope.FileResolver) []file.Refer
for path, parser := range c.pathParsers {
files, err := resolver.FilesByPath(file.Path(path))
if err != nil {
log.Errorf("cataloger failed to select files by path: %+v", err)
log.Warnf("cataloger failed to select files by path: %+v", err)
}
if files != nil {
c.register(files, parser)
@@ -79,7 +79,7 @@ func (c *GenericCataloger) selectFiles(resolver scope.FileResolver) []file.Refer
for globPattern, parser := range c.globParsers {
fileMatches, err := resolver.FilesByGlob(globPattern)
if err != nil {
log.Errorf("failed to find files by glob: %s", globPattern)
log.Warnf("failed to find files by glob: %s", globPattern)
}
if fileMatches != nil {
c.register(fileMatches, parser)
@@ -98,14 +98,14 @@ func (c *GenericCataloger) catalog(contents map[file.Reference]string) ([]pkg.Pa
for reference, parser := range c.parsers {
content, ok := contents[reference]
if !ok {
log.Errorf("cataloger '%s' missing file content: %+v", c.upstreamCataloger, reference)
log.Warnf("cataloger '%s' missing file content: %+v", c.upstreamCataloger, reference)
continue
}
entries, err := parser(string(reference.Path), strings.NewReader(content))
if err != nil {
// TODO: should we fail? or only log?
log.Errorf("cataloger '%s' failed to parse entries (reference=%+v): %+v", c.upstreamCataloger, reference, err)
log.Warnf("cataloger '%s' failed to parse entries (reference=%+v): %+v", c.upstreamCataloger, reference, err)
continue
}
+184 -5
View File
@@ -4,14 +4,193 @@ Package dpkg provides a concrete Cataloger implementation for Debian package DB
package deb
import (
"github.com/anchore/syft/syft/cataloger/common"
"fmt"
"io"
"path"
"strings"
"github.com/anchore/stereoscope/pkg/file"
"github.com/anchore/syft/syft/pkg"
"github.com/anchore/syft/syft/scope"
)
const (
dpkgStatusGlob = "**/var/lib/dpkg/status"
md5sumsExt = ".md5sums"
docsPath = "/usr/share/doc"
)
type Cataloger struct{}
// NewDpkgdbCataloger returns a new Deb package cataloger object.
func NewDpkgdbCataloger() *common.GenericCataloger {
globParsers := map[string]common.ParserFn{
"**/var/lib/dpkg/status": parseDpkgStatus,
func NewDpkgdbCataloger() *Cataloger {
return &Cataloger{}
}
// Name returns a string that uniquely describes a cataloger
func (c *Cataloger) Name() string {
return "dpkgdb-cataloger"
}
// Catalog is given an object to resolve file references and content, this function returns any discovered Packages after analyzing dpkg support files.
func (c *Cataloger) Catalog(resolver scope.Resolver) ([]pkg.Package, error) {
dbFileMatches, err := resolver.FilesByGlob(dpkgStatusGlob)
if err != nil {
return nil, fmt.Errorf("failed to find dpkg status files's by glob: %w", err)
}
return common.NewGenericCataloger(nil, globParsers, "dpkgdb-cataloger")
var pkgs []pkg.Package
for _, dbRef := range dbFileMatches {
dbContents, err := resolver.FileContentsByRef(dbRef)
if err != nil {
return nil, err
}
pkgs, err = parseDpkgStatus(strings.NewReader(dbContents))
if err != nil {
return nil, fmt.Errorf("unable to catalog dpkg package=%+v: %w", dbRef.Path, err)
}
md5ContentsByName, md5RefsByName, err := fetchMd5Contents(resolver, dbRef, pkgs)
if err != nil {
return nil, fmt.Errorf("unable to find dpkg md5 contents: %w", err)
}
copyrightContentsByName, copyrightRefsByName, err := fetchCopyrightContents(resolver, dbRef, pkgs)
if err != nil {
return nil, fmt.Errorf("unable to find dpkg copyright contents: %w", err)
}
for i := range pkgs {
p := &pkgs[i]
p.FoundBy = c.Name()
p.Source = []file.Reference{dbRef}
if md5Reader, ok := md5ContentsByName[md5Key(*p)]; ok {
// attach the file list
metadata := p.Metadata.(pkg.DpkgMetadata)
metadata.Files = parseDpkgMD5Info(md5Reader)
p.Metadata = metadata
// keep a record of the file where this was discovered
if ref, ok := md5RefsByName[md5Key(*p)]; ok {
p.Source = append(p.Source, ref)
}
}
copyrightReader, ok := copyrightContentsByName[p.Name]
if ok {
// attach the licenses
p.Licenses = parseLicensesFromCopyright(copyrightReader)
// keep a record of the file where this was discovered
if ref, ok := copyrightRefsByName[p.Name]; ok {
p.Source = append(p.Source, ref)
}
}
}
}
return pkgs, nil
}
func fetchMd5Contents(resolver scope.Resolver, dbRef file.Reference, pkgs []pkg.Package) (map[string]io.Reader, map[string]file.Reference, error) {
// fetch all MD5 file contents. This approach is more efficient than fetching each MD5 file one at a time
var md5FileMatches []file.Reference
var nameByRef = make(map[file.Reference]string)
parentPath, err := dbRef.Path.ParentPath()
if err != nil {
return nil, nil, fmt.Errorf("unable to find parent of path=%+v: %w", dbRef.Path, err)
}
for _, p := range pkgs {
// look for /var/lib/dpkg/info/NAME:ARCH.md5sums
name := md5Key(p)
md5sumPath := path.Join(string(parentPath), "info", name+md5sumsExt)
md5SumRef, err := resolver.RelativeFileByPath(dbRef, md5sumPath)
if err != nil {
return nil, nil, fmt.Errorf("unable to find relative md5sum from path=%+v: %w", dbRef.Path, err)
}
if md5SumRef == nil {
// the most specific key did not work, fallback to just the name
// look for /var/lib/dpkg/info/NAME.md5sums
name := p.Name
md5sumPath := path.Join(string(parentPath), "info", name+md5sumsExt)
md5SumRef, err = resolver.RelativeFileByPath(dbRef, md5sumPath)
if err != nil {
return nil, nil, fmt.Errorf("unable to find relative md5sum from path=%+v: %w", dbRef.Path, err)
}
}
// we should have at least one reference
if md5SumRef != nil {
md5FileMatches = append(md5FileMatches, *md5SumRef)
nameByRef[*md5SumRef] = name
}
}
// fetch the md5 contents
md5ContentsByRef, err := resolver.MultipleFileContentsByRef(md5FileMatches...)
if err != nil {
return nil, nil, err
}
// organize content results and refs by a combination of name and architecture
var contentsByName = make(map[string]io.Reader)
var refsByName = make(map[string]file.Reference)
for ref, contents := range md5ContentsByRef {
name := nameByRef[ref]
contentsByName[name] = strings.NewReader(contents)
refsByName[name] = ref
}
return contentsByName, refsByName, nil
}
func fetchCopyrightContents(resolver scope.Resolver, dbRef file.Reference, pkgs []pkg.Package) (map[string]io.Reader, map[string]file.Reference, error) {
// fetch all copyright file contents. This approach is more efficient than fetching each copyright file one at a time
var copyrightFileMatches []file.Reference
var nameByRef = make(map[file.Reference]string)
for _, p := range pkgs {
// look for /usr/share/docs/NAME/copyright files
name := p.Name
copyrightPath := path.Join(docsPath, name, "copyright")
copyrightRef, err := resolver.RelativeFileByPath(dbRef, copyrightPath)
if err != nil {
return nil, nil, fmt.Errorf("unable to find relative copyright from path=%+v: %w", dbRef.Path, err)
}
// we may not have a copyright file for each package, ignore missing files
if copyrightRef != nil {
copyrightFileMatches = append(copyrightFileMatches, *copyrightRef)
nameByRef[*copyrightRef] = name
}
}
// fetch the copyright contents
copyrightContentsByRef, err := resolver.MultipleFileContentsByRef(copyrightFileMatches...)
if err != nil {
return nil, nil, err
}
// organize content results and refs by package name
var contentsByName = make(map[string]io.Reader)
var refsByName = make(map[string]file.Reference)
for ref, contents := range copyrightContentsByRef {
name := nameByRef[ref]
contentsByName[name] = strings.NewReader(contents)
refsByName[name] = ref
}
return contentsByName, refsByName, nil
}
func md5Key(p pkg.Package) string {
metadata := p.Metadata.(pkg.DpkgMetadata)
contentKey := p.Name
if metadata.Architecture != "" && metadata.Architecture != "all" {
contentKey = contentKey + ":" + metadata.Architecture
}
return contentKey
}
+99
View File
@@ -0,0 +1,99 @@
package deb
import (
"testing"
"github.com/anchore/stereoscope/pkg/imagetest"
"github.com/anchore/syft/syft/pkg"
"github.com/anchore/syft/syft/scope"
"github.com/go-test/deep"
)
func TestDpkgCataloger(t *testing.T) {
tests := []struct {
name string
sources map[string][]string
expected []pkg.Package
}{
{
name: "go-case",
sources: map[string][]string{
"libpam-runtime": {"/var/lib/dpkg/status", "/var/lib/dpkg/info/libpam-runtime.md5sums", "/usr/share/doc/libpam-runtime/copyright"},
},
expected: []pkg.Package{
{
Name: "libpam-runtime",
Version: "1.1.8-3.6",
FoundBy: "dpkgdb-cataloger",
Licenses: []string{"GPL-2", "LGPL-2.1"},
Type: pkg.DebPkg,
MetadataType: pkg.DpkgMetadataType,
Metadata: pkg.DpkgMetadata{
Package: "libpam-runtime",
Source: "pam",
Version: "1.1.8-3.6",
Architecture: "all",
Maintainer: "Steve Langasek <vorlon@debian.org>",
InstalledSize: 1016,
Files: []pkg.DpkgFileRecord{
{Path: "/lib/x86_64-linux-gnu/libz.so.1.2.11", MD5: "55f905631797551d4d936a34c7e73474"},
{Path: "/usr/share/doc/zlib1g/changelog.Debian.gz", MD5: "cede84bda30d2380217f97753c8ccf3a"},
{Path: "/usr/share/doc/zlib1g/changelog.gz", MD5: "f3c9dafa6da7992c47328b4464f6d122"},
{Path: "/usr/share/doc/zlib1g/copyright", MD5: "a4fae96070439a5209a62ae5b8017ab2"},
},
},
},
},
},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
img, cleanup := imagetest.GetFixtureImage(t, "docker-archive", "image-dpkg")
defer cleanup()
s, err := scope.NewScopeFromImage(img, scope.AllLayersScope)
if err != nil {
t.Fatal(err)
}
c := NewDpkgdbCataloger()
actual, err := c.Catalog(s.Resolver)
if err != nil {
t.Fatalf("failed to catalog: %+v", err)
}
if len(actual) != len(test.expected) {
for _, a := range actual {
t.Logf(" %+v", a)
}
t.Fatalf("unexpected package count: %d!=%d", len(actual), len(test.expected))
}
// test sources...
for idx := range actual {
a := &actual[idx]
// we will test the sources separately
var sourcesList = make([]string, len(a.Source))
for i, s := range a.Source {
sourcesList[i] = string(s.Path)
}
a.Source = nil
for _, d := range deep.Equal(sourcesList, test.sources[a.Name]) {
t.Errorf("diff: %+v", d)
}
}
// test remaining fields...
for _, d := range deep.Equal(actual, test.expected) {
t.Errorf("diff: %+v", d)
}
})
}
}
+38
View File
@@ -0,0 +1,38 @@
package deb
import (
"bufio"
"io"
"sort"
"strings"
"github.com/anchore/syft/internal"
)
// For more information see: https://www.debian.org/doc/packaging-manuals/copyright-format/1.0/#license-syntax
func parseLicensesFromCopyright(reader io.Reader) []string {
findings := internal.NewStringSet()
scanner := bufio.NewScanner(reader)
for scanner.Scan() {
line := scanner.Text()
if strings.HasPrefix(line, "License:") {
candidate := strings.Replace(line, "License:", "", 1)
candidate = strings.TrimSpace(candidate)
if strings.Contains(candidate, " or ") || strings.Contains(candidate, " and ") {
// this is a multi-license summary, ignore this as other recurrent license lines should cover this
continue
}
if candidate != "" && strings.ToLower(candidate) != "none" {
findings.Add(candidate)
}
}
}
results := findings.ToSlice()
sort.Strings(results)
return results
}
@@ -0,0 +1,58 @@
package deb
import (
"os"
"testing"
"github.com/go-test/deep"
)
func TestParseLicensesFromCopyright(t *testing.T) {
tests := []struct {
fixture string
expected []string
}{
{
fixture: "test-fixtures/copyright/trilicense",
expected: []string{"GPL-2", "LGPL-2.1", "MPL-1.1"},
},
{
fixture: "test-fixtures/copyright/liblzma5",
expected: []string{"Autoconf", "GPL-2", "GPL-2+", "LGPL-2.1+", "PD", "PD-debian", "config-h", "noderivs", "permissive-fsf", "permissive-nowarranty", "probably-PD"},
},
{
fixture: "test-fixtures/copyright/libaudit-common",
expected: []string{"GPL-2", "LGPL-2.1"},
},
}
for _, test := range tests {
t.Run(test.fixture, func(t *testing.T) {
file, err := os.Open(test.fixture)
if err != nil {
t.Fatal("Unable to read: ", err)
}
defer func() {
err := file.Close()
if err != nil {
t.Fatal("closing file failed:", err)
}
}()
actual := parseLicensesFromCopyright(file)
if len(actual) != len(test.expected) {
for _, a := range actual {
t.Logf(" %+v", a)
}
t.Fatalf("unexpected package count: %d!=%d", len(actual), len(test.expected))
}
diffs := deep.Equal(actual, test.expected)
for _, d := range diffs {
t.Errorf("diff: %+v", d)
}
})
}
}
@@ -0,0 +1,30 @@
package deb
import (
"bufio"
"io"
"strings"
"github.com/anchore/syft/syft/pkg"
)
func parseDpkgMD5Info(reader io.Reader) []pkg.DpkgFileRecord {
var findings []pkg.DpkgFileRecord
scanner := bufio.NewScanner(reader)
for scanner.Scan() {
line := scanner.Text()
fields := strings.SplitN(line, " ", 2)
if len(fields) == 2 {
path := strings.TrimSpace(fields[1])
if !strings.HasPrefix(path, "/") {
path = "/" + path
}
findings = append(findings, pkg.DpkgFileRecord{
Path: path,
MD5: strings.TrimSpace(fields[0]),
})
}
}
return findings
}
@@ -0,0 +1,57 @@
package deb
import (
"os"
"testing"
"github.com/go-test/deep"
"github.com/anchore/syft/syft/pkg"
)
func TestMD5SumInfoParsing(t *testing.T) {
tests := []struct {
fixture string
expected []pkg.DpkgFileRecord
}{
{
fixture: "test-fixtures/info/zlib1g.md5sums",
expected: []pkg.DpkgFileRecord{
{Path: "/lib/x86_64-linux-gnu/libz.so.1.2.11", MD5: "55f905631797551d4d936a34c7e73474"},
{Path: "/usr/share/doc/zlib1g/changelog.Debian.gz", MD5: "cede84bda30d2380217f97753c8ccf3a"},
{Path: "/usr/share/doc/zlib1g/changelog.gz", MD5: "f3c9dafa6da7992c47328b4464f6d122"},
{Path: "/usr/share/doc/zlib1g/copyright", MD5: "a4fae96070439a5209a62ae5b8017ab2"},
},
},
}
for _, test := range tests {
t.Run(test.fixture, func(t *testing.T) {
file, err := os.Open(test.fixture)
if err != nil {
t.Fatal("Unable to read: ", err)
}
defer func() {
err := file.Close()
if err != nil {
t.Fatal("closing file failed:", err)
}
}()
actual := parseDpkgMD5Info(file)
if len(actual) != len(test.expected) {
for _, a := range actual {
t.Logf(" %+v", a)
}
t.Fatalf("unexpected package count: %d!=%d", len(actual), len(test.expected))
}
diffs := deep.Equal(actual, test.expected)
for _, d := range diffs {
t.Errorf("diff: %+v", d)
}
})
}
}
+57 -30
View File
@@ -2,40 +2,43 @@ package deb
import (
"bufio"
"errors"
"fmt"
"io"
"strconv"
"strings"
"github.com/anchore/syft/syft/cataloger/common"
"github.com/anchore/syft/syft/pkg"
"github.com/mitchellh/mapstructure"
)
// integrity check
var _ common.ParserFn = parseDpkgStatus
var errEndOfPackages = fmt.Errorf("no more packages to read")
// parseDpkgStatus is a parser function for Debian DB status contents, returning all Debian packages listed.
func parseDpkgStatus(_ string, reader io.Reader) ([]pkg.Package, error) {
func parseDpkgStatus(reader io.Reader) ([]pkg.Package, error) {
buffedReader := bufio.NewReader(reader)
var packages = make([]pkg.Package, 0)
for {
continueProcessing := true
for continueProcessing {
entry, err := parseDpkgStatusEntry(buffedReader)
if err != nil {
if err == errEndOfPackages {
break
if errors.Is(err, errEndOfPackages) {
continueProcessing = false
} else {
return nil, err
}
return nil, err
}
packages = append(packages, pkg.Package{
Name: entry.Package,
Version: entry.Version,
Type: pkg.DebPkg,
MetadataType: pkg.DpkgMetadataType,
Metadata: entry,
})
if entry.Package != "" {
packages = append(packages, pkg.Package{
Name: entry.Package,
Version: entry.Version,
Type: pkg.DebPkg,
MetadataType: pkg.DpkgMetadataType,
Metadata: entry,
})
}
}
return packages, nil
@@ -43,14 +46,16 @@ func parseDpkgStatus(_ string, reader io.Reader) ([]pkg.Package, error) {
// parseDpkgStatusEntry returns an individual Dpkg entry, or returns errEndOfPackages if there are no more packages to parse from the reader.
func parseDpkgStatusEntry(reader *bufio.Reader) (entry pkg.DpkgMetadata, err error) {
dpkgFields := make(map[string]string)
dpkgFields := make(map[string]interface{})
var retErr error
var key string
for {
line, err := reader.ReadString('\n')
if err != nil {
if err == io.EOF {
return pkg.DpkgMetadata{}, errEndOfPackages
retErr = errEndOfPackages
break
}
return pkg.DpkgMetadata{}, err
}
@@ -82,18 +87,16 @@ func parseDpkgStatusEntry(reader *bufio.Reader) (entry pkg.DpkgMetadata, err err
dpkgFields[key] = val
default:
// parse a new key
if i := strings.Index(line, ":"); i > 0 {
key = strings.TrimSpace(line[0:i])
val := strings.TrimSpace(line[i+1:])
if _, ok := dpkgFields[key]; ok {
return pkg.DpkgMetadata{}, fmt.Errorf("duplicate key discovered: %s", key)
}
dpkgFields[key] = val
} else {
return pkg.DpkgMetadata{}, fmt.Errorf("cannot parse field from line: '%s'", line)
var val interface{}
key, val, err = handleNewKeyValue(line)
if err != nil {
return pkg.DpkgMetadata{}, err
}
if _, ok := dpkgFields[key]; ok {
return pkg.DpkgMetadata{}, fmt.Errorf("duplicate key discovered: %s", key)
}
dpkgFields[key] = val
}
}
@@ -102,5 +105,29 @@ func parseDpkgStatusEntry(reader *bufio.Reader) (entry pkg.DpkgMetadata, err err
return pkg.DpkgMetadata{}, err
}
return entry, nil
return entry, retErr
}
// handleNewKeyValue parse a new key-value pair from the given unprocessed line
func handleNewKeyValue(line string) (string, interface{}, error) {
if i := strings.Index(line, ":"); i > 0 {
var key = strings.TrimSpace(line[0:i])
// mapstruct cant handle "-"
key = strings.ReplaceAll(key, "-", "")
val := strings.TrimSpace(line[i+1:])
// further processing of values based on the key that was discovered
switch key {
case "InstalledSize":
numVal, err := strconv.Atoi(val)
if err != nil {
return "", nil, fmt.Errorf("bad installed-size value=%q: %w", val, err)
}
return key, numVal, nil
default:
return key, val, nil
}
}
return "", nil, fmt.Errorf("cannot parse field from line: '%s'", line)
}
+20 -14
View File
@@ -24,17 +24,19 @@ func TestSinglePackage(t *testing.T) {
{
name: "Test Single Package",
expected: pkg.DpkgMetadata{
Package: "apt",
Source: "apt-dev",
Version: "1.8.2",
Architecture: "amd64",
Package: "apt",
Source: "apt-dev",
Version: "1.8.2",
Architecture: "amd64",
InstalledSize: 4064,
Maintainer: "APT Development Team <deity@lists.debian.org>",
},
},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
file, err := os.Open("test-fixtures/single")
file, err := os.Open("test-fixtures/status/single")
if err != nil {
t.Fatal("Unable to read test_fixtures/single: ", err)
}
@@ -66,15 +68,19 @@ func TestMultiplePackages(t *testing.T) {
name: "Test Multiple Package",
expected: []pkg.DpkgMetadata{
{
Package: "tzdata",
Version: "2020a-0+deb10u1",
Source: "tzdata-dev",
Architecture: "all",
Package: "tzdata",
Version: "2020a-0+deb10u1",
Source: "tzdata-dev",
Architecture: "all",
InstalledSize: 3036,
Maintainer: "GNU Libc Maintainers <debian-glibc@lists.debian.org>",
},
{
Package: "util-linux",
Version: "2.33.1-0.1",
Architecture: "amd64",
Package: "util-linux",
Version: "2.33.1-0.1",
Architecture: "amd64",
InstalledSize: 4327,
Maintainer: "LaMont Jones <lamont@debian.org>",
},
},
},
@@ -82,7 +88,7 @@ func TestMultiplePackages(t *testing.T) {
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
file, err := os.Open("test-fixtures/multiple")
file, err := os.Open("test-fixtures/status/multiple")
if err != nil {
t.Fatal("Unable to read: ", err)
}
@@ -93,7 +99,7 @@ func TestMultiplePackages(t *testing.T) {
}
}()
pkgs, err := parseDpkgStatus(file.Name(), file)
pkgs, err := parseDpkgStatus(file)
if err != nil {
t.Fatal("Unable to read file contents: ", err)
}
@@ -0,0 +1,43 @@
Format: http://www.debian.org/doc/packaging-manuals/copyright-format/1.0/
Upstream-Name: auditd
Source: https://people.redhat.com/sgrubb/audit/
Files: *
Copyright: 2012-2016 Steve Grubb <sgrubb@redhat.com>
2006-2012 Rik Faith
License: GPL-2
Files: src/libev/*
Copyright: 2007-2009 Marc Alexamder Lehmann
License: GPL-2
Files: lib/*
Copyright: 2005-2008 Steve Grubb <sgrubb@redhat.com>
License: LGPL-2.1
The audit daemon's library libaudit.* is released under LGPL
so that it may be linked with 3rd party software.
.
On Debian systems, refer to /usr/share/common-licenses/LGPL-2.1
for the complete text of the GNU Lesser General Public License.
Files: debian/*
Copyright: 2007-2011 Philipp Matthias Hahn <pmhahn@debian.org>
2012-2016 Laurent Bigonville <bigon@debian.org>
License: GPL-2
License: GPL-2
This package is free software; you can redistribute it and/or modify
it under the terms of the GNU General Public License version 2,
as published by the Free Software Foundation.
.
This package is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
.
You should have received a copy of the GNU General Public License
along with this package; if not, write to the Free Software
Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
.
On Debian systems, the complete text of the GNU General
Public License can be found in `/usr/share/common-licenses/GPL-1'.
@@ -0,0 +1,383 @@
Format: http://www.debian.org/doc/packaging-manuals/copyright-format/1.0/
Upstream-Name: XZ Utils
Upstream-Contact:
Lasse Collin <lasse.collin@tukaani.org>
http://tukaani.org/xz/lists.html
Source:
http://tukaani.org/xz
http://git.tukaani.org/xz.git
Comment:
XZ Utils is developed and maintained upstream by Lasse Collin. Major
portions are based on code by other authors; see AUTHORS for details.
Most of the source has been put into the public domain, but some files
have not (details below).
.
This file describes the source package. The binary packages contain
some files derived from other works: for example, images in the API
documentation come from Doxygen.
License:
Different licenses apply to different files in this package. Here
is a rough summary of which licenses apply to which parts of this
package (but check the individual files to be sure!):
.
- liblzma is in the public domain.
.
- xz, xzdec, and lzmadec command line tools are in the public
domain unless GNU getopt_long had to be compiled and linked
in from the lib directory. The getopt_long code is under
GNU LGPLv2.1+.
.
- The scripts to grep, diff, and view compressed files have been
adapted from gzip. These scripts and their documentation are
under GNU GPLv2+.
.
- All the documentation in the doc directory and most of the
XZ Utils specific documentation files in other directories
are in the public domain.
.
- Translated messages are in the public domain.
.
- The build system contains public domain files, and files that
are under GNU GPLv2+ or GNU GPLv3+. None of these files end up
in the binaries being built.
.
- Test files and test code in the tests directory, and debugging
utilities in the debug directory are in the public domain.
.
- The extra directory may contain public domain files, and files
that are under various free software licenses.
.
You can do whatever you want with the files that have been put into
the public domain. If you find public domain legally problematic,
take the previous sentence as a license grant. If you still find
the lack of copyright legally problematic, you have too many
lawyers.
.
As usual, this software is provided "as is", without any warranty.
.
If you copy significant amounts of public domain code from XZ Utils
into your project, acknowledging this somewhere in your software is
polite (especially if it is proprietary, non-free software), but
naturally it is not legally required. Here is an example of a good
notice to put into "about box" or into documentation:
.
This software includes code from XZ Utils <http://tukaani.org/xz/>.
.
The following license texts are included in the following files:
- COPYING.LGPLv2.1: GNU Lesser General Public License version 2.1
- COPYING.GPLv2: GNU General Public License version 2
- COPYING.GPLv3: GNU General Public License version 3
.
Note that the toolchain (compiler, linker etc.) may add some code
pieces that are copyrighted. Thus, it is possible that e.g. liblzma
binary wouldn't actually be in the public domain in its entirety
even though it contains no copyrighted code from the XZ Utils source
package.
.
If you have questions, don't hesitate to ask the author(s) for more
information.
Files: *
Copyright: 2006-2012, Lasse Collin
1999-2008, Igor Pavlov
2006, Ville Koskinen
1998, Steve Reid
2000, Wei Dai
2003, Kevin Springle
2009, Jonathan Nieder
2010, Anders F Björklund
License: PD
This file has been put in the public domain.
You can do whatever you want with this file.
Comment:
From: Lasse Collin <lasse.collin@tukaani.org>
To: Jonathan Nieder <jrnieder@gmail.com>
Subject: Re: XZ utils for Debian
Date: Sun, 19 Jul 2009 13:28:23 +0300
Message-Id: <200907191328.23816.lasse.collin@tukaani.org>
.
[...]
.
> AUTHORS, ChangeLog, COPYING, README, THANKS, TODO,
> dos/README, windows/README
.
COPYING says that most docs are in the public domain. Maybe that's not
clear enough, but on the other hand it looks a bit stupid to put
copyright information in tiny and relatively small docs like README.
.
I don't dare to say that _all_ XZ Utils specific docs are in the public
domain unless otherwise mentioned in the file. I'm including PDF files
generated by groff + ps2pdf, and some day I might include Doxygen-
generated HTML docs too. Those don't include any copyright notices, but
it seems likely that groff + ps2pdf or at least Doxygen put some
copyrighted content into the generated files.
Files: INSTALL NEWS PACKAGERS
windows/README-Windows.txt
windows/INSTALL-Windows.txt
Copyright: 2009-2010, Lasse Collin
License: probably-PD
See the note on AUTHORS, README, and so on above.
Files: src/scripts/* lib/* extra/scanlzma/scanlzma.c
Copyright: © 1993, Jean-loup Gailly
© 1989-1994, 1996-1999, 2001-2007, Free Software Foundation, Inc.
© 2006 Timo Lindfors
2005, Charles Levert
2005, 2009, Lasse Collin
2009, Andrew Dudman
Other-Authors: Paul Eggert, Ulrich Drepper
License: GPL-2+
Files: src/scripts/Makefile.am src/scripts/xzless.1
Copyright: 2009, Andrew Dudman
2009, Lasse Collin
License: PD
This file has been put in the public domain.
You can do whatever you want with this file.
Files: doc/examples/xz_pipe_comp.c doc/examples/xz_pipe_decomp.c
Copyright: 2010, Daniel Mealha Cabrita
License: PD
Not copyrighted -- provided to the public domain.
Files: lib/getopt.c lib/getopt1.c lib/getopt.in.h
Copyright: © 1987-2007 Free Software Foundation, Inc.
Other-Authors: Ulrich Drepper
License: LGPL-2.1+
Files: m4/getopt.m4 m4/posix-shell.m4
Copyright: © 2002-2006, 2008 Free Software Foundation, Inc.
© 2007-2008 Free Software Foundation, Inc.
Other-Authors: Bruno Haible, Paul Eggert
License: permissive-fsf
Files: m4/acx_pthread.m4
Copyright: © 2008, Steven G. Johnson <stevenj@alum.mit.edu>
License: Autoconf
Files: Doxyfile.in
Copyright: © 1997-2007 by Dimitri van Heesch
Origin: Doxygen 1.4.7
License: GPL-2
Files: src/liblzma/check/crc32_table_?e.h
src/liblzma/check/crc64_table_?e.h
src/liblzma/lzma/fastpos_table.c
src/liblzma/rangecoder/price_table.c
Copyright: none, automatically generated data
Generated-With:
src/liblzma/check/crc32_tablegen.c
src/liblzma/check/crc64_tablegen.c
src/liblzma/lzma/fastpos_tablegen.c
src/liblzma/rangecoder/price_tablegen.c
License: none
No copyright to license.
Files: .gitignore m4/.gitignore po/.gitignore po/LINGUAS po/POTFILES.in
Copyright: none; these are just short lists.
License: none
No copyright to license.
Files: tests/compress_prepared_bcj_*
Copyright: 2008-2009, Lasse Collin
Source-Code: tests/bcj_test.c
License: PD
This file has been put into the public domain.
You can do whatever you want with this file.
Comment:
changelog.gz (commit 975d8fd) explains:
.
Recreated the BCJ test files for x86 and SPARC. The old files
were linked with crt*.o, which are copyrighted, and thus the
old test files were not in the public domain as a whole. They
are freely distributable though, but it is better to be careful
and avoid including any copyrighted pieces in the test files.
The new files are just compiled and assembled object files,
and thus don't contain any copyrighted code.
Files: po/cs.po po/de.po po/fr.po
Copyright: 2010, Marek Černocký
2010, Andre Noll
2011, Adrien Nader
License: PD
This file is put in the public domain.
Files: po/it.po po/pl.po
Copyright: 2009, 2010, Gruppo traduzione italiano di Ubuntu-it
2010, Lorenzo De Liso
2009, 2010, 2011, Milo Casagrande
2011, Jakub Bogusz
License: PD
This file is in the public domain
Files: INSTALL.generic
Copyright: © 1994, 1995, 1996, 1999, 2000, 2001, 2002, 2004, 2005,
2006, 2007, 2008, 2009, 2010 Free Software Foundation, Inc.
License: permissive-nowarranty
Files: dos/config.h
Copyright: © 1992, 1993, 1994, 1999, 2000, 2001, 2002, 2005
Free Software Foundation, Inc.
2007-2010, Lasse Collin
Other-Authors: Roland McGrath, Akim Demaille, Paul Eggert,
David Mackenzie, Bruno Haible, and many others.
Origin: configure.ac from XZ Utils,
visibility.m4 serial 1 (gettext-0.15),
Autoconf 2.52g
License: config-h
configure.ac:
.
# Author: Lasse Collin
#
# This file has been put into the public domain.
# You can do whatever you want with this file.
.
visibility.m4:
.
dnl Copyright (C) 2005 Free Software Foundation, Inc.
dnl This file is free software; the Free Software Foundation
dnl gives unlimited permission to copy and/or distribute it,
dnl with or without modifications, as long as this notice is preserved.
.
dnl From Bruno Haible.
.
comments from Autoconf 2.52g:
.
# Copyright 1992, 1993, 1994, 1999, 2000, 2001, 2002
# Free Software Foundation, Inc.
.
[...]
.
# As a special exception, the Free Software Foundation gives unlimited
# permission to copy, distribute and modify the configure scripts that
# are the output of Autoconf. You need not follow the terms of the GNU
# General Public License when using or distributing such scripts, even
# though portions of the text of Autoconf appear in them. The GNU
# General Public License (GPL) does govern all other use of the material
# that constitutes the Autoconf program.
.
On Debian systems, the complete text of the GNU General Public
License version 2 can be found in ‘/usr/share/common-licenses/GPL-2’.
dos/config.h was generated with autoheader, which tells Autoconf to
output a script to generate a config.h file and then runs it.
Files: po/Makevars
Origin: gettext-runtime/po/Makevars (gettext-0.12)
Copyright: © 2003 Free Software Foundation, Inc.
Authors: Bruno Haible
License: LGPL-2.1+
The gettext-runtime package is under the LGPL, see files intl/COPYING.LIB-2.0
and intl/COPYING.LIB-2.1.
.
On Debian systems, the complete text of intl/COPYING.LIB-2.0 from
gettext-runtime 0.12 can be found in ‘/usr/share/common-licenses/LGPL-2’
and the text of intl/COPYING.LIB-2.1 can be found in
‘/usr/share/common-licenses/LGPL-2.1’.
.
po/Makevars consists mostly of helpful comments and does not contain a
copyright and license notice.
Files: COPYING.GPLv2 COPYING.GPLv3 COPYING.LGPLv2.1
Copyright: © 1989, 1991, 1999, 2007 Free Software Foundation, Inc.
License: noderivs
Everyone is permitted to copy and distribute verbatim copies
of this license document, but changing it is not allowed.
Files: debian/*
Copyright: 2009-2012, Jonathan Nieder
License: PD-debian
The Debian packaging files are in the public domain.
You may freely use, modify, distribute, and relicense them.
License: LGPL-2.1+
This program is free software; you can redistribute it and/or modify
it under the terms of the GNU Lesser General Public License as published by
the Free Software Foundation; either version 2.1, or (at your option)
any later version.
.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Lesser General Public License for more details.
.
You should have received a copy of the GNU Lesser General Public License along
with this program; if not, write to the Free Software Foundation,
Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
.
On Debian systems, the complete text of the GNU Lesser General Public
License version 2.1 can be found in ‘/usr/share/common-licenses/LGPL-2.1’.
License: GPL-2
Permission to use, copy, modify, and distribute this software and its
documentation under the terms of the GNU General Public License is
hereby granted. No representations are made about the suitability of
this software for any purpose. It is provided "as is" without express
or implied warranty. See the GNU General Public License for more
details.
.
Documents produced by doxygen are derivative works derived from the
input used in their production; they are not affected by this license.
.
On Debian systems, the complete text of the version of the GNU General
Public License distributed with Doxygen can be found in
‘/usr/share/common-licenses/GPL-2’.
License: GPL-2+
This program is free software; you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation; either version 2, or (at your option)
any later version.
.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
.
You should have received a copy of the GNU General Public License along
with this program; if not, write to the Free Software Foundation,
Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
.
On Debian systems, the complete text of the GNU General Public License
version 2 can be found in ‘/usr/share/common-licenses/GPL-2’.
License: Autoconf
This program is free software: you can redistribute it and/or modify it
under the terms of the GNU General Public License as published by the
Free Software Foundation, either version 3 of the License, or (at your
option) any later version.
.
This program is distributed in the hope that it will be useful, but
WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General
Public License for more details.
.
You should have received a copy of the GNU General Public License along
with this program. If not, see <http://www.gnu.org/licenses/>.
.
As a special exception, the respective Autoconf Macro's copyright owner
gives unlimited permission to copy, distribute and modify the configure
scripts that are the output of Autoconf when processing the Macro. You
need not follow the terms of the GNU General Public License when using
or distributing such scripts, even though portions of the text of the
Macro appear in them. The GNU General Public License (GPL) does govern
all other use of the material that constitutes the Autoconf Macro.
.
This special exception to the GPL applies to versions of the Autoconf
Macro released by the Autoconf Archive. When you make and distribute a
modified version of the Autoconf Macro, you may extend this special
exception to the GPL to apply to your modified version as well.
.
On Debian systems, the complete text of the GNU General Public
License version 3 can be found in ‘/usr/share/common-licenses/GPL-3’.
License: permissive-fsf
This file is free software; the Free Software Foundation
gives unlimited permission to copy and/or distribute it,
with or without modifications, as long as this notice is preserved.
License: permissive-nowarranty
Copying and distribution of this file, with or without modification,
are permitted in any medium without royalty provided the copyright
notice and this notice are preserved. This file is offered as-is,
without warranty of any kind.
@@ -0,0 +1,13 @@
Files: src/js/editline/*
Copyright: 1993, John Doe
1993, Joe Average
License: MPL-1.1 or GPL-2 or LGPL-2.1
License: MPL-1.1
[LICENSE TEXT]
License: GPL-2
[LICENSE TEXT]
License: LGPL-2.1
[LICENSE TEXT]
@@ -0,0 +1,2 @@
FROM scratch
COPY . .
@@ -0,0 +1,43 @@
Format: http://www.debian.org/doc/packaging-manuals/copyright-format/1.0/
Upstream-Name: auditd
Source: https://people.redhat.com/sgrubb/audit/
Files: *
Copyright: 2012-2016 Steve Grubb <sgrubb@redhat.com>
2006-2012 Rik Faith
License: GPL-2
Files: src/libev/*
Copyright: 2007-2009 Marc Alexamder Lehmann
License: GPL-2
Files: lib/*
Copyright: 2005-2008 Steve Grubb <sgrubb@redhat.com>
License: LGPL-2.1
The audit daemon's library libaudit.* is released under LGPL
so that it may be linked with 3rd party software.
.
On Debian systems, refer to /usr/share/common-licenses/LGPL-2.1
for the complete text of the GNU Lesser General Public License.
Files: debian/*
Copyright: 2007-2011 Philipp Matthias Hahn <pmhahn@debian.org>
2012-2016 Laurent Bigonville <bigon@debian.org>
License: GPL-2
License: GPL-2
This package is free software; you can redistribute it and/or modify
it under the terms of the GNU General Public License version 2,
as published by the Free Software Foundation.
.
This package is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
.
You should have received a copy of the GNU General Public License
along with this package; if not, write to the Free Software
Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
.
On Debian systems, the complete text of the GNU General
Public License can be found in `/usr/share/common-licenses/GPL-1'.
@@ -0,0 +1,4 @@
55f905631797551d4d936a34c7e73474 lib/x86_64-linux-gnu/libz.so.1.2.11
cede84bda30d2380217f97753c8ccf3a usr/share/doc/zlib1g/changelog.Debian.gz
f3c9dafa6da7992c47328b4464f6d122 usr/share/doc/zlib1g/changelog.gz
a4fae96070439a5209a62ae5b8017ab2 usr/share/doc/zlib1g/copyright
@@ -0,0 +1,21 @@
Package: libpam-runtime
Status: install ok installed
Priority: required
Section: admin
Installed-Size: 1016
Maintainer: Steve Langasek <vorlon@debian.org>
Architecture: all
Multi-Arch: foreign
Source: pam
Version: 1.1.8-3.6
Replaces: libpam0g-dev, libpam0g-util
Depends: debconf (>= 0.5) | debconf-2.0, debconf (>= 1.5.19) | cdebconf, libpam-modules (>= 1.0.1-6)
Conflicts: libpam0g-util
Conffiles:
/etc/pam.conf 87fc76f18e98ee7d3848f6b81b3391e5
/etc/pam.d/other 31aa7f2181889ffb00b87df4126d1701
Description: Runtime support for the PAM library
Contains configuration files and directories required for
authentication to work on Debian systems. This package is required
on almost all installations.
Homepage: http://www.linux-pam.org/
@@ -0,0 +1,4 @@
55f905631797551d4d936a34c7e73474 lib/x86_64-linux-gnu/libz.so.1.2.11
cede84bda30d2380217f97753c8ccf3a usr/share/doc/zlib1g/changelog.Debian.gz
f3c9dafa6da7992c47328b4464f6d122 usr/share/doc/zlib1g/changelog.gz
a4fae96070439a5209a62ae5b8017ab2 usr/share/doc/zlib1g/copyright
+1 -1
View File
@@ -69,7 +69,7 @@ func (a archiveFilename) pkgType() pkg.Type {
func (a archiveFilename) version() string {
if len(a.fields) > 1 {
log.Errorf("discovered multiple name-version pairings from %q: %+v", a.raw, a.fields)
log.Warnf("discovered multiple name-version pairings from %q: %+v", a.raw, a.fields)
return ""
} else if len(a.fields) < 1 {
return ""
+1 -1
View File
@@ -266,7 +266,7 @@ func (j *archiveParser) discoverPkgsFromNestedArchives(parentPkg *pkg.Package) (
nestedPkgs, err := parseJavaArchive(nestedPath, archiveReadCloser)
if err != nil {
if closeErr := archiveReadCloser.Close(); closeErr != nil {
log.Errorf("unable to close archived file from tempdir: %+v", closeErr)
log.Warnf("unable to close archived file from tempdir: %+v", closeErr)
}
return nil, fmt.Errorf("unable to process nested java archive (%s): %w", archivePath, err)
}
+1 -1
View File
@@ -77,7 +77,7 @@ func parseJavaManifest(path string, reader io.Reader) (*pkg.JavaManifest, error)
// per the manifest spec (https://docs.oracle.com/en/java/javase/11/docs/specs/jar/jar.html#jar-manifest)
// this should never happen. If it does we want to know about it, but not necessarily stop
// cataloging entirely... for this reason we only log.
log.Errorf("java manifest section found without a name: %s", path)
log.Warnf("java manifest section found without a name: %s", path)
name = strconv.Itoa(i)
} else {
delete(s, "Name")
+1 -1
View File
@@ -40,7 +40,7 @@ func parseYarnLock(_ string, reader io.Reader) ([]pkg.Package, error) {
case composedNameExp.MatchString(line):
name := composedNameExp.FindString(line)
if len(name) == 0 {
log.Errorf("unable to parse line: '%s'", line)
log.Warnf("unable to parse yarn.lock line: %q", line)
}
currentName = strings.TrimLeft(name, "\"")
case simpleNameExp.MatchString(line):
+4 -1
View File
@@ -6,6 +6,8 @@ import (
"path/filepath"
"strings"
"github.com/anchore/syft/internal/log"
"github.com/anchore/stereoscope/pkg/file"
"github.com/anchore/syft/syft/pkg"
@@ -123,7 +125,8 @@ func (c *PackageCataloger) fetchTopLevelPackages(resolver scope.Resolver, metada
return nil, nil, err
}
if topLevelRef == nil {
return nil, nil, fmt.Errorf("missing python package top_level.txt (package=%q)", string(metadataRef.Path))
log.Warnf("missing python package top_level.txt (package=%q)", string(metadataRef.Path))
return nil, nil, nil
}
sources = append(sources, *topLevelRef)
@@ -69,20 +69,20 @@ func newTestResolver(metaPath, recordPath, topPath string) *pythonTestResolverMo
}
func (r *pythonTestResolverMock) FileContentsByRef(ref file.Reference) (string, error) {
switch ref.Path {
case r.topLevelRef.Path:
switch {
case r.topLevelRef != nil && ref.Path == r.topLevelRef.Path:
b, err := ioutil.ReadAll(r.topLevelReader)
if err != nil {
return "", err
}
return string(b), nil
case r.metadataRef.Path:
case ref.Path == r.metadataRef.Path:
b, err := ioutil.ReadAll(r.metadataReader)
if err != nil {
return "", err
}
return string(b), nil
case r.recordRef.Path:
case ref.Path == r.recordRef.Path:
b, err := ioutil.ReadAll(r.recordReader)
if err != nil {
return "", err
@@ -188,7 +188,6 @@ func TestPythonPackageWheelCataloger(t *testing.T) {
// in cases where the metadata file is available and the record is not we should still record there is a package
// additionally empty top_level.txt files should not result in an error
MetadataFixture: "test-fixtures/partial.dist-info/METADATA",
TopLevelFixture: "test-fixtures/partial.dist-info/top_level.txt",
ExpectedPackage: pkg.Package{
Name: "Pygments",
Version: "2.6.1",
+39 -5
View File
@@ -4,13 +4,47 @@ Package rpmdb provides a concrete Cataloger implementation for RPM "Package" DB
package rpmdb
import (
"github.com/anchore/syft/syft/cataloger/common"
"fmt"
"strings"
"github.com/anchore/syft/syft/pkg"
"github.com/anchore/syft/syft/scope"
)
const (
packagesGlob = "**/var/lib/rpm/Packages"
)
type Cataloger struct{}
// NewRpmdbCataloger returns a new RPM DB cataloger object.
func NewRpmdbCataloger() *common.GenericCataloger {
globParsers := map[string]common.ParserFn{
"**/var/lib/rpm/Packages": parseRpmDB,
func NewRpmdbCataloger() *Cataloger {
return &Cataloger{}
}
// Name returns a string that uniquely describes a cataloger
func (c *Cataloger) Name() string {
return "rpmdb-cataloger"
}
// Catalog is given an object to resolve file references and content, this function returns any discovered Packages after analyzing rpm db installation.
func (c *Cataloger) Catalog(resolver scope.Resolver) ([]pkg.Package, error) {
fileMatches, err := resolver.FilesByGlob(packagesGlob)
if err != nil {
return nil, fmt.Errorf("failed to find rpmdb's by glob: %w", err)
}
return common.NewGenericCataloger(nil, globParsers, "rpmdb-cataloger")
var pkgs []pkg.Package
for _, dbRef := range fileMatches {
dbContents, err := resolver.FileContentsByRef(dbRef)
if err != nil {
return nil, err
}
pkgs, err = parseRpmDB(resolver, dbRef, strings.NewReader(dbContents))
if err != nil {
return nil, fmt.Errorf("unable to catalog rpmdb package=%+v: %w", dbRef.Path, err)
}
}
return pkgs, nil
}
+35 -5
View File
@@ -6,18 +6,18 @@ import (
"io/ioutil"
"os"
"github.com/anchore/stereoscope/pkg/file"
"github.com/anchore/syft/syft/scope"
rpmdb "github.com/anchore/go-rpmdb/pkg"
"github.com/anchore/syft/internal"
"github.com/anchore/syft/internal/log"
"github.com/anchore/syft/syft/cataloger/common"
"github.com/anchore/syft/syft/pkg"
)
// integrity check
var _ common.ParserFn = parseRpmDB
// parseApkDb parses an "Packages" RPM DB and returns the Packages listed within it.
func parseRpmDB(_ string, reader io.Reader) ([]pkg.Package, error) {
func parseRpmDB(resolver scope.FileResolver, dbRef file.Reference, reader io.Reader) ([]pkg.Package, error) {
f, err := ioutil.TempFile("", internal.ApplicationName+"-rpmdb")
if err != nil {
return nil, fmt.Errorf("failed to create temp rpmdb file: %w", err)
@@ -48,10 +48,16 @@ func parseRpmDB(_ string, reader io.Reader) ([]pkg.Package, error) {
allPkgs := make([]pkg.Package, 0)
for _, entry := range pkgList {
records, err := extractRpmdbFileRecords(resolver, entry)
if err != nil {
return nil, err
}
p := pkg.Package{
Name: entry.Name,
Version: fmt.Sprintf("%s-%s", entry.Version, entry.Release), // this is what engine does
//Version: fmt.Sprintf("%d:%s-%s.%s", entry.Epoch, entry.Version, entry.Release, entry.Arch),
Source: []file.Reference{dbRef},
Type: pkg.RpmPkg,
MetadataType: pkg.RpmdbMetadataType,
Metadata: pkg.RpmdbMetadata{
@@ -64,6 +70,7 @@ func parseRpmDB(_ string, reader io.Reader) ([]pkg.Package, error) {
Vendor: entry.Vendor,
License: entry.License,
Size: entry.Size,
Files: records,
},
}
@@ -72,3 +79,26 @@ func parseRpmDB(_ string, reader io.Reader) ([]pkg.Package, error) {
return allPkgs, nil
}
func extractRpmdbFileRecords(resolver scope.FileResolver, entry *rpmdb.PackageInfo) ([]pkg.RpmdbFileRecord, error) {
var records = make([]pkg.RpmdbFileRecord, 0)
for _, record := range entry.Files {
refs, err := resolver.FilesByPath(file.Path(record.Path))
if err != nil {
return nil, fmt.Errorf("failed to resolve path=%+v: %w", record.Path, err)
}
//only persist RPMDB file records which exist in the image/directory, otherwise ignore them
if len(refs) == 0 {
continue
}
records = append(records, pkg.RpmdbFileRecord{
Path: record.Path,
Mode: pkg.RpmdbFileMode(record.Mode),
Size: int(record.Size),
SHA256: record.SHA256,
})
}
return records, nil
}
+127 -40
View File
@@ -1,58 +1,145 @@
package rpmdb
import (
"fmt"
"os"
"testing"
"github.com/anchore/stereoscope/pkg/file"
"github.com/anchore/syft/syft/pkg"
"github.com/go-test/deep"
)
type rpmdbTestFileResolverMock struct {
ignorePaths bool
}
func newTestFileResolver(ignorePaths bool) *rpmdbTestFileResolverMock {
return &rpmdbTestFileResolverMock{
ignorePaths: ignorePaths,
}
}
func (r *rpmdbTestFileResolverMock) FilesByPath(paths ...file.Path) ([]file.Reference, error) {
if r.ignorePaths {
// act as if no paths exist
return nil, nil
}
// act as if all files exist
var refs = make([]file.Reference, len(paths))
for i, p := range paths {
refs[i] = file.NewFileReference(p)
}
return refs, nil
}
func (r *rpmdbTestFileResolverMock) FilesByGlob(...string) ([]file.Reference, error) {
return nil, fmt.Errorf("not implemented")
}
func (r *rpmdbTestFileResolverMock) RelativeFileByPath(file.Reference, string) (*file.Reference, error) {
return nil, fmt.Errorf("not implemented")
}
func TestParseRpmDB(t *testing.T) {
expected := map[string]pkg.Package{
"dive": {
Name: "dive",
Version: "0.9.2-1",
Type: pkg.RpmPkg,
MetadataType: pkg.RpmdbMetadataType,
Metadata: pkg.RpmdbMetadata{
Name: "dive",
Epoch: 0,
Arch: "x86_64",
Release: "1",
Version: "0.9.2",
SourceRpm: "dive-0.9.2-1.src.rpm",
Size: 12406784,
License: "MIT",
Vendor: "",
dbRef := file.NewFileReference("test-path")
tests := []struct {
fixture string
expected map[string]pkg.Package
ignorePaths bool
}{
{
fixture: "test-fixtures/Packages",
// we only surface package paths for files that exist (here we DO NOT expect a path)
ignorePaths: true,
expected: map[string]pkg.Package{
"dive": {
Name: "dive",
Version: "0.9.2-1",
Source: []file.Reference{dbRef},
Type: pkg.RpmPkg,
MetadataType: pkg.RpmdbMetadataType,
Metadata: pkg.RpmdbMetadata{
Name: "dive",
Epoch: 0,
Arch: "x86_64",
Release: "1",
Version: "0.9.2",
SourceRpm: "dive-0.9.2-1.src.rpm",
Size: 12406784,
License: "MIT",
Vendor: "",
Files: []pkg.RpmdbFileRecord{},
},
},
},
},
{
fixture: "test-fixtures/Packages",
// we only surface package paths for files that exist (here we expect a path)
ignorePaths: false,
expected: map[string]pkg.Package{
"dive": {
Name: "dive",
Version: "0.9.2-1",
Source: []file.Reference{dbRef},
Type: pkg.RpmPkg,
MetadataType: pkg.RpmdbMetadataType,
Metadata: pkg.RpmdbMetadata{
Name: "dive",
Epoch: 0,
Arch: "x86_64",
Release: "1",
Version: "0.9.2",
SourceRpm: "dive-0.9.2-1.src.rpm",
Size: 12406784,
License: "MIT",
Vendor: "",
Files: []pkg.RpmdbFileRecord{
{
Path: "/usr/local/bin/dive",
Mode: 33261,
Size: 12406784,
SHA256: "81d29f327ba23096b3c52ff6fe1c425641e618bc87b5c05ee377edc650afaa55",
},
},
},
},
},
},
}
fixture, err := os.Open("test-fixtures/Packages")
if err != nil {
t.Fatalf("failed to open fixture: %+v", err)
}
actual, err := parseRpmDB(fixture.Name(), fixture)
if err != nil {
t.Fatalf("failed to parse rpmdb: %+v", err)
}
if len(actual) != len(expected) {
for _, a := range actual {
t.Log(" ", a)
}
t.Fatalf("unexpected package count: %d!=%d", len(actual), len(expected))
}
for _, a := range actual {
e := expected[a.Name]
diffs := deep.Equal(a, e)
if len(diffs) > 0 {
for _, d := range diffs {
t.Errorf("diff: %+v", d)
for _, test := range tests {
t.Run(test.fixture, func(t *testing.T) {
fixture, err := os.Open(test.fixture)
if err != nil {
t.Fatalf("failed to open fixture: %+v", err)
}
}
fileResolver := newTestFileResolver(test.ignorePaths)
actual, err := parseRpmDB(fileResolver, dbRef, fixture)
if err != nil {
t.Fatalf("failed to parse rpmdb: %+v", err)
}
if len(actual) != len(test.expected) {
for _, a := range actual {
t.Log(" ", a)
}
t.Fatalf("unexpected package count: %d!=%d", len(actual), len(test.expected))
}
for _, a := range actual {
e := test.expected[a.Name]
diffs := deep.Equal(a, e)
if len(diffs) > 0 {
for _, d := range diffs {
t.Errorf("diff: %+v", d)
}
}
}
})
}
}
+3 -1
View File
@@ -10,6 +10,7 @@ type Distro struct {
Type Type
Version *hashiVer.Version
RawVersion string
IDLike string
}
// NewUnknownDistro creates a standardized Distro object for unidentifiable distros
@@ -19,7 +20,7 @@ func NewUnknownDistro() Distro {
}
}
func NewDistro(t Type, ver string) (Distro, error) {
func NewDistro(t Type, ver, like string) (Distro, error) {
if ver == "" {
return Distro{Type: t}, nil
}
@@ -31,6 +32,7 @@ func NewDistro(t Type, ver string) (Distro, error) {
Type: t,
Version: verObj,
RawVersion: ver,
IDLike: like,
}, nil
}
+7 -2
View File
@@ -33,7 +33,7 @@ func TestDistro_FullVersion(t *testing.T) {
for _, test := range tests {
name := fmt.Sprintf("%s:%s", test.dist, test.version)
t.Run(name, func(t *testing.T) {
d, err := NewDistro(test.dist, test.version)
d, err := NewDistro(test.dist, test.version, "")
if err != nil {
t.Errorf("could not create distro='%+v:%+v': %+v", test.dist, test.version, err)
}
@@ -53,29 +53,34 @@ func TestDistro_MajorVersion(t *testing.T) {
dist Type
version string
expected string
like string
}{
{
version: "8",
expected: "8",
like: "",
},
{
version: "18.04",
expected: "18",
like: "debian",
},
{
version: "0",
expected: "0",
like: "",
},
{
version: "18.1.2",
expected: "18",
like: "debian",
},
}
for _, test := range tests {
name := fmt.Sprintf("%s:%s", test.dist, test.version)
t.Run(name, func(t *testing.T) {
d, err := NewDistro(test.dist, test.version)
d, err := NewDistro(test.dist, test.version, test.like)
if err != nil {
t.Errorf("could not create distro='%+v:%+v': %+v", test.dist, test.version, err)
}
+7 -5
View File
@@ -81,7 +81,7 @@ identifyLoop:
return distro
}
func assemble(name, version string) *Distro {
func assemble(name, version, like string) *Distro {
distroType, ok := IDMapping[name]
// Both distro and version must be present
@@ -90,7 +90,7 @@ func assemble(name, version string) *Distro {
}
if ok {
distro, err := NewDistro(distroType, version)
distro, err := NewDistro(distroType, version, like)
if err != nil {
return nil
}
@@ -101,7 +101,7 @@ func assemble(name, version string) *Distro {
}
func parseOsRelease(contents string) *Distro {
id, vers := "", ""
id, vers, like := "", "", ""
for _, line := range strings.Split(contents, "\n") {
parts := strings.Split(line, "=")
prefix := parts[0]
@@ -112,10 +112,12 @@ func parseOsRelease(contents string) *Distro {
id = strings.TrimSpace(value)
case "VERSION_ID":
vers = strings.TrimSpace(value)
case "ID_LIKE":
like = strings.TrimSpace(value)
}
}
return assemble(id, vers)
return assemble(id, vers, like)
}
var busyboxVersionMatcher = regexp.MustCompile(`BusyBox v[\d\.]+`)
@@ -125,7 +127,7 @@ func parseBusyBox(contents string) *Distro {
for _, match := range matches {
parts := strings.Split(match, " ")
version := strings.ReplaceAll(parts[1], "v", "")
distro := assemble("busybox", version)
distro := assemble("busybox", version, "")
if distro != nil {
return distro
}
+3
View File
@@ -5,6 +5,9 @@ package logger
type Logger interface {
Errorf(format string, args ...interface{})
Error(args ...interface{})
Warnf(format string, args ...interface{})
Warn(args ...interface{})
Infof(format string, args ...interface{})
Info(args ...interface{})
Debugf(format string, args ...interface{})
+4 -4
View File
@@ -29,10 +29,10 @@ type ApkMetadata struct {
// ApkFileRecord represents a single file listing and metadata from a APK DB entry (which may have many of these file records).
type ApkFileRecord struct {
Path string `json:"path"`
OwnerUID string `json:"ownerUid"`
OwnerGUI string `json:"ownerGid"`
Permissions string `json:"permissions"`
Checksum string `json:"checksum"`
OwnerUID string `json:"ownerUid,omitempty"`
OwnerGID string `json:"ownerGid,omitempty"`
Permissions string `json:"permissions,omitempty"`
Checksum string `json:"checksum,omitempty"`
}
func (m ApkMetadata) PackageURL() string {
+12 -5
View File
@@ -8,11 +8,18 @@ import (
// DpkgMetadata represents all captured data for a Debian package DB entry; available fields are described
// at http://manpages.ubuntu.com/manpages/xenial/man1/dpkg-query.1.html in the --showformat section.
type DpkgMetadata struct {
Package string `mapstructure:"Package" json:"package"`
Source string `mapstructure:"Source" json:"source"`
Version string `mapstructure:"Version" json:"version"`
Architecture string `mapstructure:"Architecture" json:"architecture"`
// TODO: consider keeping the remaining values as an embedded map
Package string `mapstructure:"Package" json:"package"`
Source string `mapstructure:"Source" json:"source"`
Version string `mapstructure:"Version" json:"version"`
Architecture string `mapstructure:"Architecture" json:"architecture"`
Maintainer string `mapstructure:"Maintainer" json:"maintainer"`
InstalledSize int `mapstructure:"InstalledSize" json:"installedSize"`
Files []DpkgFileRecord `json:"files"`
}
type DpkgFileRecord struct {
Path string `json:"path"`
MD5 string `json:"md5"`
}
func (m DpkgMetadata) PackageURL(d distro.Distro) string {
+19 -9
View File
@@ -9,17 +9,27 @@ import (
// RpmdbMetadata represents all captured data for a RPM DB package entry.
type RpmdbMetadata struct {
Name string `json:"name"`
Version string `json:"version"`
Epoch int `json:"epoch"`
Arch string `json:"architecture"`
Release string `json:"release"`
SourceRpm string `json:"sourceRpm"`
Size int `json:"size"`
License string `json:"license"`
Vendor string `json:"vendor"`
Name string `json:"name"`
Version string `json:"version"`
Epoch int `json:"epoch"`
Arch string `json:"architecture"`
Release string `json:"release"`
SourceRpm string `json:"sourceRpm"`
Size int `json:"size"`
License string `json:"license"`
Vendor string `json:"vendor"`
Files []RpmdbFileRecord `json:"files"`
}
type RpmdbFileRecord struct {
Path string `json:"path"`
Mode RpmdbFileMode `json:"mode"`
Size int `json:"size"`
SHA256 string `json:"sha256"`
}
type RpmdbFileMode uint16
func (m RpmdbMetadata) PackageURL(d distro.Distro) string {
pURL := packageurl.NewPackageURL(
packageurl.TypeRPM,
+2 -2
View File
@@ -62,7 +62,7 @@ func TestCycloneDxDirsPresenter(t *testing.T) {
t.Fatal(err)
}
d, err := distro.NewDistro(distro.Ubuntu, "20.04")
d, err := distro.NewDistro(distro.Ubuntu, "20.04", "debian")
if err != nil {
t.Fatal(err)
}
@@ -152,7 +152,7 @@ func TestCycloneDxImgsPresenter(t *testing.T) {
t.Fatal(err)
}
d, err := distro.NewDistro(distro.RedHat, "8")
d, err := distro.NewDistro(distro.RedHat, "8", "")
if err != nil {
t.Fatal(err)
}
+2
View File
@@ -11,6 +11,7 @@ type Artifact struct {
Type string `json:"type"`
FoundBy []string `json:"foundBy"`
Locations Locations `json:"locations,omitempty"`
Licenses []string `json:"licenses"`
Metadata interface{} `json:"metadata,omitempty"`
}
@@ -26,6 +27,7 @@ func NewArtifact(p *pkg.Package, s scope.Scope) (Artifact, error) {
Type: string(p.Type),
FoundBy: []string{p.FoundBy},
Locations: locations,
Licenses: p.Licenses,
Metadata: p.Metadata,
}, nil
}
+21 -3
View File
@@ -1,16 +1,25 @@
package json
import (
"github.com/anchore/syft/syft/distro"
"github.com/anchore/syft/syft/pkg"
"github.com/anchore/syft/syft/scope"
)
type Document struct {
Artifacts []Artifact `json:"artifacts"`
Source Source `json:"source"`
Artifacts []Artifact `json:"artifacts"`
Source Source `json:"source"`
Distro Distribution `json:"distro"`
}
func NewDocument(catalog *pkg.Catalog, s scope.Scope) (Document, error) {
// Distritbution provides information about a detected Linux Distribution
type Distribution struct {
Name string `json:"name"`
Version string `json:"version"`
IDLike string `json:"idLike"`
}
func NewDocument(catalog *pkg.Catalog, s scope.Scope, d distro.Distro) (Document, error) {
doc := Document{
Artifacts: make([]Artifact, 0),
}
@@ -20,6 +29,15 @@ func NewDocument(catalog *pkg.Catalog, s scope.Scope) (Document, error) {
return Document{}, nil
}
doc.Source = src
distroName := d.Name()
if distroName == "UnknownDistroType" {
distroName = ""
}
doc.Distro = Distribution{
Name: distroName,
Version: d.FullVersion(),
IDLike: d.IDLike,
}
for _, p := range catalog.Sorted() {
art, err := NewArtifact(p, s)
+3 -1
View File
@@ -1,6 +1,8 @@
package json
import "github.com/anchore/syft/syft/scope"
import (
"github.com/anchore/syft/syft/scope"
)
type Image struct {
Layers []Layer `json:"layers"`
+5 -2
View File
@@ -4,6 +4,7 @@ import (
"encoding/json"
"io"
"github.com/anchore/syft/syft/distro"
"github.com/anchore/syft/syft/pkg"
"github.com/anchore/syft/syft/scope"
)
@@ -11,17 +12,19 @@ import (
type Presenter struct {
catalog *pkg.Catalog
scope scope.Scope
distro distro.Distro
}
func NewPresenter(catalog *pkg.Catalog, s scope.Scope) *Presenter {
func NewPresenter(catalog *pkg.Catalog, s scope.Scope, d distro.Distro) *Presenter {
return &Presenter{
catalog: catalog,
scope: s,
distro: d,
}
}
func (pres *Presenter) Present(output io.Writer) error {
doc, err := NewDocument(pres.catalog, pres.scope)
doc, err := NewDocument(pres.catalog, pres.scope, pres.distro)
if err != nil {
return err
}
+5 -3
View File
@@ -8,6 +8,7 @@ import (
"github.com/anchore/go-testutils"
"github.com/anchore/stereoscope/pkg/file"
"github.com/anchore/stereoscope/pkg/imagetest"
"github.com/anchore/syft/syft/distro"
"github.com/anchore/syft/syft/pkg"
"github.com/anchore/syft/syft/scope"
"github.com/sergi/go-diff/diffmatchpatch"
@@ -39,12 +40,12 @@ func TestJsonDirsPresenter(t *testing.T) {
{Path: "/some/path/pkg1"},
},
})
d := distro.NewUnknownDistro()
s, err := scope.NewScopeFromDir("/some/path")
if err != nil {
t.Fatal(err)
}
pres := NewPresenter(catalog, s)
pres := NewPresenter(catalog, s, d)
// run presenter
err = pres.Present(&buffer)
@@ -100,7 +101,8 @@ func TestJsonImgsPresenter(t *testing.T) {
})
s, err := scope.NewScopeFromImage(img, scope.AllLayersScope)
pres := NewPresenter(catalog, s)
d := distro.NewUnknownDistro()
pres := NewPresenter(catalog, s, d)
// run presenter
err = pres.Present(&buffer)
@@ -9,7 +9,8 @@
],
"locations": [
"/some/path/pkg1"
]
],
"licenses": null
},
{
"name": "package-2",
@@ -20,11 +21,17 @@
],
"locations": [
"/some/path/pkg1"
]
],
"licenses": null
}
],
"source": {
"type": "directory",
"target": "/some/path"
},
"distro": {
"name": "",
"version": "",
"idLike": ""
}
}
@@ -12,7 +12,8 @@
"path": "/somefile-1.txt",
"layerIndex": 0
}
]
],
"licenses": null
},
{
"name": "package-2",
@@ -26,7 +27,8 @@
"path": "/somefile-2.txt",
"layerIndex": 1
}
]
],
"licenses": null
}
],
"source": {
@@ -35,26 +37,31 @@
"layers": [
{
"mediaType": "application/vnd.docker.image.rootfs.diff.tar.gzip",
"digest": "sha256:e158b57d6f5a96ef5fd22f2fe76c70b5ba6ff5b2619f9d83125b2aad0492ac7b",
"digest": "sha256:78783bfc74fef84f899b4977561ad1172f87753f82cc2157b06bf097e56dfbce",
"size": 22
},
{
"mediaType": "application/vnd.docker.image.rootfs.diff.tar.gzip",
"digest": "sha256:da21056e7bf4308ecea0c0836848a7fe92f38fdcf35bc09ee6d98e7ab7beeebf",
"digest": "sha256:54ec7f643dafbf9f27032a5e60afe06248c0e99b50aed54bb0fe28ea4825ccaf",
"size": 16
},
{
"mediaType": "application/vnd.docker.image.rootfs.diff.tar.gzip",
"digest": "sha256:f0e18aa6032c24659a9c741fc36ca56f589782ea132061ccf6f52b952403da94",
"digest": "sha256:ec4775a139c45b1ddf9ea8e1cb43385e92e5c0bf6ec2e3f4192372785b18c106",
"size": 27
}
],
"size": 65,
"digest": "sha256:2731251dc34951c0e50fcc643b4c5f74922dad1a5d98f302b504cf46cd5d9368",
"digest": "sha256:fedd7bcc0b90f071501b662d8e7c9ac7548b88daba6b3deedfdf33f22ed8d95b",
"mediaType": "application/vnd.docker.distribution.manifest.v2+json",
"tags": [
"stereoscope-fixture-image-simple:04e16e44161c8888a1a963720fd0443cbf7eef8101434c431de8725cd98cc9f7"
]
}
},
"distro": {
"name": "",
"version": "",
"idLike": ""
}
}
+1 -1
View File
@@ -28,7 +28,7 @@ type Presenter interface {
func GetPresenter(option Option, s scope.Scope, catalog *pkg.Catalog, d *distro.Distro) Presenter {
switch option {
case JSONPresenter:
return json.NewPresenter(catalog, s)
return json.NewPresenter(catalog, s, *d)
case TextPresenter:
return text.NewPresenter(catalog, s)
case TablePresenter:
+28 -1
View File
@@ -67,12 +67,26 @@ func (r *AllLayersResolver) FilesByPath(paths ...file.Path) ([]file.Reference, e
for _, path := range paths {
for idx, layerIdx := range r.layers {
ref := r.img.Layers[layerIdx].Tree.File(path)
tree := r.img.Layers[layerIdx].Tree
ref := tree.File(path)
if ref == nil {
// no file found, keep looking through layers
continue
}
// don't consider directories (special case: there is no path information for /)
if ref.Path == "/" {
continue
} else if r.img.FileCatalog.Exists(*ref) {
metadata, err := r.img.FileCatalog.Get(*ref)
if err != nil {
return nil, fmt.Errorf("unable to get file metadata for path=%q: %w", ref.Path, err)
}
if metadata.Metadata.IsDir {
continue
}
}
results, err := r.fileByRef(*ref, uniqueFileIDs, idx)
if err != nil {
return nil, err
@@ -97,6 +111,19 @@ func (r *AllLayersResolver) FilesByGlob(patterns ...string) ([]file.Reference, e
}
for _, ref := range refs {
// don't consider directories (special case: there is no path information for /)
if ref.Path == "/" {
continue
} else if r.img.FileCatalog.Exists(ref) {
metadata, err := r.img.FileCatalog.Get(ref)
if err != nil {
return nil, fmt.Errorf("unable to get file metadata for path=%q: %w", ref.Path, err)
}
if metadata.Metadata.IsDir {
continue
}
}
results, err := r.fileByRef(ref, uniqueFileIDs, idx)
if err != nil {
return nil, err
@@ -1,9 +1,10 @@
package resolvers
import (
"github.com/anchore/stereoscope/pkg/imagetest"
"testing"
"github.com/anchore/stereoscope/pkg/imagetest"
"github.com/anchore/stereoscope/pkg/file"
)
@@ -80,6 +81,11 @@ func TestAllLayersResolver_FilesByPath(t *testing.T) {
},
},
},
{
name: "ignore directories",
linkPath: "/bin",
resolutions: []resolution{},
},
}
for _, c := range cases {
t.Run(c.name, func(t *testing.T) {
@@ -188,6 +194,11 @@ func TestAllLayersResolver_FilesByGlob(t *testing.T) {
},
},
},
{
name: "ignore directories",
glob: "**/bin",
resolutions: []resolution{},
},
}
for _, c := range cases {
t.Run(c.name, func(t *testing.T) {
+10 -1
View File
@@ -33,12 +33,18 @@ func (s DirectoryResolver) FilesByPath(userPaths ...file.Path) ([]file.Reference
// a path relative to root should be prefixed with the resolvers directory path, otherwise it should be left as is
userStrPath = path.Join(s.Path, userStrPath)
}
_, err := os.Stat(userStrPath)
fileMeta, err := os.Stat(userStrPath)
if os.IsNotExist(err) {
continue
} else if err != nil {
log.Errorf("path (%s) is not valid: %v", userStrPath, err)
}
// don't consider directories
if fileMeta.IsDir() {
continue
}
references = append(references, file.NewFileReference(file.Path(userStrPath)))
}
@@ -69,9 +75,12 @@ func (s DirectoryResolver) FilesByGlob(patterns ...string) ([]file.Reference, er
if err != nil {
continue
}
// don't consider directories
if fileMeta.IsDir() {
continue
}
matchedPath := file.Path(match)
result = append(result, file.NewFileReference(matchedPath))
}
@@ -48,6 +48,12 @@ func TestDirectoryResolver_FilesByPath(t *testing.T) {
expected: "test-fixtures/image-symlinks/file-1.txt",
refCount: 1,
},
{
name: "directories ignored",
root: "./test-fixtures/",
input: "/image-symlinks",
refCount: 0,
},
}
for _, c := range cases {
t.Run(c.name, func(t *testing.T) {
+30 -1
View File
@@ -26,16 +26,32 @@ func (r *ImageSquashResolver) FilesByPath(paths ...file.Path) ([]file.Reference,
uniqueFiles := make([]file.Reference, 0)
for _, path := range paths {
ref := r.img.SquashedTree().File(path)
tree := r.img.SquashedTree()
ref := tree.File(path)
if ref == nil {
// no file found, keep looking through layers
continue
}
// don't consider directories (special case: there is no path information for /)
if ref.Path == "/" {
continue
} else if r.img.FileCatalog.Exists(*ref) {
metadata, err := r.img.FileCatalog.Get(*ref)
if err != nil {
return nil, fmt.Errorf("unable to get file metadata for path=%q: %w", ref.Path, err)
}
if metadata.Metadata.IsDir {
continue
}
}
// a file may be a symlink, process it as such and resolve it
resolvedRef, err := r.img.ResolveLinkByImageSquash(*ref)
if err != nil {
return nil, fmt.Errorf("failed to resolve link from img (ref=%+v): %w", ref, err)
}
if resolvedRef != nil && !uniqueFileIDs.Contains(*resolvedRef) {
uniqueFileIDs.Add(*resolvedRef)
uniqueFiles = append(uniqueFiles, *resolvedRef)
@@ -57,6 +73,19 @@ func (r *ImageSquashResolver) FilesByGlob(patterns ...string) ([]file.Reference,
}
for _, ref := range refs {
// don't consider directories (special case: there is no path information for /)
if ref.Path == "/" {
continue
} else if r.img.FileCatalog.Exists(ref) {
metadata, err := r.img.FileCatalog.Get(ref)
if err != nil {
return nil, fmt.Errorf("unable to get file metadata for path=%q: %w", ref.Path, err)
}
if metadata.Metadata.IsDir {
continue
}
}
resolvedRefs, err := r.FilesByPath(ref.Path)
if err != nil {
return nil, fmt.Errorf("failed to find files by path (ref=%+v): %w", ref, err)
@@ -1,9 +1,10 @@
package resolvers
import (
"github.com/anchore/stereoscope/pkg/imagetest"
"testing"
"github.com/anchore/stereoscope/pkg/imagetest"
"github.com/anchore/stereoscope/pkg/file"
)
@@ -44,6 +45,11 @@ func TestImageSquashResolver_FilesByPath(t *testing.T) {
resolveLayer: 8,
resolvePath: "/link-dead",
},
{
name: "ignore directories",
linkPath: "/bin",
resolvePath: "",
},
}
for _, c := range cases {
t.Run(c.name, func(t *testing.T) {
@@ -60,10 +66,20 @@ func TestImageSquashResolver_FilesByPath(t *testing.T) {
t.Fatalf("could not use resolver: %+v", err)
}
if len(refs) != 1 {
expectedRefs := 1
if c.resolvePath == "" {
expectedRefs = 0
}
if len(refs) != expectedRefs {
t.Fatalf("unexpected number of resolutions: %d", len(refs))
}
if expectedRefs == 0 {
// nothing else to assert
return
}
actual := refs[0]
if actual.Path != file.Path(c.resolvePath) {
@@ -119,6 +135,11 @@ func TestImageSquashResolver_FilesByGlob(t *testing.T) {
resolveLayer: 8,
resolvePath: "/link-dead",
},
{
name: "ignore directories",
glob: "**/bin",
resolvePath: "",
},
}
for _, c := range cases {
t.Run(c.name, func(t *testing.T) {
@@ -135,10 +156,20 @@ func TestImageSquashResolver_FilesByGlob(t *testing.T) {
t.Fatalf("could not use resolver: %+v", err)
}
if len(refs) != 1 {
expectedRefs := 1
if c.resolvePath == "" {
expectedRefs = 0
}
if len(refs) != expectedRefs {
t.Fatalf("unexpected number of resolutions: %d", len(refs))
}
if expectedRefs == 0 {
// nothing else to assert
return
}
actual := refs[0]
if actual.Path != file.Path(c.resolvePath) {
+7 -1
View File
@@ -61,9 +61,15 @@ func TestDirectoryScope(t *testing.T) {
{
desc: "path detected",
input: "test-fixtures",
inputPaths: []file.Path{file.Path("test-fixtures/path-detected")},
inputPaths: []file.Path{file.Path("test-fixtures/path-detected/.vimrc")},
expRefs: 1,
},
{
desc: "directory ignored",
input: "test-fixtures",
inputPaths: []file.Path{file.Path("test-fixtures/path-detected")},
expRefs: 0,
},
{
desc: "no files-by-path detected",
input: "test-fixtures",
+4 -3
View File
@@ -37,9 +37,10 @@ skopeo --override-os linux copy docker://docker.io/${TEST_IMAGE} docker-archive:
ls -alh ${TEST_IMAGE_TAR}
# run syft
chmod 755 ${DISTDIR}/syft_darwin_amd64/syft
${DISTDIR}/syft_darwin_amd64/syft version
SYFT_CHECK_FOR_APP_UPDATE=0 ${DISTDIR}/syft_darwin_amd64/syft docker-archive://${TEST_IMAGE_TAR} -vv -o json > ${REPORT}
SYFT_PATH="${DISTDIR}/syft-macos_darwin_amd64/syft"
chmod 755 "${SYFT_PATH}"
"${SYFT_PATH}" version
SYFT_CHECK_FOR_APP_UPDATE=0 "${SYFT_PATH}" docker-archive://${TEST_IMAGE_TAR} -vv -o json > "${REPORT}"
# keep the generated report around
mkdir -p ${RESULTSDIR}
+1 -1
View File
@@ -24,7 +24,7 @@ func TestDistroImage(t *testing.T) {
t.Fatalf("could not find distro")
}
expected, err := distro.NewDistro(distro.Busybox, "1.31.1")
expected, err := distro.NewDistro(distro.Busybox, "1.31.1", "")
if err != nil {
t.Fatalf("could not create distro: %+v", err)
}
+1 -1
View File
@@ -62,7 +62,7 @@ func testJsonSchema(t *testing.T, catalog *pkg.Catalog, theScope *scope.Scope, p
output := bytes.NewBufferString("")
d, err := distro.NewDistro(distro.CentOS, "5")
d, err := distro.NewDistro(distro.CentOS, "5", "rhel fedora")
if err != nil {
t.Fatalf("bad distro: %+v", err)
}