Merge pull request #303 from anchore/fix-site-packages-dir

Handle site packages based on which egg file is parsed

.github/ISSUE_TEMPLATE/bug_report.md

@@ -16,5 +16,5 @@ assignees: ''
 **Anything else we need to know?**:
 
 **Environment**:
-- syft version (use `syft version -v`):
+- Output of `syft version`:
 - OS (e.g: `cat /etc/os-release` or similar):

.github/scripts/mac-prepare-for-signing.sh

@@ -0,0 +1,31 @@
+#!/usr/bin/env bash
+set -eu
+
+CI_HOME="/Users/runner"
+if [[ "${HOME}" != "${CI_HOME}" ]]; then
+  printf "WARNING! It looks like this isn't the CI environment. This script modifies the macOS Keychain setup in ways you probably wouldn't want for your own machine. It also requires an Apple Developer ID Certificate that you shouldn't have outside of the CI environment.\n\nExiting early to make sure nothing bad happens.\n"
+  exit 1
+fi
+
+# Install gon (see https://github.com/mitchellh/gon for details).
+brew tap mitchellh/gon
+brew install mitchellh/gon/gon
+
+# Write signing certificate to disk from environment variable.
+CERT_FILE="$HOME/developer_id_certificate.p12"
+echo -n "$APPLE_DEVELOPER_ID_CERT" | base64 --decode > "$CERT_FILE"
+
+# In order to have all keychain interactions avoid an interactive user prompt, we need to control the password for the keychain in question, which means we need to create a new keychain into which we'll import the signing certificate and from which we'll later access this certificate during code signing.
+EPHEMERAL_KEYCHAIN="ci-ephemeral-keychain"
+EPHEMERAL_KEYCHAIN_PASSWORD="$(openssl rand -base64 100)"
+security create-keychain -p "${EPHEMERAL_KEYCHAIN_PASSWORD}" "${EPHEMERAL_KEYCHAIN}"
+
+# Import signing certificate into the keychain. (This is a pre-requisite for gon, which is invoked via goreleaser.)
+EPHEMERAL_KEYCHAIN_FULL_PATH="$HOME/Library/Keychains/${EPHEMERAL_KEYCHAIN}-db"
+security import "${CERT_FILE}" -k "${EPHEMERAL_KEYCHAIN_FULL_PATH}" -P "${APPLE_DEVELOPER_ID_CERT_PASS}" -T "$(command -v codesign)"
+
+# Setting the partition list for this certificate's private key to include "apple-tool:" and "apple:" allows the codesign command to access this keychain item without an interactive user prompt. (codesign is invoked by gon.)
+security set-key-partition-list -S "apple-tool:,apple:" -s -k "${EPHEMERAL_KEYCHAIN_PASSWORD}" "${EPHEMERAL_KEYCHAIN_FULL_PATH}"
+
+# Make this new keychain the user's default keychain, so that codesign will be able to find this certificate when we specify it during signing.
+security default-keychain -d "user" -s "${EPHEMERAL_KEYCHAIN_FULL_PATH}"

.github/scripts/mac-sign-and-notarize.sh

@@ -0,0 +1,19 @@
+#!/usr/bin/env bash
+set -eu
+
+IS_SNAPSHOT="$1"                        # e.g. "true", "false"
+
+if [[ "${IS_SNAPSHOT}" == "true" ]]; then
+  # This is a snapshot build —— skipping signing and notarization...
+  exit 0
+fi
+
+GON_CONFIG="$2"                                       # e.g. "gon.hcl"
+NEW_NAME_WITHOUT_EXTENSION="$3"                       # e.g. "./dist/syft-0.1.0"
+ORIGINAL_NAME_WITHOUT_EXTENSION="./dist/output"       # This should match dmg and zip output_path in the gon config file, without the extension.
+
+gon "${GON_CONFIG}"
+
+# Rename outputs with specified desired name
+mv -v "${ORIGINAL_NAME_WITHOUT_EXTENSION}.dmg" "${NEW_NAME_WITHOUT_EXTENSION}.dmg"
+mv -v "${ORIGINAL_NAME_WITHOUT_EXTENSION}.zip" "${NEW_NAME_WITHOUT_EXTENSION}.zip"

.github/scripts/update-version-file.sh

@@ -15,14 +15,9 @@ fi
 echo "creating and publishing version file"
 
 # create a version file for version-update checks
-echo "${VERSION}" | tee ${DISTDIR}/VERSION
+VERSION_FILE="${DISTDIR}/VERSION"
+echo "${VERSION}" | tee "${VERSION_FILE}"
 
 # upload the version file that supports the application version update check
-docker run --rm \
-    -i \
-    -e AWS_DEFAULT_REGION=us-west-2 \
-    -e AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID} \
-    -e AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY} \
-    -v $(pwd)/${DISTDIR}/:/distmount \
-    amazon/aws-cli \
-        s3 cp /distmount/VERSION s3://toolbox-data.anchore.io/${BIN}/releases/latest/VERSION
+export AWS_DEFAULT_REGION=us-west-2
+aws s3 cp "${VERSION_FILE}" s3://toolbox-data.anchore.io/${BIN}/releases/latest/VERSION

.github/workflows/acceptance-test.yaml

@@ -14,7 +14,7 @@ env:
 
 jobs:
   Build-Snapshot-Artifacts:
-    runs-on: ubuntu-latest
+    runs-on: macos-latest # We're creating these snapshot builds on macOS to be consistent with our release workflow's build process, which also takes place on macOS (due to code signing requirements).
     steps:
 
       - uses: actions/setup-go@v2
@@ -35,9 +35,9 @@ jobs:
             ${{ runner.os }}-go-${{ env.GO_VERSION }}-${{ hashFiles('**/go.sum') }}-
             ${{ runner.os }}-go-${{ env.GO_VERSION }}-
 
-      - name: Bootstrap dependencies
-        if: steps.cache.outputs.cache-hit != 'true'
-        run: make ci-bootstrap
+      - name: Bootstrap project dependencies
+        if: steps.bootstrap-cache.outputs.cache-hit != 'true'
+        run: make bootstrap
 
       - name: Import GPG key
         id: import_gpg
@@ -76,7 +76,8 @@ jobs:
   # Note: changing this job name requires making the same update in the .github/workflows/release.yaml pipeline
   Acceptance-Linux:
     needs: [ Build-Snapshot-Artifacts ]
-    runs-on: ubuntu-latest
+    # come Nov 30 2020 ubuntu-latest will be ubuntu-20.04, until then it needs to be explicitly referenced due to python 3.7 specific features being used
+    runs-on: ubuntu-20.04
     steps:
 
       - uses: actions/checkout@v2
@@ -126,7 +127,8 @@ jobs:
   # Note: changing this job name requires making the same update in the .github/workflows/release.yaml pipeline
   Inline-Compare:
     needs: [ Build-Snapshot-Artifacts ]
-    runs-on: ubuntu-latest
+    # come Nov 30 2020 ubuntu-latest will be ubuntu-20.04, until then it needs to be explicitly referenced due to python 3.7 specific features being used
+    runs-on: ubuntu-20.04
     steps:
 
       - uses: actions/checkout@v2
@@ -156,4 +158,4 @@ jobs:
           text: The syft acceptance tests have failed tragically!
         env:
           SLACK_WEBHOOK_URL: ${{ secrets.SLACK_TOOLBOX_WEBHOOK_URL }}
-        if: ${{ failure() }}
+        if: ${{ failure() }}

.github/workflows/codeql-analysis.yml

@@ -0,0 +1,71 @@
+# For most projects, this workflow file will not need changing; you simply need
+# to commit it to your repository.
+#
+# You may wish to alter this file to override the set of languages analyzed,
+# or to provide custom queries or build logic.
+name: "CodeQL Security Scan"
+
+on:
+  push:
+    branches: [main]
+  pull_request:
+    # The branches below must be a subset of the branches above
+    branches: [main]
+  schedule:
+    - cron: '0 0 * * 3'
+
+jobs:
+  analyze:
+    name: Analyze
+    runs-on: ubuntu-latest
+
+    strategy:
+      fail-fast: false
+      matrix:
+        # Override automatic language detection by changing the below list
+        # Supported options are ['csharp', 'cpp', 'go', 'java', 'javascript', 'python']
+        language: ['go', 'python']
+        # Learn more...
+        # https://docs.github.com/en/github/finding-security-vulnerabilities-and-errors-in-your-code/configuring-code-scanning#overriding-automatic-language-detection
+
+    steps:
+    - name: Checkout repository
+      uses: actions/checkout@v2
+      with:
+        # We must fetch at least the immediate parents so that if this is
+        # a pull request then we can checkout the head.
+        fetch-depth: 2
+
+    # If this run was triggered by a pull request event, then checkout
+    # the head of the pull request instead of the merge commit.
+    - run: git checkout HEAD^2
+      if: ${{ github.event_name == 'pull_request' }}
+
+    # Initializes the CodeQL tools for scanning.
+    - name: Initialize CodeQL
+      uses: github/codeql-action/init@v1
+      with:
+        languages: ${{ matrix.language }}
+        # If you wish to specify custom queries, you can do so here or in a config file.
+        # By default, queries listed here will override any specified in a config file. 
+        # Prefix the list here with "+" to use these queries and those in the config file.
+        # queries: ./path/to/local/query, your-org/your-repo/queries@main
+
+    # Autobuild attempts to build any compiled languages  (C/C++, C#, or Java).
+    # If this step fails, then you should remove it and run the build manually (see below)
+    - name: Autobuild
+      uses: github/codeql-action/autobuild@v1
+
+    # ℹ️ Command-line programs to run using the OS shell.
+    # 📚 https://git.io/JvXDl
+
+    # ✏️ If the Autobuild fails above, remove it and uncomment the following three lines
+    #    and modify them (or add more) to build your code if your project
+    #    uses a compiled language
+
+    #- run: |
+    #   make bootstrap
+    #   make release
+
+    - name: Perform CodeQL Analysis
+      uses: github/codeql-action/analyze@v1

.github/workflows/release.yaml

@@ -13,7 +13,7 @@ env:
 
 jobs:
   wait-for-checks:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-latest # This OS choice is arbitrary. None of the steps in this job are specific to either Linux or macOS.
     steps:
 
       - uses: actions/checkout@v2
@@ -82,7 +82,7 @@ jobs:
 
   release:
     needs: [ wait-for-checks ]
-    runs-on: ubuntu-latest
+    runs-on: macos-latest # Due to our code signing process, it's vital that we run our release steps on macOS.
     steps:
 
       - uses: actions/setup-go@v2
@@ -93,6 +93,7 @@ jobs:
         with:
           fetch-depth: 0
 
+      # We are expecting this cache to have been created during the "Build-Snapshot-Artifacts" job in the "Acceptance" workflow.
       - name: Restore bootstrap cache
         id: cache
         uses: actions/cache@v2
@@ -105,10 +106,6 @@ jobs:
             ${{ runner.os }}-go-${{ env.GO_VERSION }}-${{ hashFiles('**/go.sum') }}-
             ${{ runner.os }}-go-${{ env.GO_VERSION }}-
 
-      - name: Bootstrap dependencies
-        if: steps.cache.outputs.cache-hit != 'true'
-        run: make ci-bootstrap
-
       - name: Import GPG key
         id: import_gpg
         uses: crazy-max/ghaction-import-gpg@v2
@@ -126,12 +123,16 @@ jobs:
       - name: Build & publish release artifacts
         run: make release
         env:
-          GITHUB_TOKEN: ${{ secrets.TEMP_RELEASE_TOKEN }}
+          GITHUB_TOKEN: ${{ secrets.ANCHORE_GIT_READ_TOKEN }}
           GPG_PRIVATE_KEY: ${{ secrets.SIGNING_GPG_PRIVATE_KEY }}
           PASSPHRASE: ${{ secrets.SIGNING_GPG_PASSPHRASE }}
           SIGNING_FINGERPRINT: ${{ steps.import_gpg.outputs.fingerprint }}
           AWS_ACCESS_KEY_ID: ${{ secrets.TOOLBOX_AWS_ACCESS_KEY_ID }}
           AWS_SECRET_ACCESS_KEY: ${{ secrets.TOOLBOX_AWS_SECRET_ACCESS_KEY }}
+          APPLE_DEVELOPER_ID_CERT: ${{ secrets.APPLE_DEVELOPER_ID_CERT }} # Used during macOS code signing.
+          APPLE_DEVELOPER_ID_CERT_PASS: ${{ secrets.APPLE_DEVELOPER_ID_CERT_PASS }} # Used during macOS code signing.
+          AC_USERNAME: ${{ secrets.ENG_CI_APPLE_ID }} # Used during macOS notarization.
+          AC_PASSWORD: ${{ secrets.ENG_CI_APPLE_ID_PASS }} # Used during macOS notarization.
 
       - uses: 8398a7/action-slack@v3
         with:
@@ -145,4 +146,4 @@ jobs:
       - uses: actions/upload-artifact@v2
         with:
           name: artifacts
-          path: dist/**/*
+          path: dist/**/*

.github/workflows/static-unit-integration.yaml

@@ -2,6 +2,7 @@ name: 'Static Analysis + Unit + Integration'
 on:
   workflow_dispatch:
   push:
+  pull_request:
 jobs:
 
   Static-Analysis:

.gitignore

@@ -3,6 +3,7 @@ CHANGELOG.md
 /snapshot
 .server/
 .vscode/
+.history/
 *.fingerprint
 *.tar
 *.jar
@@ -29,3 +30,8 @@ coverage.txt
 
 # Output of the go coverage tool, specifically when used with LiteIDE
 *.out
+
+# macOS Finder metadata
+.DS_STORE
+
+*.profile

.golangci.yaml

@@ -31,7 +31,6 @@ linters:
     - misspell
     - nakedret
     - nolintlint
-    - prealloc
     - rowserrcheck
     - scopelint
     - staticcheck
@@ -55,5 +54,6 @@ linters:
 #    - lll         # without a way to specify per-line exception cases, this is not usable
 #    - maligned    # this is an excellent linter, but tricky to optimize and we are not sensitive to memory layout optimizations
 #    - nestif
+#    - prealloc    # following this rule isn't consistently a good idea, as it sometimes forces unnecessary allocations that result in less idiomatic code
 #    - testpackage
-#    - wsl
+#    - wsl

.goreleaser.yaml

@@ -6,18 +6,33 @@ release:
   # If set to true, will not auto-publish the release. This is done to allow us to review the changelog before publishing.
   draft: true
 
-signs:
-  - artifacts: checksum
-    args: ["--output", "${signature}", "--detach-sign", "${artifact}"]
-
 builds:
   - binary: syft
+    id: syft
     env:
       - CGO_ENABLED=0
     goos:
       # windows not supported yet (due to jotframe)
       # - windows
       - linux
+    goarch:
+      - amd64
+    # Set the modified timestamp on the output binary to the git timestamp (to ensure a reproducible build)
+    mod_timestamp: '{{ .CommitTimestamp }}'
+    ldflags: |
+      -w
+      -s
+      -extldflags '-static'
+      -X github.com/anchore/syft/internal/version.version={{.Version}}
+      -X github.com/anchore/syft/internal/version.gitCommit={{.Commit}}
+      -X github.com/anchore/syft/internal/version.buildDate={{.Date}}
+      -X github.com/anchore/syft/internal/version.gitTreeState={{.Env.BUILD_GIT_TREE_STATE}}
+  # For more info on this macOS build, see: https://github.com/mitchellh/gon#usage-with-goreleaser
+  - binary: syft
+    id: syft-macos
+    env:
+      - CGO_ENABLED=0
+    goos:
       - darwin
     goarch:
       - amd64
@@ -32,6 +47,34 @@ builds:
       -X github.com/anchore/syft/internal/version.buildDate={{.Date}}
       -X github.com/anchore/syft/internal/version.gitTreeState={{.Env.BUILD_GIT_TREE_STATE}}
 
+archives:
+  - format: tar.gz
+    builds:
+      - syft # i.e. Linux only
+  - format: zip # This is a hack! We don't actually intend to use _this_ ZIP file, we just need goreleaser to consider the ZIP file produced by gon (which will have the same file name) to be an artifact so we can use it downstream in publishing (e.g. to a homebrew tap)
+    id: syft-zip
+    builds:
+      - syft-macos
+
+signs:
+  - artifacts: checksum
+    cmd: sh
+    args:
+      - '-c'
+      # we should not include the zip artifact, as the artifact is mutated throughout the next macOS notarization step
+      # note: sed -i is not portable
+      - 'sed "/.*\.zip/d" ${artifact} > tmpfile && mv tmpfile ${artifact} && gpg --output ${signature} --detach-sign ${artifact}'
+  - id: syft-macos-signing
+    ids:
+      - syft-macos
+    cmd: ./.github/scripts/mac-sign-and-notarize.sh
+    signature: "syft_${VERSION}_darwin_amd64.dmg" # This is somewhat unintuitive. This gets the DMG file recognized as an artifact. In fact, both a DMG and a ZIP file are being produced by this signing step.
+    args:
+      - "{{ .IsSnapshot }}"
+      - "gon.hcl"
+      - "./dist/syft_{{ .Version }}_darwin_amd64"
+    artifacts: all
+
 nfpms:
   - license: "Apache 2.0"
     maintainer: "Anchore, Inc"
@@ -47,10 +90,3 @@ brews:
       name: homebrew-syft
     homepage: *website
     description: *description
-
-archives:
-  - format: tar.gz
-    format_overrides:
-      - goos: windows
-        format: zip
-

Makefile

@@ -15,7 +15,10 @@ RESET := $(shell tput -T linux sgr0)
 TITLE := $(BOLD)$(PURPLE)
 SUCCESS := $(BOLD)$(GREEN)
 # the quality gate lower threshold for unit test total % coverage (by function statements)
-COVERAGE_THRESHOLD := 72
+COVERAGE_THRESHOLD := 68
+# CI cache busting values; change these if you want CI to not use previous stored cache
+COMPARE_CACHE_BUSTER="f7e689d76a9"
+INTEGRATION_CACHE_BUSTER="789bacdf"
 
 ## Build variables
 DISTDIR=./dist
@@ -74,6 +77,10 @@ help:
 ci-bootstrap:
 	DEBIAN_FRONTEND=noninteractive sudo apt update && sudo -E apt install -y bc jq libxml2-utils
 
+.PHONY:
+ci-bootstrap-mac:
+	github_changelog_generator --version || sudo gem install github_changelog_generator
+
 .PHONY: bootstrap
 bootstrap: ## Download and install all go dependencies (+ prep tooling in the ./tmp dir)
 	$(call title,Bootstrapping dependencies)
@@ -122,7 +129,7 @@ validate-cyclonedx-schema:
 .PHONY: unit
 unit: fixtures ## Run unit tests (with coverage)
 	$(call title,Running unit tests)
-	go test -coverprofile $(COVER_REPORT) ./...
+	go test -coverprofile $(COVER_REPORT) $(shell go list ./... | grep -v anchore/syft/test)
 	@go tool cover -func $(COVER_REPORT) | grep total |  awk '{print substr($$3, 1, length($$3)-1)}' > $(COVER_TOTAL)
 	@echo "Coverage: $$(cat $(COVER_TOTAL))"
 	@if [ $$(echo "$$(cat $(COVER_TOTAL)) >= $(COVERAGE_THRESHOLD)" | bc -l) -ne 1 ]; then echo "$(RED)$(BOLD)Failed coverage quality gate (> $(COVERAGE_THRESHOLD)%)$(RESET)" && false; fi
@@ -130,11 +137,13 @@ unit: fixtures ## Run unit tests (with coverage)
 .PHONY: integration
 integration: ## Run integration tests
 	$(call title,Running integration tests)
-	go test -tags=integration ./test/integration
+
+	go test -v ./test/integration
+
 
 # note: this is used by CI to determine if the integration test fixture cache (docker image tars) should be busted
 integration-fingerprint:
-	find test/integration/test-fixtures/image-* -type f -exec md5sum {} + | awk '{print $1}' | sort | md5sum | tee test/integration/test-fixtures/cache.fingerprint
+	find test/integration/test-fixtures/image-* -type f -exec md5sum {} + | awk '{print $1}' | sort | md5sum | tee test/integration/test-fixtures/cache.fingerprint && echo "$(INTEGRATION_CACHE_BUSTER)" >> test/integration/test-fixtures/cache.fingerprint
 
 .PHONY: java-packages-fingerprint
 java-packages-fingerprint:
@@ -147,17 +156,8 @@ fixtures:
 	cd syft/cataloger/java/test-fixtures/java-builds && make
 
 .PHONY: generate-json-schema
-generate-json-schema: clean-json-schema-examples integration ## Generate a new json schema for the json presenter, derived from integration test cases
-	docker run \
-		-i \
-		--rm \
-		-v $(shell pwd)/schema/json:/work \
-		-w /work \
-		python:3.8 \
-			bash -x -c "\
-				pip install -r requirements.txt && \
-				python generate.py \
-			"
+generate-json-schema:  ## Generate a new json schema
+	cd schema/json && go run generate.go
 
 .PHONY: clear-test-cache
 clear-test-cache: ## Delete all test cache (built docker image tars)
@@ -191,7 +191,7 @@ acceptance-linux: acceptance-test-deb-package-install acceptance-test-rpm-packag
 # note: this is used by CI to determine if the inline-scan report cache should be busted for the inline-compare tests
 .PHONY: compare-fingerprint
 compare-fingerprint:
-	find test/inline-compare/* -type f -exec md5sum {} + | grep -v '\-reports' | grep -v 'fingerprint' | awk '{print $1}' | sort | md5sum | tee test/inline-compare/inline-compare.fingerprint
+	find test/inline-compare/* -type f -exec md5sum {} + | grep -v '\-reports' | grep -v 'fingerprint' | awk '{print $1}' | sort | md5sum | tee test/inline-compare/inline-compare.fingerprint && echo "$(COMPARE_CACHE_BUSTER)" >> test/inline-compare/inline-compare.fingerprint
 
 .PHONY: compare-snapshot
 compare-snapshot: $(SNAPSHOTDIR) ## Compare the reports of a run of a snapshot build of syft against inline-scan
@@ -224,8 +224,7 @@ acceptance-test-rpm-package-install: $(SNAPSHOTDIR)
 changelog-release:
 	@echo "Last tag: $(SECOND_TO_LAST_TAG)"
 	@echo "Current tag: $(VERSION)"
-	@docker run -i --rm  \
-		-v "$(shell pwd)":/usr/local/src/your-app ferrarimarco/github-changelog-generator \
+	@github_changelog_generator \
 		--user anchore \
 		--project $(BIN) \
 		-t ${GITHUB_TOKEN} \
@@ -239,8 +238,9 @@ changelog-release:
 
 .PHONY: changelog-unreleased
 changelog-unreleased: ## show the current changelog that will be produced on the next release (note: requires GITHUB_TOKEN set)
-	@docker run -it --rm  \
-		-v "$(shell pwd)":/usr/local/src/your-app ferrarimarco/github-changelog-generator \
+	@docker run -it --rm \
+		-v "$(shell pwd)":/usr/local/src/your-app \
+		ferrarimarco/github-changelog-generator \
 		--user anchore \
 		--project $(BIN) \
 		-t ${GITHUB_TOKEN} \
@@ -256,17 +256,24 @@ changelog-unreleased: ## show the current changelog that will be produced on the
 			/CHANGELOG.md
 
 .PHONY: release
-release: clean-dist changelog-release ## Build and publish final binaries and packages
+release: clean-dist ci-bootstrap-mac changelog-release ## Build and publish final binaries and packages. Intended to be run only on macOS.
 	$(call title,Publishing release artifacts)
+
+	# Prepare for macOS-specific signing process
+	.github/scripts/mac-prepare-for-signing.sh
+
 	# create a config with the dist dir overridden
 	echo "dist: $(DISTDIR)" > $(TEMPDIR)/goreleaser.yaml
 	cat .goreleaser.yaml >> $(TEMPDIR)/goreleaser.yaml
 
-	# release
-	bash -c "BUILD_GIT_TREE_STATE=$(GITTREESTATE) $(TEMPDIR)/goreleaser \
-		--rm-dist \
-		--config $(TEMPDIR)/goreleaser.yaml \
-		--release-notes <(cat CHANGELOG.md)"
+	# release (note the version transformation from v0.7.0 --> 0.7.0)
+	bash -c "\
+		BUILD_GIT_TREE_STATE=$(GITTREESTATE) \
+		VERSION=$(VERSION:v%=%) \
+		$(TEMPDIR)/goreleaser \
+			--rm-dist \
+			--config $(TEMPDIR)/goreleaser.yaml \
+			--release-notes <(cat CHANGELOG.md)"
 
 	# verify checksum signatures
 	.github/scripts/verify-signature.sh "$(DISTDIR)"
@@ -275,7 +282,7 @@ release: clean-dist changelog-release ## Build and publish final binaries and pa
 	.github/scripts/update-version-file.sh "$(DISTDIR)" "$(VERSION)"
 
 .PHONY: clean
-clean: clean-dist clean-snapshot clean-json-schema-examples ## Remove previous builds and result reports
+clean: clean-dist clean-snapshot ## Remove previous builds and result reports
 	rm -rf $(RESULTSDIR)/*
 
 .PHONY: clean-snapshot
@@ -285,7 +292,3 @@ clean-snapshot:
 .PHONY: clean-dist
 clean-dist:
 	rm -rf $(DISTDIR) $(TEMPDIR)/goreleaser.yaml
-
-.PHONY: clean-json-schema-examples
-clean-json-schema-examples:
-	rm -f schema/json/examples/*

README.md

@@ -1,6 +1,7 @@
 # syft
 
-[![CircleCI](https://circleci.com/gh/anchore/syft.svg?style=svg)](https://circleci.com/gh/anchore/syft)
+[![Static Analysis + Unit + Integration](https://github.com/anchore/syft/workflows/Static%20Analysis%20+%20Unit%20+%20Integration/badge.svg)](https://github.com/anchore/syft/actions?query=workflow%3A%22Static+Analysis+%2B+Unit+%2B+Integration%22)
+[![Acceptance](https://github.com/anchore/syft/workflows/Acceptance/badge.svg)](https://github.com/anchore/syft/actions?query=workflow%3AAcceptance)
 [![Go Report Card](https://goreportcard.com/badge/github.com/anchore/syft)](https://goreportcard.com/report/github.com/anchore/syft)
 [![GitHub release](https://img.shields.io/github/release/anchore/syft.svg)](https://github.com/anchore/syft/releases/latest)
 [![License: Apache-2.0](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](https://github.com/anchore/syft/blob/main/LICENSE)
@@ -15,7 +16,7 @@ A CLI tool and go library for generating a Software Bill of Materials (SBOM) fro
 - Linux distribution identification (supports Alpine, BusyBox, CentOS/RedHat, Debian/Ubuntu flavored distributions)
 - Supports Docker and OCI image formats
 
-> :warning: **This is pre-release software** and it may not work as expected. If you encounter an issue, please [let us know using the issue tracker](https://github.com/anchore/syft/issues).
+If you encounter an issue, please [let us know using the issue tracker](https://github.com/anchore/syft/issues).
 
 ## Getting started
 
@@ -53,26 +54,21 @@ Where the `format`s available are:
 
 ## Installation
 
-**Recommended**
+**Recommended (macOS and Linux)**
 ```bash
 # install the latest version to /usr/local/bin
 curl -sSfL https://raw.githubusercontent.com/anchore/syft/main/install.sh | sh -s -- -b /usr/local/bin
 
 # install a specific version into a specific dir
-curl -sSfL https://raw.githubusercontent.com/anchore/syft/main/install.sh | sh -s <RELEASE_VERSION> -b <SOME_BIN_PATH>
+curl -sSfL https://raw.githubusercontent.com/anchore/syft/main/install.sh | sh -s -- -b <SOME_BIN_PATH> <RELEASE_VERSION>
 ```
 
-**macOS**
+**Homebrew (macOS)**
 ```bash
 brew tap anchore/syft
 brew install syft
 ```
 
-You may experience a "macOS cannot verify app is free from malware" error upon running Syft because it is not yet signed and notarized. You can override this using `xattr`.
-```bash
-xattr -rd com.apple.quarantine syft
-```
-
 ## Configuration
 
 Configuration search paths:
@@ -85,31 +81,58 @@ Configuration search paths:
 Configuration options (example values are the default):
 
 ```yaml
-# same as -o ; the output format of the SBOM report (options: table, text, json)
+# the output format of the SBOM report (options: table, text, json)
+# same as -o ; SYFT_OUTPUT env var
 output: "table"
 
-# same as -s ; the search space to look for packages (options: all-layers, squashed)
+# the search space to look for packages (options: all-layers, squashed)
+# same as -s ; SYFT_SCOPE env var
 scope: "squashed"
 
-# same as -q ; suppress all output (except for the SBOM report)
+# suppress all output (except for the SBOM report)
+# same as -q ; SYFT_QUIET env var
 quiet: false
 
+# enable/disable checking for application updates on startup
+# same as SYFT_CHECK_FOR_APP_UPDATE env var
+check-for-app-update: true
+
 log:
   # use structured logging
+  # same as SYFT_LOG_STRUCTURED env var
   structured: false
 
   # the log level; note: detailed logging suppress the ETUI
+  # same as SYFT_LOG_LEVEL env var
   level: "error"
 
   # location to write the log file (default is not to have a log file)
+  # same as SYFT_LOG_FILE env var
   file: ""
 
-# enable/disable checking for application updates on startup
-check-for-app-update: true
-```
+anchore:
+  # (feature-preview) enable uploading of results to Anchore Enterprise automatically (supported on Enterprise 3.0+)
+  # same as SYFT_ANCHORE_UPLOAD_ENABLED env var
+  upload-enabled: false
 
-## Future plans
+  # (feature-preview) the Anchore Enterprise Host or URL to upload results to (supported on Enterprise 3.0+)
+  # same as -H ; SYFT_ANCHORE_HOST env var
+  host: ""
 
-The following areas of potential development are currently being investigated:
-- Add CycloneDX to list of output formats
-- Establish a stable interchange format w/Grype
+  # (feature-preview) the path after the host to the Anchore External API (supported on Enterprise 3.0+)
+  # same as SYFT_ANCHORE_PATH env var
+  path: ""
+
+  # (feature-preview) the username to authenticate against Anchore Enterprise (supported on Enterprise 3.0+)
+  # same as -u ; SYFT_ANCHORE_USERNAME env var
+  username: ""
+
+  # (feature-preview) the password to authenticate against Anchore Enterprise (supported on Enterprise 3.0+)
+  # same as -p ; SYFT_ANCHORE_PASSWORD env var
+  password: ""
+
+  # (feature-preview) path to dockerfile to be uploaded with the syft results to Anchore Enterprise (supported on Enterprise 3.0+)
+  # same as -d ; SYFT_ANCHORE_DOCKERFILE env var
+  dockerfile: ""
+
+```

cmd/cmd.go

@@ -4,20 +4,17 @@ import (
 	"fmt"
 	"os"
 
-	"github.com/spf13/cobra"
-
-	"github.com/anchore/syft/syft/presenter"
-	"github.com/anchore/syft/syft/scope"
-
 	"github.com/anchore/stereoscope"
 	"github.com/anchore/syft/internal/config"
-	"github.com/anchore/syft/internal/format"
 	"github.com/anchore/syft/internal/log"
 	"github.com/anchore/syft/internal/logger"
 	"github.com/anchore/syft/syft"
+	"github.com/anchore/syft/syft/presenter"
+	"github.com/anchore/syft/syft/source"
+	"github.com/gookit/color"
+	"github.com/spf13/cobra"
 	"github.com/spf13/viper"
 	"github.com/wagoodman/go-partybus"
-	"gopkg.in/yaml.v2"
 )
 
 var appConfig *config.Application
@@ -49,15 +46,20 @@ func setGlobalCliOptions() {
 	// scan options
 	flag := "scope"
 	rootCmd.Flags().StringP(
-		"scope", "s", scope.SquashedScope.String(),
-		fmt.Sprintf("selection of layers to catalog, options=%v", scope.Options))
+		"scope", "s", source.SquashedScope.String(),
+		fmt.Sprintf("selection of layers to catalog, options=%v", source.AllScopes))
 	if err := viper.BindPFlag(flag, rootCmd.Flags().Lookup(flag)); err != nil {
 		fmt.Printf("unable to bind flag '%s': %+v", flag, err)
 		os.Exit(1)
 	}
 
+	setGlobalFormatOptions()
+	setGlobalUploadOptions()
+}
+
+func setGlobalFormatOptions() {
 	// output & formatting options
-	flag = "output"
+	flag := "output"
 	rootCmd.Flags().StringP(
 		flag, "o", string(presenter.TablePresenter),
 		fmt.Sprintf("report output formatter, options=%v", presenter.Options),
@@ -80,12 +82,67 @@ func setGlobalCliOptions() {
 	rootCmd.Flags().CountVarP(&cliOpts.Verbosity, "verbose", "v", "increase verbosity (-v = info, -vv = debug)")
 }
 
+func setGlobalUploadOptions() {
+	flag := "host"
+	rootCmd.Flags().StringP(
+		flag, "H", "",
+		"the hostname or URL of the Anchore Enterprise instance to upload to",
+	)
+	if err := viper.BindPFlag("anchore.host", rootCmd.Flags().Lookup(flag)); err != nil {
+		fmt.Printf("unable to bind flag '%s': %+v", flag, err)
+		os.Exit(1)
+	}
+
+	flag = "username"
+	rootCmd.Flags().StringP(
+		flag, "u", "",
+		"the username to authenticate against Anchore Enterprise",
+	)
+	if err := viper.BindPFlag("anchore.username", rootCmd.Flags().Lookup(flag)); err != nil {
+		fmt.Printf("unable to bind flag '%s': %+v", flag, err)
+		os.Exit(1)
+	}
+
+	flag = "password"
+	rootCmd.Flags().StringP(
+		flag, "p", "",
+		"the password to authenticate against Anchore Enterprise",
+	)
+	if err := viper.BindPFlag("anchore.password", rootCmd.Flags().Lookup(flag)); err != nil {
+		fmt.Printf("unable to bind flag '%s': %+v", flag, err)
+		os.Exit(1)
+	}
+
+	flag = "dockerfile"
+	rootCmd.Flags().StringP(
+		flag, "d", "",
+		"include dockerfile for upload to Anchore Enterprise",
+	)
+	if err := viper.BindPFlag("anchore.dockerfile", rootCmd.Flags().Lookup(flag)); err != nil {
+		fmt.Printf("unable to bind flag '#{flag}': #{err}")
+		os.Exit(1)
+	}
+
+	flag = "overwrite-existing-image"
+	rootCmd.Flags().Bool(
+		flag, false,
+		"overwrite an existing image during the upload to Anchore Enterprise",
+	)
+	if err := viper.BindPFlag("anchore.overwrite-existing-image", rootCmd.Flags().Lookup(flag)); err != nil {
+		fmt.Printf("unable to bind flag '#{flag}': #{err}")
+		os.Exit(1)
+	}
+}
+
 func initAppConfig() {
-	cfg, err := config.LoadConfigFromFile(viper.GetViper(), &cliOpts)
+	cfgVehicle := viper.GetViper()
+	wasHostnameSet := rootCmd.Flags().Changed("host")
+	cfg, err := config.LoadApplicationConfig(cfgVehicle, cliOpts, wasHostnameSet)
 	if err != nil {
 		fmt.Printf("failed to load application config: \n\t%+v\n", err)
 		os.Exit(1)
 	}
+
 	appConfig = cfg
 }
 
@@ -101,18 +158,12 @@ func initLogging() {
 	logWrapper := logger.NewLogrusLogger(cfg)
 	syft.SetLogger(logWrapper)
 	stereoscope.SetLogger(&logger.LogrusNestedLogger{
-		Logger: logWrapper.Logger.WithField("from-lib", "steroscope"),
+		Logger: logWrapper.Logger.WithField("from-lib", "stereoscope"),
 	})
 }
 
 func logAppConfig() {
-	appCfgStr, err := yaml.Marshal(&appConfig)
-
-	if err != nil {
-		log.Debugf("Could not display application config: %+v", err)
-	} else {
-		log.Debugf("Application config:\n%+v", format.Magenta.Format(string(appCfgStr)))
-	}
+	log.Debugf("Application config:\n%+v", color.Magenta.Sprint(appConfig.String()))
 }
 
 func initEventBus() {

cmd/root.go

@@ -3,17 +3,24 @@ package cmd
 import (
 	"context"
 	"fmt"
+	"io/ioutil"
 	"os"
 	"strings"
 
+	"github.com/pkg/profile"
+
 	"github.com/anchore/syft/internal"
+	"github.com/anchore/syft/internal/anchore"
 	"github.com/anchore/syft/internal/bus"
 	"github.com/anchore/syft/internal/log"
 	"github.com/anchore/syft/internal/ui"
 	"github.com/anchore/syft/internal/version"
 	"github.com/anchore/syft/syft"
+	"github.com/anchore/syft/syft/distro"
 	"github.com/anchore/syft/syft/event"
+	"github.com/anchore/syft/syft/pkg"
 	"github.com/anchore/syft/syft/presenter"
+	"github.com/anchore/syft/syft/source"
 	"github.com/docker/docker/api/types"
 	"github.com/docker/docker/api/types/filters"
 	"github.com/docker/docker/client"
@@ -23,7 +30,7 @@ import (
 
 var rootCmd = &cobra.Command{
 	Use:   fmt.Sprintf("%s [SOURCE]", internal.ApplicationName),
-	Short: "A tool for generating a Software Bill Of Materials (SBOM) from container images and filesystems",
+	Short: "A tool for generating a Software Bill Of Materials (PackageSBOM) from container images and filesystems",
 	Long: internal.Tprintf(`
 Supports the following image sources:
     {{.appName}} yourrepo/yourimage:tag     defaults to using images from a Docker daemon
@@ -48,7 +55,20 @@ You can also explicitly specify the scheme to use:
 			}
 			os.Exit(1)
 		}
+
+		if appConfig.Dev.ProfileCPU && appConfig.Dev.ProfileMem {
+			log.Errorf("cannot profile CPU and memory simultaneously")
+			os.Exit(1)
+		}
+
+		if appConfig.Dev.ProfileCPU {
+			defer profile.Start(profile.CPUProfile).Stop()
+		} else if appConfig.Dev.ProfileMem {
+			defer profile.Start(profile.MemProfile).Stop()
+		}
+
 		err := doRunCmd(cmd, args)
+
 		if err != nil {
 			log.Errorf(err.Error())
 			os.Exit(1)
@@ -92,20 +112,85 @@ func startWorker(userInput string) <-chan error {
 			}
 		}
 
-		catalog, scope, distro, err := syft.Catalog(userInput, appConfig.ScopeOpt)
+		src, catalog, distro, err := syft.Catalog(userInput, appConfig.ScopeOpt)
 		if err != nil {
 			errs <- fmt.Errorf("failed to catalog input: %+v", err)
 			return
 		}
 
+		if appConfig.Anchore.UploadEnabled {
+			if err := doImport(src, src.Metadata, catalog, distro); err != nil {
+				errs <- err
+				return
+			}
+		}
+
 		bus.Publish(partybus.Event{
 			Type:  event.CatalogerFinished,
-			Value: presenter.GetPresenter(appConfig.PresenterOpt, *scope, catalog, distro),
+			Value: presenter.GetPresenter(appConfig.PresenterOpt, src.Metadata, catalog, distro),
 		})
 	}()
 	return errs
 }
 
+func doImport(src source.Source, s source.Metadata, catalog *pkg.Catalog, d *distro.Distro) error {
+	// TODO: ETUI element for this
+	log.Infof("uploading results to %s", appConfig.Anchore.Host)
+
+	if src.Metadata.Scheme != source.ImageScheme {
+		return fmt.Errorf("unable to upload results: only images are supported")
+	}
+
+	var dockerfileContents []byte
+	if appConfig.Anchore.Dockerfile != "" {
+		if _, err := os.Stat(appConfig.Anchore.Dockerfile); os.IsNotExist(err) {
+			return fmt.Errorf("unable to read dockerfile=%q: %w", appConfig.Anchore.Dockerfile, err)
+		}
+
+		fh, err := os.Open(appConfig.Anchore.Dockerfile)
+		if err != nil {
+			return fmt.Errorf("unable to open dockerfile=%q: %w", appConfig.Anchore.Dockerfile, err)
+		}
+
+		dockerfileContents, err = ioutil.ReadAll(fh)
+		if err != nil {
+			return fmt.Errorf("unable to read dockerfile=%q: %w", appConfig.Anchore.Dockerfile, err)
+		}
+	}
+
+	var scheme string
+	var hostname = appConfig.Anchore.Host
+	urlFields := strings.Split(hostname, "://")
+	if len(urlFields) > 1 {
+		scheme = urlFields[0]
+		hostname = urlFields[1]
+	}
+
+	c, err := anchore.NewClient(anchore.Configuration{
+		Hostname: hostname,
+		Username: appConfig.Anchore.Username,
+		Password: appConfig.Anchore.Password,
+		Scheme:   scheme,
+	})
+	if err != nil {
+		return fmt.Errorf("failed to create anchore client: %+v", err)
+	}
+
+	importCfg := anchore.ImportConfig{
+		ImageMetadata:           src.Image.Metadata,
+		SourceMetadata:          s,
+		Catalog:                 catalog,
+		Distro:                  d,
+		Dockerfile:              dockerfileContents,
+		OverwriteExistingUpload: appConfig.Anchore.OverwriteExistingImage,
+	}
+
+	if err := c.Import(context.Background(), importCfg); err != nil {
+		return fmt.Errorf("failed to upload results to host=%s: %+v", appConfig.Anchore.Host, err)
+	}
+	return nil
+}
+
 func doRunCmd(_ *cobra.Command, args []string) error {
 	userInput := args[0]
 	errs := startWorker(userInput)

go.mod

@@ -4,14 +4,18 @@ go 1.14
 
 require (
 	github.com/adrg/xdg v0.2.1
-	github.com/anchore/go-rpmdb v0.0.0-20200811175839-cbc751c28e8e
+	github.com/alecthomas/jsonschema v0.0.0-20200530073317-71f438968921
+	github.com/anchore/client-go v0.0.0-20201216213038-a486b838e238
+	github.com/anchore/go-rpmdb v0.0.0-20201106153645-0043963c2e12
 	github.com/anchore/go-testutils v0.0.0-20200925183923-d5f45b0d3c04
 	github.com/anchore/go-version v1.2.2-0.20200701162849-18adb9c92b9b
-	github.com/anchore/stereoscope v0.0.0-20200925184903-c82da54e98fe
-	github.com/bmatcuk/doublestar v1.3.1
+	github.com/anchore/stereoscope v0.0.0-20201210022249-091f9bddb42e
+	github.com/antihax/optional v1.0.0
+	github.com/bmatcuk/doublestar v1.3.3
 	github.com/docker/docker v17.12.0-ce-rc1.0.20200309214505-aa6a9891b09c+incompatible
 	github.com/dustin/go-humanize v1.0.0
-	github.com/go-test/deep v1.0.6
+	github.com/facebookincubator/nvdtools v0.1.4
+	github.com/go-test/deep v1.0.7
 	github.com/google/uuid v1.1.1
 	github.com/gookit/color v1.2.7
 	github.com/hashicorp/go-multierror v1.1.0
@@ -21,6 +25,8 @@ require (
 	github.com/olekukonko/tablewriter v0.0.4
 	github.com/package-url/packageurl-go v0.1.0
 	github.com/pelletier/go-toml v1.8.0
+	github.com/pkg/profile v1.5.0
+	github.com/scylladb/go-set v1.0.2
 	github.com/sergi/go-diff v1.1.0
 	github.com/sirupsen/logrus v1.6.0
 	github.com/spf13/afero v1.2.2

go.sum

@@ -119,20 +119,28 @@ github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdko
 github.com/StackExchange/wmi v0.0.0-20180116203802-5d049714c4a6/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg=
 github.com/adrg/xdg v0.2.1 h1:VSVdnH7cQ7V+B33qSJHTCRlNgra1607Q8PzEmnvb2Ic=
 github.com/adrg/xdg v0.2.1/go.mod h1:ZuOshBmzV4Ta+s23hdfFZnBsdzmoR3US0d7ErpqSbTQ=
+github.com/alecthomas/jsonschema v0.0.0-20200530073317-71f438968921 h1:T3+cD5fYvuH36h7EZq+TDpm+d8a6FSD4pQsbmuGGQ8o=
+github.com/alecthomas/jsonschema v0.0.0-20200530073317-71f438968921/go.mod h1:/n6+1/DWPltRLWL/VKyUxg6tzsl5kHUCcraimt4vr60=
 github.com/alecthomas/kingpin v2.2.6+incompatible/go.mod h1:59OFYbFVLKQKq+mqrL6Rw5bR0c3ACQaawgXx0QYndlE=
 github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
 github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
 github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
 github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho=
-github.com/anchore/go-rpmdb v0.0.0-20200811175839-cbc751c28e8e h1:kty6r0R2JeaNPeWKSYDC+HW3hkqwFh4PP5TQ8pUPYFw=
-github.com/anchore/go-rpmdb v0.0.0-20200811175839-cbc751c28e8e/go.mod h1:iYuIG0Nai8dR0ri3LhZQKUyO1loxUWAGvoWhXDmjy1A=
+github.com/anchore/client-go v0.0.0-20201210022459-59e7a0749c74 h1:9kkKTIyXJC+/syUcY6KWxFoJZJ+GWwrIscF+gBY067k=
+github.com/anchore/client-go v0.0.0-20201210022459-59e7a0749c74/go.mod h1:FaODhIA06mxO1E6R32JE0TL1JWZZkmjRIAd4ULvHUKk=
+github.com/anchore/client-go v0.0.0-20201216213038-a486b838e238 h1:/iI+1cj1a27ow0wj378pPJIm8sCSy6I21Tz6oLbLDQY=
+github.com/anchore/client-go v0.0.0-20201216213038-a486b838e238/go.mod h1:FaODhIA06mxO1E6R32JE0TL1JWZZkmjRIAd4ULvHUKk=
+github.com/anchore/go-rpmdb v0.0.0-20201106153645-0043963c2e12 h1:xbeIbn5F52JVx3RUIajxCj8b0y+9lywspql4sFhcxWQ=
+github.com/anchore/go-rpmdb v0.0.0-20201106153645-0043963c2e12/go.mod h1:juoyWXIj7sJ1IDl4E/KIfyLtovbs5XQVSIdaQifFQT8=
 github.com/anchore/go-testutils v0.0.0-20200925183923-d5f45b0d3c04 h1:VzprUTpc0vW0nnNKJfJieyH/TZ9UYAnTZs5/gHTdAe8=
 github.com/anchore/go-testutils v0.0.0-20200925183923-d5f45b0d3c04/go.mod h1:6dK64g27Qi1qGQZ67gFmBFvEHScy0/C8qhQhNe5B5pQ=
 github.com/anchore/go-version v1.2.2-0.20200701162849-18adb9c92b9b h1:e1bmaoJfZVsCYMrIZBpFxwV26CbsuoEh5muXD5I1Ods=
 github.com/anchore/go-version v1.2.2-0.20200701162849-18adb9c92b9b/go.mod h1:Bkc+JYWjMCF8OyZ340IMSIi2Ebf3uwByOk6ho4wne1E=
-github.com/anchore/stereoscope v0.0.0-20200925184903-c82da54e98fe h1:m4NSyTo2fVUoUHAV/ZVqE/PFMr/y8oz9HRrhWLk9It0=
-github.com/anchore/stereoscope v0.0.0-20200925184903-c82da54e98fe/go.mod h1:2Jja/4l0zYggW52og+nn0rut4i+OYjCf9vTyrM8RT4E=
+github.com/anchore/stereoscope v0.0.0-20201210022249-091f9bddb42e h1:vHUqHTvH9/oxdDDh1fxS9Ls9gWGytKO7XbbzcQ9MBwI=
+github.com/anchore/stereoscope v0.0.0-20201210022249-091f9bddb42e/go.mod h1:/dHAFjYflH/1tzhdHAcnMCjprMch+YzHJKi59m/1KCM=
 github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883/go.mod h1:rCTlJbsFo29Kk6CurOXKm700vrz8f0KW0JNfpkRJY/8=
+github.com/antihax/optional v1.0.0 h1:xK2lYat7ZLaVVcIuj82J8kIro4V6kDe0AUDFboUCwcg=
+github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY=
 github.com/apex/log v1.1.4/go.mod h1:AlpoD9aScyQfJDVHmLMEcx4oU6LqzkWp4Mg9GdAcEvQ=
 github.com/apex/log v1.3.0 h1:1fyfbPvUwD10nMoh3hY6MXzvZShJQn9/ck7ATgAt5pA=
 github.com/apex/log v1.3.0/go.mod h1:jd8Vpsr46WAe3EZSQ/IUMs2qQD/GOycT5rPWCO1yGcs=
@@ -160,8 +168,8 @@ github.com/bketelsen/crypt v0.0.3-0.20200106085610-5cbc8cc4026c/go.mod h1:MKsuJm
 github.com/blakesmith/ar v0.0.0-20190502131153-809d4375e1fb h1:m935MPodAbYS46DG4pJSv7WO+VECIWUQ7OJYSoTrMh4=
 github.com/blakesmith/ar v0.0.0-20190502131153-809d4375e1fb/go.mod h1:PkYb9DJNAwrSvRx5DYA+gUcOIgTGVMNkfSCbZM8cWpI=
 github.com/blang/semver v3.5.0+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk=
-github.com/bmatcuk/doublestar v1.3.1 h1:rT8rxDPsavp9G+4ZULzqhhUSaI/OPsTZNG88Z3i0xvY=
-github.com/bmatcuk/doublestar v1.3.1/go.mod h1:wiQtGV+rzVYxB7WIlirSN++5HPtPlXEo9MEoZQC/PmE=
+github.com/bmatcuk/doublestar v1.3.3 h1:pVP1d49CcQQaNOl+PI6sPybIrIOD/6sux31PFdmhTH0=
+github.com/bmatcuk/doublestar v1.3.3/go.mod h1:wiQtGV+rzVYxB7WIlirSN++5HPtPlXEo9MEoZQC/PmE=
 github.com/bombsimon/wsl/v2 v2.0.0/go.mod h1:mf25kr/SqFEPhhcxW1+7pxzGlW+hIl/hYTKY95VwV8U=
 github.com/bombsimon/wsl/v2 v2.2.0/go.mod h1:Azh8c3XGEJl9LyX0/sFC+CKMc7Ssgua0g+6abzXN4Pg=
 github.com/bombsimon/wsl/v3 v3.0.0/go.mod h1:st10JtZYLE4D5sC7b8xV4zTKZwAQjCH/Hy2Pm1FNZIc=
@@ -254,9 +262,13 @@ github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.m
 github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98=
 github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
 github.com/evanphx/json-patch v4.2.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk=
+github.com/facebookincubator/nvdtools v0.1.4 h1:x1Ucw9+bSkMd8DJJN4jNQ1Lk4PSFlJarGOxp9D6WUMo=
+github.com/facebookincubator/nvdtools v0.1.4/go.mod h1:0/FIVnSEl9YHXLq3tKBPpKaI0iUceDhdSHPlIwIX44Y=
 github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4=
 github.com/fatih/color v1.9.0 h1:8xPHl4/q1VyqGIPif1F+1V3Y3lSmrq01EabUW3CoW5s=
 github.com/fatih/color v1.9.0/go.mod h1:eQcE1qtQxscV5RaZvpXrrb8Drkc3/DdQ+uUYCNjL+zU=
+github.com/fatih/set v0.2.1 h1:nn2CaJyknWE/6txyUDGwysr3G5QC6xWB/PtVjPBbeaA=
+github.com/fatih/set v0.2.1/go.mod h1:+RKtMCH+favT2+3YecHGxcc0b4KyVWA1QWWJUs4E0CI=
 github.com/fortytw2/leaktest v1.2.0/go.mod h1:jDsjWgpAGjm2CA7WthBh/CdZYEPF31XHquHwclZch5g=
 github.com/fortytw2/leaktest v1.3.0/go.mod h1:jDsjWgpAGjm2CA7WthBh/CdZYEPF31XHquHwclZch5g=
 github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
@@ -295,8 +307,8 @@ github.com/go-sql-driver/mysql v1.4.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG
 github.com/go-sql-driver/mysql v1.4.1/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w=
 github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg=
 github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
-github.com/go-test/deep v1.0.6 h1:UHSEyLZUwX9Qoi99vVwvewiMC8mM2bf7XEM2nqvzEn8=
-github.com/go-test/deep v1.0.6/go.mod h1:QV8Hv/iy04NyLBxAdO9njL0iVPN1S4d/A3NVv1V36o8=
+github.com/go-test/deep v1.0.7 h1:/VSMRlnY/JSyqxQUzQLKVMAskpY/NZKFA5j2P+0pP2M=
+github.com/go-test/deep v1.0.7/go.mod h1:QV8Hv/iy04NyLBxAdO9njL0iVPN1S4d/A3NVv1V36o8=
 github.com/go-toolsmith/astcast v1.0.0 h1:JojxlmI6STnFVG9yOImLeGREv8W2ocNUM+iOhR6jE7g=
 github.com/go-toolsmith/astcast v1.0.0/go.mod h1:mt2OdQTeAQcY4DQgPSArJjHCcOwlX+Wl/kwN+LbLGQ4=
 github.com/go-toolsmith/astcopy v1.0.0 h1:OMgl1b1MEpjFQ1m5ztEO06rz5CUd3oBv9RF7+DyvdG8=
@@ -499,6 +511,8 @@ github.com/hashicorp/memberlist v0.1.3/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2p
 github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/JwenrHc=
 github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI=
 github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
+github.com/iancoleman/orderedmap v0.0.0-20190318233801-ac98e3ecb4b0 h1:i462o439ZjprVSFSZLZxcsoAe592sZB1rci2Z8j4wdk=
+github.com/iancoleman/orderedmap v0.0.0-20190318233801-ac98e3ecb4b0/go.mod h1:N0Wam8K1arqPXNWjMo21EXnBPOPp36vB07FNRdD2geA=
 github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
 github.com/imdario/mergo v0.3.5/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA=
 github.com/imdario/mergo v0.3.8/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA=
@@ -680,6 +694,8 @@ github.com/pkg/errors v0.8.1-0.20171018195549-f15c970de5b7/go.mod h1:bwawxfHBFNV
 github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
 github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pkg/profile v1.5.0 h1:042Buzk+NhDI+DeSAA62RwJL8VAuZUMQZUjCsRz1Mug=
+github.com/pkg/profile v1.5.0/go.mod h1:qBsxPvzyUincmltOk6iyRVxHYg4adc0OFOv72ZdLa18=
 github.com/pmezard/go-difflib v0.0.0-20151028094244-d8ed2627bdf0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
@@ -718,6 +734,8 @@ github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb
 github.com/sassoftware/go-rpmutils v0.0.0-20190420191620-a8f1baeba37b/go.mod h1:am+Fp8Bt506lA3Rk3QCmSqmYmLMnPDhdDUcosQCAx+I=
 github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0=
 github.com/sclevine/spec v1.2.0/go.mod h1:W4J29eT/Kzv7/b9IWLB055Z+qvVC9vt0Arko24q7p+U=
+github.com/scylladb/go-set v1.0.2 h1:SkvlMCKhP0wyyct6j+0IHJkBkSZL+TDzZ4E7f7BCcRE=
+github.com/scylladb/go-set v1.0.2/go.mod h1:DkpGd78rljTxKAnTDPFqXSGxvETQnJyuSOQwsHycqfs=
 github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc=
 github.com/securego/gosec v0.0.0-20200103095621-79fbf3af8d83/go.mod h1:vvbZ2Ae7AzSq3/kywjUDxSNq2SJ27RxCz2un0H3ePqE=
 github.com/securego/gosec v0.0.0-20200401082031-e946c8c39989 h1:rq2/kILQnPtq5oL4+IAjgVOjh5e2yj2aaCYi7squEvI=
@@ -785,6 +803,7 @@ github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoH
 github.com/stretchr/testify v0.0.0-20151208002404-e3a8ff8ce365/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
 github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
+github.com/stretchr/testify v1.3.1-0.20190311161405-34c6fa2dc709/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
 github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
 github.com/stretchr/testify v1.5.1 h1:nOGnQDM7FYENwehXlg/kFVnos3rEvtKTjRvOWSzb6H4=
 github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
@@ -801,6 +820,7 @@ github.com/tetafro/godot v0.4.2/go.mod h1:/7NLHhv08H1+8DNj0MElpAACw1ajsCuf3TKNQx
 github.com/timakin/bodyclose v0.0.0-20190930140734-f7f2e9bca95e/go.mod h1:Qimiffbc6q9tBWlVV6x0P9sat/ao1xEkREYPPj9hphk=
 github.com/timakin/bodyclose v0.0.0-20200424151742-cb6215831a94 h1:ig99OeTyDwQWhPe2iw9lwfQVF1KB3Q4fpP3X7/2VBG8=
 github.com/timakin/bodyclose v0.0.0-20200424151742-cb6215831a94/go.mod h1:Qimiffbc6q9tBWlVV6x0P9sat/ao1xEkREYPPj9hphk=
+github.com/tj/assert v0.0.0-20171129193455-018094318fb0 h1:Rw8kxzWo1mr6FSaYXjQELRe88y2KdfynXdnK72rdjtA=
 github.com/tj/assert v0.0.0-20171129193455-018094318fb0/go.mod h1:mZ9/Rh9oLWpLLDRpvE+3b7gP/C2YyLFYxNmcLnPTMe0=
 github.com/tj/go-elastic v0.0.0-20171221160941-36157cbbebc2/go.mod h1:WjeM0Oo1eNAjXGDx2yma7uG2XoyRZTq1uv3M/o7imD0=
 github.com/tj/go-kinesis v0.0.0-20171128231115-08b17f58cb1b/go.mod h1:/yhzCV0xPfx6jb1bBgRFjl5lytqVqZXEaeqWP8lTEao=

gon.hcl

@@ -0,0 +1,15 @@
+source = ["./dist/syft-macos_darwin_amd64/syft"] # The 'dist' directory path should ideally reference an env var, where the source of truth is the Makefile. I wasn't able to figure out how to solve this.
+bundle_id = "com.anchore.toolbox.syft"
+
+sign {
+  application_identity = "Developer ID Application: ANCHORE, INC. (9MJHKYX5AT)"
+}
+
+dmg {
+  output_path = "./dist/output.dmg"
+  volume_name = "Syft"
+}
+
+zip {
+  output_path = "./dist/output.zip"
+}

install.sh

@@ -1,6 +1,6 @@
 #!/bin/sh
 set -e
-# Code generated by godownloader on 2020-08-10T20:55:46Z. DO NOT EDIT.
+# Code generated by godownloader on 2020-08-10T20:55:46Z.
 #
 
 usage() {
@@ -45,11 +45,16 @@ parse_args() {
 execute() {
   tmpdir=$(mktemp -d)
   log_debug "downloading files into ${tmpdir}"
-  http_download "${tmpdir}/${TARBALL}" "${TARBALL_URL}"
+  http_download "${tmpdir}/${ARCHIVE}" "${ARCHIVE_URL}"
   http_download "${tmpdir}/${CHECKSUM}" "${CHECKSUM_URL}"
-  hash_sha256_verify "${tmpdir}/${TARBALL}" "${tmpdir}/${CHECKSUM}"
+
+  # macOS has its own secure verification mechanism, and checksums.txt is not used.
+  if [ "$OS" != "darwin" ]; then
+    hash_sha256_verify "${tmpdir}/${ARCHIVE}" "${tmpdir}/${CHECKSUM}"
+  fi
+
   srcdir="${tmpdir}"
-  (cd "${tmpdir}" && untar "${TARBALL}")
+  (cd "${tmpdir}" && unpack "${ARCHIVE}")
   test ! -d "${BINDIR}" && install -d "${BINDIR}"
   for binexe in $BINARIES; do
     if [ "$OS" = "windows" ]; then
@@ -89,6 +94,7 @@ tag_to_version() {
 adjust_format() {
   # change format (tar.gz or zip) based on OS
   case ${OS} in
+    darwin) FORMAT=dmg ;;
     windows) FORMAT=zip ;;
   esac
   true
@@ -221,18 +227,26 @@ uname_arch_check() {
   log_crit "uname_arch_check '$(uname -m)' got converted to '$arch' which is not a GOARCH value.  Please file bug report at https://github.com/client9/shlib"
   return 1
 }
-untar() {
-  tarball=$1
-  case "${tarball}" in
-    *.tar.gz | *.tgz) tar --no-same-owner -xzf "${tarball}" ;;
-    *.tar) tar --no-same-owner -xf "${tarball}" ;;
-    *.zip) unzip "${tarball}" ;;
+unpack() {
+  archive=$1
+  case "${archive}" in
+    *.tar.gz | *.tgz) tar --no-same-owner -xzf "${archive}" ;;
+    *.tar) tar --no-same-owner -xf "${archive}" ;;
+    *.zip) unzip "${archive}" ;;
+    *.dmg) extract_from_dmg "${archive}" ;;
     *)
-      log_err "untar unknown archive format for ${tarball}"
+      log_err "unpack unknown archive format for ${archive}"
       return 1
       ;;
   esac
 }
+extract_from_dmg() {
+  dmg_file=$1
+  mount_point="/Volumes/tmp-dmg"
+  hdiutil attach -quiet -nobrowse -mountpoint "${mount_point}" "${dmg_file}"
+  cp -fR "${mount_point}/" ./
+  hdiutil detach -quiet -force "${mount_point}"
+}
 http_download_curl() {
   local_file=$1
   source_url=$2
@@ -366,8 +380,8 @@ adjust_arch
 log_info "found version: ${VERSION} for ${TAG}/${OS}/${ARCH}"
 
 NAME=${PROJECT_NAME}_${VERSION}_${OS}_${ARCH}
-TARBALL=${NAME}.${FORMAT}
-TARBALL_URL=${GITHUB_DOWNLOAD}/${TAG}/${TARBALL}
+ARCHIVE=${NAME}.${FORMAT}
+ARCHIVE_URL=${GITHUB_DOWNLOAD}/${TAG}/${ARCHIVE}
 CHECKSUM=${PROJECT_NAME}_${VERSION}_checksums.txt
 CHECKSUM_URL=${GITHUB_DOWNLOAD}/${TAG}/${CHECKSUM}
 

internal/anchore/client.go

@@ -0,0 +1,58 @@
+package anchore
+
+import (
+	"context"
+	"fmt"
+
+	"github.com/anchore/client-go/pkg/external"
+	"github.com/anchore/syft/internal"
+	"github.com/anchore/syft/internal/version"
+)
+
+type Configuration struct {
+	Hostname  string
+	Username  string
+	Password  string
+	UserAgent string
+	Scheme    string
+}
+
+type Client struct {
+	config Configuration
+	client *external.APIClient
+}
+
+func NewClient(cfg Configuration) (*Client, error) {
+	if cfg.UserAgent == "" {
+		versionInfo := version.FromBuild()
+		// format: product / product-version comment
+		cfg.UserAgent = fmt.Sprintf("%s / %s %s", internal.ApplicationName, versionInfo.Version, versionInfo.Platform)
+	}
+
+	if cfg.Scheme == "" {
+		cfg.Scheme = "https"
+	}
+
+	return &Client{
+		config: cfg,
+		client: external.NewAPIClient(&external.Configuration{
+			Host:      cfg.Hostname,
+			UserAgent: cfg.UserAgent,
+			Scheme:    cfg.Scheme,
+		}),
+	}, nil
+}
+
+func (c *Client) newRequestContext(parentContext context.Context) context.Context {
+	if parentContext == nil {
+		parentContext = context.Background()
+	}
+	return context.WithValue(
+		parentContext,
+		external.ContextBasicAuth,
+		external.BasicAuth{
+			UserName: c.config.Username,
+			Password: c.config.Password,
+		},
+	)
+}

internal/anchore/import.go

@@ -0,0 +1,143 @@
+package anchore
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"time"
+
+	"github.com/antihax/optional"
+
+	"github.com/anchore/client-go/pkg/external"
+	"github.com/anchore/stereoscope/pkg/image"
+	"github.com/anchore/syft/internal/bus"
+	"github.com/anchore/syft/syft/distro"
+	"github.com/anchore/syft/syft/event"
+	"github.com/anchore/syft/syft/pkg"
+	"github.com/anchore/syft/syft/source"
+	"github.com/wagoodman/go-partybus"
+	"github.com/wagoodman/go-progress"
+)
+
+type ImportConfig struct {
+	ImageMetadata           image.Metadata
+	SourceMetadata          source.Metadata
+	Catalog                 *pkg.Catalog
+	Distro                  *distro.Distro
+	Dockerfile              []byte
+	OverwriteExistingUpload bool
+}
+
+func importProgress(source string) (*progress.Stage, *progress.Manual) {
+	stage := &progress.Stage{}
+	prog := &progress.Manual{
+		// this is the number of stages to expect; start + individual endpoints + stop
+		Total: 6,
+	}
+	bus.Publish(partybus.Event{
+		Type:   event.ImportStarted,
+		Source: source,
+		Value: progress.StagedProgressable(&struct {
+			progress.Stager
+			progress.Progressable
+		}{
+			Stager:       progress.Stager(stage),
+			Progressable: prog,
+		}),
+	})
+
+	return stage, prog
+}
+
+// nolint:funlen
+func (c *Client) Import(ctx context.Context, cfg ImportConfig) error {
+	stage, prog := importProgress(c.config.Hostname)
+
+	ctxWithTimeout, cancel := context.WithTimeout(ctx, time.Second*30)
+	defer cancel()
+
+	authedCtx := c.newRequestContext(ctxWithTimeout)
+
+	stage.Current = "starting session"
+	startOperation, _, err := c.client.ImportsApi.CreateOperation(authedCtx)
+	if err != nil {
+		var detail = "no details given"
+		var openAPIErr external.GenericOpenAPIError
+		if errors.As(err, &openAPIErr) {
+			detail = string(openAPIErr.Body())
+		}
+		return fmt.Errorf("unable to start import session: %w: %s", err, detail)
+	}
+	prog.N++
+	sessionID := startOperation.Uuid
+
+	packageDigest, err := importPackageSBOM(authedCtx, c.client.ImportsApi, sessionID, cfg.SourceMetadata, cfg.Catalog, cfg.Distro, stage)
+	if err != nil {
+		return fmt.Errorf("failed to import Package SBOM: %w", err)
+	}
+	prog.N++
+
+	manifestDigest, err := importManifest(authedCtx, c.client.ImportsApi, sessionID, cfg.ImageMetadata.RawManifest, stage)
+	if err != nil {
+		return fmt.Errorf("failed to import Manifest: %w", err)
+	}
+	prog.N++
+
+	configDigest, err := importConfig(authedCtx, c.client.ImportsApi, sessionID, cfg.ImageMetadata.RawConfig, stage)
+	if err != nil {
+		return fmt.Errorf("failed to import Config: %w", err)
+	}
+	prog.N++
+
+	dockerfileDigest, err := importDockerfile(authedCtx, c.client.ImportsApi, sessionID, cfg.Dockerfile, stage)
+	if err != nil {
+		return fmt.Errorf("failed to import Dockerfile: %w", err)
+	}
+	prog.N++
+
+	stage.Current = "finalizing"
+	imageModel := addImageModel(cfg.ImageMetadata, packageDigest, manifestDigest, dockerfileDigest, configDigest, sessionID)
+	opts := external.AddImageOpts{
+		Force: optional.NewBool(cfg.OverwriteExistingUpload),
+	}
+
+	_, _, err = c.client.ImagesApi.AddImage(authedCtx, imageModel, &opts)
+	if err != nil {
+		var detail = "no details given"
+		var openAPIErr external.GenericOpenAPIError
+		if errors.As(err, &openAPIErr) {
+			detail = string(openAPIErr.Body())
+		}
+		return fmt.Errorf("unable to complete import session=%q: %w: %s", sessionID, err, detail)
+	}
+	prog.N++
+
+	stage.Current = ""
+	prog.SetCompleted()
+
+	return nil
+}
+
+func addImageModel(imageMetadata image.Metadata, packageDigest, manifestDigest, dockerfileDigest, configDigest, sessionID string) external.ImageAnalysisRequest {
+	var tags = make([]string, len(imageMetadata.Tags))
+	for i, t := range imageMetadata.Tags {
+		tags[i] = t.String()
+	}
+
+	return external.ImageAnalysisRequest{
+		Source: external.ImageSource{
+			Import: &external.ImageImportManifest{
+				Contents: external.ImportContentDigests{
+					Packages:    packageDigest,
+					Manifest:    manifestDigest,
+					Dockerfile:  dockerfileDigest,
+					ImageConfig: configDigest,
+				},
+				Tags:          tags,
+				Digest:        imageMetadata.ManifestDigest,
+				LocalImageId:  imageMetadata.ID,
+				OperationUuid: sessionID,
+			},
+		},
+	}
+}

internal/anchore/import_config.go

@@ -0,0 +1,50 @@
+// nolint:dupl
+package anchore
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"net/http"
+
+	"github.com/wagoodman/go-progress"
+
+	"github.com/anchore/client-go/pkg/external"
+	"github.com/anchore/syft/internal/log"
+)
+
+type configImportAPI interface {
+	ImportImageConfig(ctx context.Context, sessionID string, contents interface{}) (external.ImageImportContentResponse, *http.Response, error)
+}
+
+func importConfig(ctx context.Context, api configImportAPI, sessionID string, config []byte, stage *progress.Stage) (string, error) {
+	if len(config) > 0 {
+		log.Debug("importing image config")
+		stage.Current = "image config"
+
+		// API requires an object, but we do not verify the shape of this object locally
+		var sender map[string]interface{}
+		if err := json.Unmarshal(config, &sender); err != nil {
+			return "", err
+		}
+
+		response, httpResponse, err := api.ImportImageConfig(ctx, sessionID, sender)
+		if err != nil {
+			var openAPIErr external.GenericOpenAPIError
+			if errors.As(err, &openAPIErr) {
+				log.Errorf("api response: %+v", string(openAPIErr.Body()))
+			}
+			return "", fmt.Errorf("unable to import Config: %w", err)
+		}
+
+		defer httpResponse.Body.Close()
+
+		if httpResponse.StatusCode != 200 {
+			return "", fmt.Errorf("unable to import Config: %s", httpResponse.Status)
+		}
+
+		return response.Digest, nil
+	}
+	return "", nil
+}

internal/anchore/import_config_test.go

@@ -0,0 +1,129 @@
+package anchore
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"net/http"
+	"strings"
+	"testing"
+
+	"github.com/wagoodman/go-progress"
+
+	"github.com/anchore/client-go/pkg/external"
+	"github.com/docker/docker/pkg/ioutils"
+	"github.com/go-test/deep"
+)
+
+type mockConfigImportAPI struct {
+	sessionID      string
+	model          interface{}
+	httpResponse   *http.Response
+	err            error
+	ctx            context.Context
+	responseDigest string
+	wasCalled      bool
+}
+
+func (m *mockConfigImportAPI) ImportImageConfig(ctx context.Context, sessionID string, contents interface{}) (external.ImageImportContentResponse, *http.Response, error) {
+	m.wasCalled = true
+	m.model = contents
+	m.sessionID = sessionID
+	m.ctx = ctx
+	if m.httpResponse == nil {
+		m.httpResponse = &http.Response{}
+	}
+	m.httpResponse.Body = ioutils.NewReadCloserWrapper(strings.NewReader(""), func() error { return nil })
+	return external.ImageImportContentResponse{Digest: m.responseDigest}, m.httpResponse, m.err
+}
+
+func TestConfigImport(t *testing.T) {
+
+	sessionID := "my-session"
+
+	tests := []struct {
+		name            string
+		manifestJSONStr string
+		api             *mockConfigImportAPI
+		expectsError    bool
+		expectsCall     bool
+	}{
+
+		{
+			name:            "Go case: import works",
+			manifestJSONStr: `{ "key": "the-manifest-contents!" }`,
+			api: &mockConfigImportAPI{
+				httpResponse:   &http.Response{StatusCode: 200},
+				responseDigest: "digest!",
+			},
+			expectsCall: true,
+		},
+		{
+			name:            "No manifest provided",
+			manifestJSONStr: "",
+			api:             &mockConfigImportAPI{},
+			expectsCall:     false,
+		},
+		{
+			name:            "API returns an error",
+			manifestJSONStr: `{ "key": "the-manifest-contents!" }`,
+			api: &mockConfigImportAPI{
+				err: fmt.Errorf("api error, something went wrong"),
+			},
+			expectsError: true,
+			expectsCall:  true,
+		},
+		{
+			name:            "API HTTP-level error",
+			manifestJSONStr: `{ "key": "the-manifest-contents!" }`,
+			api: &mockConfigImportAPI{
+				httpResponse: &http.Response{StatusCode: 404},
+			},
+			expectsError: true,
+			expectsCall:  true,
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+
+			digest, err := importConfig(context.TODO(), test.api, sessionID, []byte(test.manifestJSONStr), &progress.Stage{})
+
+			// validate error handling
+			if err != nil && !test.expectsError {
+				t.Fatalf("did not expect an error, but got: %+v", err)
+			} else if err == nil && test.expectsError {
+				t.Fatalf("did expect an error, but got none")
+			}
+
+			if !test.api.wasCalled && test.expectsCall {
+				t.Fatalf("was not called!")
+			} else if test.api.wasCalled && !test.expectsCall {
+				t.Fatalf("should not have been called")
+			}
+
+			if !test.expectsCall {
+				return
+			}
+
+			if digest != test.api.responseDigest {
+				t.Errorf("unexpected content digest: %q != %q", digest, test.api.responseDigest)
+			}
+
+			// validating that the mock got the right parameters
+			if test.api.sessionID != sessionID {
+				t.Errorf("different session ID: %s != %s", test.api.sessionID, sessionID)
+			}
+
+			var expected map[string]interface{}
+			if err := json.Unmarshal([]byte(test.manifestJSONStr), &expected); err != nil {
+				t.Fatalf("could not unmarshal expected results")
+			}
+
+			for _, d := range deep.Equal(test.api.model, expected) {
+				t.Errorf("model difference: %s", d)
+			}
+
+		})
+	}
+}

internal/anchore/import_dockerfile.go

@@ -0,0 +1,43 @@
+package anchore
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"net/http"
+
+	"github.com/wagoodman/go-progress"
+
+	"github.com/anchore/syft/internal/log"
+
+	"github.com/anchore/client-go/pkg/external"
+)
+
+type dockerfileImportAPI interface {
+	ImportImageDockerfile(ctx context.Context, sessionID string, contents string) (external.ImageImportContentResponse, *http.Response, error)
+}
+
+func importDockerfile(ctx context.Context, api dockerfileImportAPI, sessionID string, dockerfile []byte, stage *progress.Stage) (string, error) {
+	if len(dockerfile) > 0 {
+		log.Debug("importing dockerfile")
+		stage.Current = "dockerfile"
+
+		response, httpResponse, err := api.ImportImageDockerfile(ctx, sessionID, string(dockerfile))
+		if err != nil {
+			var openAPIErr external.GenericOpenAPIError
+			if errors.As(err, &openAPIErr) {
+				log.Errorf("api response: %+v", string(openAPIErr.Body()))
+			}
+			return "", fmt.Errorf("unable to import Dockerfile: %w", err)
+		}
+
+		defer httpResponse.Body.Close()
+
+		if httpResponse.StatusCode != 200 {
+			return "", fmt.Errorf("unable to import Dockerfile: %s", httpResponse.Status)
+		}
+
+		return response.Digest, nil
+	}
+	return "", nil
+}

internal/anchore/import_dockerfile_test.go

@@ -0,0 +1,124 @@
+package anchore
+
+import (
+	"context"
+	"fmt"
+	"net/http"
+	"strings"
+	"testing"
+
+	"github.com/wagoodman/go-progress"
+
+	"github.com/docker/docker/pkg/ioutils"
+
+	"github.com/anchore/client-go/pkg/external"
+	"github.com/go-test/deep"
+)
+
+type mockDockerfileImportAPI struct {
+	sessionID      string
+	model          string
+	httpResponse   *http.Response
+	err            error
+	ctx            context.Context
+	responseDigest string
+	wasCalled      bool
+}
+
+func (m *mockDockerfileImportAPI) ImportImageDockerfile(ctx context.Context, sessionID string, contents string) (external.ImageImportContentResponse, *http.Response, error) {
+	m.wasCalled = true
+	m.model = contents
+	m.sessionID = sessionID
+	m.ctx = ctx
+	if m.httpResponse == nil {
+		m.httpResponse = &http.Response{}
+	}
+	m.httpResponse.Body = ioutils.NewReadCloserWrapper(strings.NewReader(""), func() error { return nil })
+	return external.ImageImportContentResponse{Digest: m.responseDigest}, m.httpResponse, m.err
+}
+
+func TestDockerfileImport(t *testing.T) {
+
+	sessionID := "my-session"
+
+	tests := []struct {
+		name         string
+		dockerfile   string
+		api          *mockDockerfileImportAPI
+		expectsError bool
+		expectsCall  bool
+	}{
+
+		{
+			name:       "Go case: import works",
+			dockerfile: "the-manifest-contents!",
+			api: &mockDockerfileImportAPI{
+				httpResponse:   &http.Response{StatusCode: 200},
+				responseDigest: "digest!",
+			},
+			expectsCall: true,
+		},
+		{
+			name:        "No manifest provided",
+			dockerfile:  "",
+			api:         &mockDockerfileImportAPI{},
+			expectsCall: false,
+		},
+		{
+			name:       "API returns an error",
+			dockerfile: "the-manifest-contents!",
+			api: &mockDockerfileImportAPI{
+				err: fmt.Errorf("api error, something went wrong"),
+			},
+			expectsError: true,
+			expectsCall:  true,
+		},
+		{
+			name:       "API HTTP-level error",
+			dockerfile: "the-manifest-contents!",
+			api: &mockDockerfileImportAPI{
+				httpResponse: &http.Response{StatusCode: 404},
+			},
+			expectsError: true,
+			expectsCall:  true,
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+
+			digest, err := importDockerfile(context.TODO(), test.api, sessionID, []byte(test.dockerfile), &progress.Stage{})
+
+			// validate error handling
+			if err != nil && !test.expectsError {
+				t.Fatalf("did not expect an error, but got: %+v", err)
+			} else if err == nil && test.expectsError {
+				t.Fatalf("did expect an error, but got none")
+			}
+
+			if !test.api.wasCalled && test.expectsCall {
+				t.Fatalf("was not called!")
+			} else if test.api.wasCalled && !test.expectsCall {
+				t.Fatalf("should not have been called")
+			}
+
+			if !test.expectsCall {
+				return
+			}
+
+			if digest != test.api.responseDigest {
+				t.Errorf("unexpected content digest: %q != %q", digest, test.api.responseDigest)
+			}
+
+			// validating that the mock got the right parameters
+			if test.api.sessionID != sessionID {
+				t.Errorf("different session ID: %s != %s", test.api.sessionID, sessionID)
+			}
+
+			for _, d := range deep.Equal(test.api.model, test.dockerfile) {
+				t.Errorf("model difference: %s", d)
+			}
+
+		})
+	}
+}

internal/anchore/import_manifest.go

@@ -0,0 +1,50 @@
+// nolint: dupl
+package anchore
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"net/http"
+
+	"github.com/wagoodman/go-progress"
+
+	"github.com/anchore/client-go/pkg/external"
+	"github.com/anchore/syft/internal/log"
+)
+
+type manifestImportAPI interface {
+	ImportImageManifest(ctx context.Context, sessionID string, contents interface{}) (external.ImageImportContentResponse, *http.Response, error)
+}
+
+func importManifest(ctx context.Context, api manifestImportAPI, sessionID string, manifest []byte, stage *progress.Stage) (string, error) {
+	if len(manifest) > 0 {
+		log.Debug("importing image manifest")
+		stage.Current = "image manifest"
+
+		// API requires an object, but we do not verify the shape of this object locally
+		var sender map[string]interface{}
+		if err := json.Unmarshal(manifest, &sender); err != nil {
+			return "", err
+		}
+
+		response, httpResponse, err := api.ImportImageManifest(ctx, sessionID, sender)
+		if err != nil {
+			var openAPIErr external.GenericOpenAPIError
+			if errors.As(err, &openAPIErr) {
+				log.Errorf("api response: %+v", string(openAPIErr.Body()))
+			}
+			return "", fmt.Errorf("unable to import Manifest: %w", err)
+		}
+
+		defer httpResponse.Body.Close()
+
+		if httpResponse.StatusCode != 200 {
+			return "", fmt.Errorf("unable to import Manifest: %s", httpResponse.Status)
+		}
+
+		return response.Digest, nil
+	}
+	return "", nil
+}

internal/anchore/import_manifest_test.go

@@ -0,0 +1,129 @@
+package anchore
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"net/http"
+	"strings"
+	"testing"
+
+	"github.com/wagoodman/go-progress"
+
+	"github.com/anchore/client-go/pkg/external"
+	"github.com/docker/docker/pkg/ioutils"
+	"github.com/go-test/deep"
+)
+
+type mockManifestImportAPI struct {
+	sessionID      string
+	model          interface{}
+	httpResponse   *http.Response
+	err            error
+	ctx            context.Context
+	responseDigest string
+	wasCalled      bool
+}
+
+func (m *mockManifestImportAPI) ImportImageManifest(ctx context.Context, sessionID string, contents interface{}) (external.ImageImportContentResponse, *http.Response, error) {
+	m.wasCalled = true
+	m.model = contents
+	m.sessionID = sessionID
+	m.ctx = ctx
+	if m.httpResponse == nil {
+		m.httpResponse = &http.Response{}
+	}
+	m.httpResponse.Body = ioutils.NewReadCloserWrapper(strings.NewReader(""), func() error { return nil })
+	return external.ImageImportContentResponse{Digest: m.responseDigest}, m.httpResponse, m.err
+}
+
+func TestManifestImport(t *testing.T) {
+
+	sessionID := "my-session"
+
+	tests := []struct {
+		name         string
+		manifest     string
+		api          *mockManifestImportAPI
+		expectsError bool
+		expectsCall  bool
+	}{
+
+		{
+			name:     "Go case: import works",
+			manifest: `{ "key": "the-config-contents!" }`,
+			api: &mockManifestImportAPI{
+				httpResponse:   &http.Response{StatusCode: 200},
+				responseDigest: "digest!",
+			},
+			expectsCall: true,
+		},
+		{
+			name:        "No manifest provided",
+			manifest:    "",
+			api:         &mockManifestImportAPI{},
+			expectsCall: false,
+		},
+		{
+			name:     "API returns an error",
+			manifest: `{ "key": "the-config-contents!" }`,
+			api: &mockManifestImportAPI{
+				err: fmt.Errorf("api error, something went wrong"),
+			},
+			expectsError: true,
+			expectsCall:  true,
+		},
+		{
+			name:     "API HTTP-level error",
+			manifest: `{ "key": "the-config-contents!" }`,
+			api: &mockManifestImportAPI{
+				httpResponse: &http.Response{StatusCode: 404},
+			},
+			expectsError: true,
+			expectsCall:  true,
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+
+			digest, err := importManifest(context.TODO(), test.api, sessionID, []byte(test.manifest), &progress.Stage{})
+
+			// validate error handling
+			if err != nil && !test.expectsError {
+				t.Fatalf("did not expect an error, but got: %+v", err)
+			} else if err == nil && test.expectsError {
+				t.Fatalf("did expect an error, but got none")
+			}
+
+			if !test.api.wasCalled && test.expectsCall {
+				t.Fatalf("was not called!")
+			} else if test.api.wasCalled && !test.expectsCall {
+				t.Fatalf("should not have been called")
+			}
+
+			if !test.expectsCall {
+				return
+			}
+
+			if digest != test.api.responseDigest {
+				t.Errorf("unexpected content digest: %q != %q", digest, test.api.responseDigest)
+			}
+
+			// validating that the mock got the right parameters
+			if test.api.sessionID != sessionID {
+				t.Errorf("different session ID: %s != %s", test.api.sessionID, sessionID)
+			}
+
+			var expected map[string]interface{}
+			if err := json.Unmarshal([]byte(test.manifest), &expected); err != nil {
+				t.Fatalf("could not unmarshal expected results")
+			}
+
+			for _, d := range deep.Equal(test.api.model, expected) {
+				t.Errorf("model difference: %s", d)
+			}
+
+		})
+	}
+}

internal/anchore/import_package_sbom.go

@@ -0,0 +1,69 @@
+package anchore
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"net/http"
+
+	"github.com/wagoodman/go-progress"
+
+	jsonPresenter "github.com/anchore/syft/syft/presenter/json"
+
+	"github.com/anchore/syft/syft/distro"
+	"github.com/anchore/syft/syft/source"
+
+	"github.com/anchore/client-go/pkg/external"
+	"github.com/anchore/syft/internal/log"
+	"github.com/anchore/syft/syft/pkg"
+)
+
+type packageSBOMImportAPI interface {
+	ImportImagePackages(context.Context, string, external.ImagePackageManifest) (external.ImageImportContentResponse, *http.Response, error)
+}
+
+func packageSbomModel(s source.Metadata, catalog *pkg.Catalog, d *distro.Distro) (*external.ImagePackageManifest, error) {
+	var buf bytes.Buffer
+	pres := jsonPresenter.NewPresenter(catalog, s, d)
+	err := pres.Present(&buf)
+	if err != nil {
+		return nil, fmt.Errorf("unable to serialize results: %w", err)
+	}
+
+	// the model is 1:1 the JSON output of today. As the schema changes, this will need to be converted into individual mappings.
+	var model external.ImagePackageManifest
+	if err = json.Unmarshal(buf.Bytes(), &model); err != nil {
+		return nil, fmt.Errorf("unable to convert JSON presenter output to import model: %w", err)
+	}
+
+	return &model, nil
+}
+
+func importPackageSBOM(ctx context.Context, api packageSBOMImportAPI, sessionID string, s source.Metadata, catalog *pkg.Catalog, d *distro.Distro, stage *progress.Stage) (string, error) {
+	log.Debug("importing package SBOM")
+	stage.Current = "package SBOM"
+
+	model, err := packageSbomModel(s, catalog, d)
+	if err != nil {
+		return "", fmt.Errorf("unable to create PackageSBOM model: %w", err)
+	}
+
+	response, httpResponse, err := api.ImportImagePackages(ctx, sessionID, *model)
+	if err != nil {
+		var openAPIErr external.GenericOpenAPIError
+		if errors.As(err, &openAPIErr) {
+			log.Errorf("api response: %+v", string(openAPIErr.Body()))
+		}
+		return "", fmt.Errorf("unable to import PackageSBOM: %w", err)
+	}
+
+	defer httpResponse.Body.Close()
+
+	if httpResponse.StatusCode != 200 {
+		return "", fmt.Errorf("unable to import PackageSBOM: %s", httpResponse.Status)
+	}
+
+	return response.Digest, nil
+}

internal/anchore/import_package_sbom_test.go

@@ -0,0 +1,254 @@
+package anchore
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"fmt"
+	"net/http"
+	"strings"
+	"testing"
+
+	"github.com/wagoodman/go-progress"
+
+	jsonPresenter "github.com/anchore/syft/syft/presenter/json"
+
+	"github.com/anchore/syft/syft/distro"
+
+	"github.com/docker/docker/pkg/ioutils"
+
+	"github.com/anchore/client-go/pkg/external"
+	"github.com/anchore/syft/syft/pkg"
+	"github.com/anchore/syft/syft/source"
+	"github.com/go-test/deep"
+)
+
+func must(c pkg.CPE, e error) pkg.CPE {
+	if e != nil {
+		panic(e)
+	}
+	return c
+}
+
+// this test is tailored towards the assumption that the import doc shape and the syft json shape are the same.
+// TODO: replace this as the document shapes diverge.
+func TestPackageSbomToModel(t *testing.T) {
+
+	m := source.Metadata{
+		Scheme: source.ImageScheme,
+		ImageMetadata: source.ImageMetadata{
+			UserInput: "user-in",
+			Scope:     "scope!",
+			Layers: []source.LayerMetadata{
+				{
+					MediaType: "layer-metadata-type!",
+					Digest:    "layer-digest",
+					Size:      20,
+				},
+			},
+			Size:           10,
+			ManifestDigest: "sha256:digest!",
+			MediaType:      "mediatype!",
+			Tags:           nil,
+		},
+	}
+
+	d, _ := distro.NewDistro(distro.CentOS, "8.0", "")
+
+	p := pkg.Package{
+		Name:    "name",
+		Version: "version",
+		FoundBy: "foundBy",
+		Locations: []source.Location{
+			{
+				Path:         "path",
+				FileSystemID: "layerID",
+			},
+		},
+		Licenses: []string{"license"},
+		Language: pkg.Python,
+		Type:     pkg.PythonPkg,
+		CPEs: []pkg.CPE{
+			must(pkg.NewCPE("cpe:2.3:*:some:package:1:*:*:*:*:*:*:*")),
+		},
+		PURL: "purl",
+	}
+
+	c := pkg.NewCatalog(p)
+
+	model, err := packageSbomModel(m, c, &d)
+	if err != nil {
+		t.Fatalf("unable to generate model from source material: %+v", err)
+	}
+
+	var modelJSON []byte
+
+	modelJSON, err = json.Marshal(&model)
+	if err != nil {
+		t.Fatalf("unable to marshal model: %+v", err)
+	}
+
+	var buf bytes.Buffer
+	pres := jsonPresenter.NewPresenter(c, m, &d)
+	if err := pres.Present(&buf); err != nil {
+		t.Fatalf("unable to get expected json: %+v", err)
+	}
+
+	// unmarshal expected result
+	var expectedDoc jsonPresenter.Document
+	if err := json.Unmarshal(buf.Bytes(), &expectedDoc); err != nil {
+		t.Fatalf("unable to parse json doc: %+v", err)
+	}
+
+	// unmarshal actual result
+	var actualDoc jsonPresenter.Document
+	if err := json.Unmarshal(modelJSON, &actualDoc); err != nil {
+		t.Fatalf("unable to parse json doc: %+v", err)
+	}
+
+	for _, d := range deep.Equal(actualDoc, expectedDoc) {
+		t.Errorf("diff: %+v", d)
+	}
+}
+
+type mockPackageSBOMImportAPI struct {
+	sessionID      string
+	model          external.ImagePackageManifest
+	httpResponse   *http.Response
+	err            error
+	ctx            context.Context
+	responseDigest string
+}
+
+func (m *mockPackageSBOMImportAPI) ImportImagePackages(ctx context.Context, sessionID string, model external.ImagePackageManifest) (external.ImageImportContentResponse, *http.Response, error) {
+	m.model = model
+	m.sessionID = sessionID
+	m.ctx = ctx
+	if m.httpResponse == nil {
+		m.httpResponse = &http.Response{}
+	}
+	m.httpResponse.Body = ioutils.NewReadCloserWrapper(strings.NewReader(""), func() error { return nil })
+	return external.ImageImportContentResponse{Digest: m.responseDigest}, m.httpResponse, m.err
+}
+
+func TestPackageSbomImport(t *testing.T) {
+
+	catalog := pkg.NewCatalog(pkg.Package{
+		Name:    "name",
+		Version: "version",
+		FoundBy: "foundBy",
+		Locations: []source.Location{
+			{
+				Path:         "path",
+				FileSystemID: "layerID",
+			},
+		},
+		Licenses: []string{"license"},
+		Language: pkg.Python,
+		Type:     pkg.PythonPkg,
+		CPEs: []pkg.CPE{
+			must(pkg.NewCPE("cpe:2.3:*:some:package:1:*:*:*:*:*:*:*")),
+		},
+		PURL:         "purl",
+		MetadataType: pkg.PythonPackageMetadataType,
+		Metadata: pkg.PythonPackageMetadata{
+			Name:        "p-name",
+			Version:     "p-version",
+			License:     "p-license",
+			Author:      "p-author",
+			AuthorEmail: "p-email",
+			Platform:    "p-platform",
+			Files: []pkg.PythonFileRecord{
+				{
+					Path: "p-path",
+					Digest: &pkg.PythonFileDigest{
+						Algorithm: "p-alg",
+						Value:     "p-digest",
+					},
+					Size: "p-size",
+				},
+			},
+			SitePackagesRootPath: "p-site-packages-root",
+			TopLevelPackages:     []string{"top-level"},
+		},
+	})
+
+	m := source.Metadata{
+		Scheme: "a-schema",
+		ImageMetadata: source.ImageMetadata{
+			UserInput:      "user-in",
+			Scope:          "scope!",
+			Layers:         nil,
+			Size:           10,
+			ManifestDigest: "sha256:digest!",
+			MediaType:      "mediatype!",
+			Tags:           nil,
+		},
+	}
+
+	d, _ := distro.NewDistro(distro.CentOS, "8.0", "")
+
+	theModel, err := packageSbomModel(m, catalog, &d)
+	if err != nil {
+		t.Fatalf("could not get sbom model: %+v", err)
+	}
+
+	sessionID := "my-session"
+
+	tests := []struct {
+		name         string
+		api          *mockPackageSBOMImportAPI
+		expectsError bool
+	}{
+
+		{
+			name: "Go case: import works",
+			api: &mockPackageSBOMImportAPI{
+				httpResponse:   &http.Response{StatusCode: 200},
+				responseDigest: "digest!",
+			},
+		},
+		{
+			name: "API returns an error",
+			api: &mockPackageSBOMImportAPI{
+				err: fmt.Errorf("API error, something went wrong."),
+			},
+			expectsError: true,
+		},
+		{
+			name: "API HTTP-level error",
+			api: &mockPackageSBOMImportAPI{
+				httpResponse: &http.Response{StatusCode: 404},
+			},
+			expectsError: true,
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+
+			digest, err := importPackageSBOM(context.TODO(), test.api, sessionID, m, catalog, &d, &progress.Stage{})
+
+			// validate error handling
+			if err != nil && !test.expectsError {
+				t.Fatalf("did not expect an error, but got: %+v", err)
+			} else if err == nil && test.expectsError {
+				t.Fatalf("did expect an error, but got none")
+			}
+
+			if digest != test.api.responseDigest {
+				t.Errorf("unexpected content digest: %q != %q", digest, test.api.responseDigest)
+			}
+
+			// validating that the mock got the right parameters (api.ImportImagePackages)
+			if test.api.sessionID != sessionID {
+				t.Errorf("different session ID: %s != %s", test.api.sessionID, sessionID)
+			}
+
+			for _, d := range deep.Equal(&test.api.model, theModel) {
+				t.Errorf("model difference: %s", d)
+			}
+
+		})
+	}
+}

internal/config/config.go

@@ -8,70 +8,82 @@ import (
 	"github.com/adrg/xdg"
 	"github.com/anchore/syft/internal"
 	"github.com/anchore/syft/syft/presenter"
-	"github.com/anchore/syft/syft/scope"
+	"github.com/anchore/syft/syft/source"
 	"github.com/mitchellh/go-homedir"
 	"github.com/sirupsen/logrus"
 	"github.com/spf13/viper"
+	"gopkg.in/yaml.v2"
 )
 
-type CliOnlyOptions struct {
-	ConfigPath string
-	Verbosity  int
-}
-
+// Application is the main syft application configuration.
 type Application struct {
-	ConfigPath        string
-	PresenterOpt      presenter.Option
-	Output            string `mapstructure:"output"`
-	ScopeOpt          scope.Option
-	Scope             string  `mapstructure:"scope"`
-	Quiet             bool    `mapstructure:"quiet"`
-	Log               Logging `mapstructure:"log"`
-	CliOptions        CliOnlyOptions
-	CheckForAppUpdate bool `mapstructure:"check-for-app-update"`
+	ConfigPath        string           `yaml:",omitempty"`                   // the location where the application config was read from (either from -c or discovered while loading)
+	PresenterOpt      presenter.Option `yaml:"-"`                            // -o, the native Presenter.Option to use for report formatting
+	Output            string           `yaml:"output" mapstructure:"output"` // -o, the Presenter hint string to use for report formatting
+	ScopeOpt          source.Scope     `yaml:"-"`                            // -s, the native source.Scope option to use for how to catalog the container image
+	Scope             string           `yaml:"scope" mapstructure:"scope"`   // -s, the source.Scope string hint for how to catalog the container image
+	Quiet             bool             `yaml:"quiet" mapstructure:"quiet"`   // -q, indicates to not show any status output to stderr (ETUI or logging UI)
+	Log               logging          `yaml:"log"  mapstructure:"log"`      // all logging-related options
+	CliOptions        CliOnlyOptions   `yaml:"-"`                            // all options only available through the CLI (not via env vars or config)
+	Dev               Development      `mapstructure:"dev"`
+	CheckForAppUpdate bool             `yaml:"check-for-app-update" mapstructure:"check-for-app-update"` // whether to check for an application update on start up or not
+	Anchore           anchore          `yaml:"anchore" mapstructure:"anchore"`                           // options for interacting with Anchore Engine/Enterprise
 }
 
-type Logging struct {
-	Structured   bool `mapstructure:"structured"`
-	LevelOpt     logrus.Level
-	Level        string `mapstructure:"level"`
-	FileLocation string `mapstructure:"file"`
+// CliOnlyOptions are options that are in the application config in memory, but are only exposed via CLI switches (not from unmarshaling a config file)
+type CliOnlyOptions struct {
+	ConfigPath string // -c. where the read config is on disk
+	Verbosity  int    // -v or -vv , controlling which UI (ETUI vs logging) and what the log level should be
 }
 
-func setNonCliDefaultValues(v *viper.Viper) {
-	v.SetDefault("log.level", "")
-	v.SetDefault("log.file", "")
-	v.SetDefault("log.structured", false)
-	v.SetDefault("check-for-app-update", true)
+// logging contains all logging-related configuration options available to the user via the application config.
+type logging struct {
+	Structured   bool         `yaml:"structured" mapstructure:"structured"` // show all log entries as JSON formatted strings
+	LevelOpt     logrus.Level `yaml:"level"`                                // the native log level object used by the logger
+	Level        string       `yaml:"-" mapstructure:"level"`               // the log level string hint
+	FileLocation string       `yaml:"file" mapstructure:"file"`             // the file path to write logs to
 }
 
-func LoadConfigFromFile(v *viper.Viper, cliOpts *CliOnlyOptions) (*Application, error) {
+type anchore struct {
+	// upload options
+	UploadEnabled          bool   `yaml:"upload-enabled"  mapstructure:"upload-enabled"`                    // whether to upload results to Anchore Engine/Enterprise (defaults to "false" unless there is the presence of -h CLI option)
+	Host                   string `yaml:"host" mapstructure:"host"`                                         // -H , hostname of the engine/enterprise instance to upload to
+	Path                   string `yaml:"path" mapstructure:"path"`                                         // override the engine/enterprise API upload path
+	Username               string `yaml:"username" mapstructure:"username"`                                 // -u , username to authenticate upload
+	Password               string `yaml:"password" mapstructure:"password"`                                 // -p , password to authenticate upload
+	Dockerfile             string `yaml:"dockerfile" mapstructure:"dockerfile"`                             // -d , dockerfile to attach for upload
+	OverwriteExistingImage bool   `yaml:"overwrite-existing-image" mapstructure:"overwrite-existing-image"` // --overwrite-existing-image , if any of the SBOM components have already been uploaded this flag will ensure they are overwritten with the current upload
+}
+
+type Development struct {
+	ProfileCPU bool `mapstructure:"profile-cpu"`
+	ProfileMem bool `mapstructure:"profile-mem"`
+}
+
+// LoadApplicationConfig populates the given viper object with application configuration discovered on disk
+func LoadApplicationConfig(v *viper.Viper, cliOpts CliOnlyOptions, wasHostnameSet bool) (*Application, error) {
 	// the user may not have a config, and this is OK, we can use the default config + default cobra cli values instead
 	setNonCliDefaultValues(v)
-	if cliOpts != nil {
-		_ = readConfig(v, cliOpts.ConfigPath)
-	} else {
-		_ = readConfig(v, "")
-	}
+	_ = readConfig(v, cliOpts.ConfigPath)
 
 	config := &Application{
-		CliOptions: *cliOpts,
+		CliOptions: cliOpts,
 	}
-	err := v.Unmarshal(config)
-	if err != nil {
+
+	if err := v.Unmarshal(config); err != nil {
 		return nil, fmt.Errorf("unable to parse config: %w", err)
 	}
 	config.ConfigPath = v.ConfigFileUsed()
 
-	err = config.Build()
-	if err != nil {
+	if err := config.build(v, wasHostnameSet); err != nil {
 		return nil, fmt.Errorf("invalid config: %w", err)
 	}
 
 	return config, nil
 }
 
-func (cfg *Application) Build() error {
+// build inflates simple config values into syft native objects (or other complex objects) after the config is fully read in.
+func (cfg *Application) build(v *viper.Viper, wasHostnameSet bool) error {
 	// set the presenter
 	presenterOption := presenter.ParseOption(cfg.Output)
 	if presenterOption == presenter.UnknownPresenter {
@@ -79,9 +91,9 @@ func (cfg *Application) Build() error {
 	}
 	cfg.PresenterOpt = presenterOption
 
-	// set the scope
-	scopeOption := scope.ParseOption(cfg.Scope)
-	if scopeOption == scope.UnknownScope {
+	// set the source
+	scopeOption := source.ParseScope(cfg.Scope)
+	if scopeOption == source.UnknownScope {
 		return fmt.Errorf("bad --scope value '%s'", cfg.Scope)
 	}
 	cfg.ScopeOpt = scopeOption
@@ -111,14 +123,44 @@ func (cfg *Application) Build() error {
 			case v >= 2:
 				cfg.Log.LevelOpt = logrus.DebugLevel
 			default:
-				cfg.Log.LevelOpt = logrus.ErrorLevel
+				cfg.Log.LevelOpt = logrus.WarnLevel
 			}
 		}
 	}
+	// check if upload should be done relative to the CLI and config behavior
+	if !v.IsSet("anchore.upload-enabled") && wasHostnameSet {
+		// we know the user didn't specify to upload in the config file and a --hostname option was provided (so set upload)
+		cfg.Anchore.UploadEnabled = true
+	}
+
+	if !cfg.Anchore.UploadEnabled && cfg.Anchore.Dockerfile != "" {
+		return fmt.Errorf("cannot provide dockerfile option without enabling upload")
+	}
 
 	return nil
 }
 
+func (cfg Application) String() string {
+	// redact sensitive information
+	if cfg.Anchore.Username != "" {
+		cfg.Anchore.Username = "********"
+	}
+
+	if cfg.Anchore.Password != "" {
+		cfg.Anchore.Password = "********"
+	}
+
+	// yaml is pretty human friendly (at least when compared to json)
+	appCfgStr, err := yaml.Marshal(&cfg)
+
+	if err != nil {
+		return err.Error()
+	}
+
+	return string(appCfgStr)
+}
+
+// readConfig attempts to read the given config path from disk or discover an alternate store location
 func readConfig(v *viper.Viper, configPath string) error {
 	v.AutomaticEnv()
 	v.SetEnvPrefix(internal.ApplicationName)
@@ -174,3 +216,13 @@ func readConfig(v *viper.Viper, configPath string) error {
 
 	return fmt.Errorf("application config not found")
 }
+
+// setNonCliDefaultValues ensures that there are sane defaults for values that do not have CLI equivalent options (where there would already be a default value)
+func setNonCliDefaultValues(v *viper.Viper) {
+	v.SetDefault("log.level", "")
+	v.SetDefault("log.file", "")
+	v.SetDefault("log.structured", false)
+	v.SetDefault("check-for-app-update", true)
+	v.SetDefault("dev.profile-cpu", false)
+	v.SetDefault("dev.profile-mem", false)
+}

internal/constants.go

@@ -1,4 +1,10 @@
 package internal
 
-// note: do not change this
-const ApplicationName = "syft"
+const (
+	// ApplicationName is the non-capitalized name of the application (do not change this)
+	ApplicationName = "syft"
+
+	// JSONSchemaVersion is the current schema version output by the JSON presenter
+	// This is roughly following the "SchemaVer" guidelines for versioning the JSON schema. Please see schema/json/README.md for details on how to increment.
+	JSONSchemaVersion = "1.0.1"
+)

internal/docs.go

@@ -0,0 +1,4 @@
+/*
+Package internal contains miscellaneous functions and objects useful within syft but should not be used externally.
+*/
+package internal

internal/file/glob_match.go

@@ -1,5 +1,6 @@
 package file
 
+// GlobMatch evaluates the given glob pattern against the given "name" string, indicating if there is a match or not.
 // Source: https://research.swtch.com/glob.go
 func GlobMatch(pattern, name string) bool {
 	px := 0

internal/file/opener.go

@@ -0,0 +1,16 @@
+package file
+
+import (
+	"io"
+	"os"
+)
+
+// Opener is an object that stores a path to later be opened as a file.
+type Opener struct {
+	path string
+}
+
+// Open the stored path as a io.ReadCloser.
+func (o Opener) Open() (io.ReadCloser, error) {
+	return os.Open(o.path)
+}

internal/file/test-fixtures/generate-zip-fixture.sh

@@ -1,4 +1,7 @@
 #!/usr/bin/env bash
 set -eux
 
-zip -r "$1" zip-source
+# $1 —— absolute path to destination file, should end with .zip, ideally
+# $2 —— absolute path to directory from which to add entries to the archive
+
+pushd "$2" && find . -print | zip "$1" -@ && popd

internal/file/zip_file_helpers_test.go

@@ -0,0 +1,131 @@
+package file
+
+import (
+	"fmt"
+	"io/ioutil"
+	"os"
+	"os/exec"
+	"path"
+	"path/filepath"
+	"syscall"
+	"testing"
+)
+
+var expectedZipArchiveEntries = []string{
+	"some-dir" + string(os.PathSeparator),
+	filepath.Join("some-dir", "a-file.txt"),
+	"b-file.txt",
+	"nested.zip",
+}
+
+// fatalIfError calls the supplied function. If the function returns a non-nil error, t.Fatal(err) is called.
+func fatalIfError(t *testing.T, fn func() error) {
+	t.Helper()
+
+	if fn == nil {
+		return
+	}
+
+	err := fn()
+	if err != nil {
+		t.Fatal(err)
+	}
+}
+
+// createZipArchive creates a new ZIP archive file at destinationArchivePath based on the directory found at
+// sourceDirPath.
+func createZipArchive(t *testing.T, sourceDirPath, destinationArchivePath string) error {
+	t.Helper()
+
+	cwd, err := os.Getwd()
+	if err != nil {
+		return fmt.Errorf("unable to get cwd: %+v", err)
+	}
+
+	cmd := exec.Command("./generate-zip-fixture.sh", destinationArchivePath, path.Base(sourceDirPath))
+	cmd.Dir = filepath.Join(cwd, "test-fixtures")
+
+	if err := cmd.Start(); err != nil {
+		return fmt.Errorf("unable to start generate zip fixture script: %+v", err)
+	}
+
+	if err := cmd.Wait(); err != nil {
+		if exiterr, ok := err.(*exec.ExitError); ok {
+			// The program has exited with an exit code != 0
+
+			// This works on both Unix and Windows. Although package
+			// syscall is generally platform dependent, WaitStatus is
+			// defined for both Unix and Windows and in both cases has
+			// an ExitStatus() method with the same signature.
+			if status, ok := exiterr.Sys().(syscall.WaitStatus); ok {
+				if status.ExitStatus() != 0 {
+					return fmt.Errorf("failed to generate fixture: rc=%d", status.ExitStatus())
+				}
+			}
+		} else {
+			return fmt.Errorf("unable to get generate fixture script result: %+v", err)
+		}
+	}
+
+	return nil
+}
+
+// setupZipFileTest encapsulates common test setup work for zip file tests. It returns a cleanup function,
+// which should be called (typically deferred) by the caller, the path of the created zip archive, and an error,
+// which should trigger a fatal test failure in the consuming test. The returned cleanup function will never be nil
+// (even if there's an error), and it should always be called.
+func setupZipFileTest(t *testing.T, sourceDirPath string) (func() error, string, error) {
+	t.Helper()
+
+	// Keep track of any needed cleanup work as we go
+	var cleanupFns []func() error
+	cleanup := func(fns []func() error) func() error {
+		return func() error {
+			for _, fn := range fns {
+				err := fn()
+				if err != nil {
+					return err
+				}
+			}
+
+			return nil
+		}
+	}
+
+	archivePrefix, err := ioutil.TempFile("", "syft-ziputil-archive-TEST-")
+	if err != nil {
+		return cleanup(cleanupFns), "", fmt.Errorf("unable to create tempfile: %+v", err)
+	}
+	cleanupFns = append(cleanupFns, func() error { return os.Remove(archivePrefix.Name()) })
+
+	destinationArchiveFilePath := archivePrefix.Name() + ".zip"
+	t.Logf("archive path: %s", destinationArchiveFilePath)
+	err = createZipArchive(t, sourceDirPath, destinationArchiveFilePath)
+	cleanupFns = append(cleanupFns, func() error { return os.Remove(destinationArchiveFilePath) })
+	if err != nil {
+		return cleanup(cleanupFns), "", err
+	}
+
+	cwd, err := os.Getwd()
+	if err != nil {
+		return cleanup(cleanupFns), "", fmt.Errorf("unable to get cwd: %+v", err)
+	}
+
+	t.Logf("running from: %s", cwd)
+
+	return cleanup(cleanupFns), destinationArchiveFilePath, nil
+}
+
+// TODO: Consider moving any non-git asset generation to a task (e.g. make) that's run ahead of running go tests.
+func ensureNestedZipExists(t *testing.T, sourceDirPath string) error {
+	t.Helper()
+
+	nestedArchiveFilePath := path.Join(sourceDirPath, "nested.zip")
+	err := createZipArchive(t, sourceDirPath, nestedArchiveFilePath)
+
+	if err != nil {
+		return fmt.Errorf("unable to create nested archive for test fixture: %+v", err)
+	}
+
+	return nil
+}

internal/file/zip_file_manifest.go

@@ -5,28 +5,37 @@ import (
 	"fmt"
 	"os"
 	"sort"
+	"strings"
 
 	"github.com/anchore/syft/internal"
 
 	"github.com/anchore/syft/internal/log"
 )
 
+// ZipFileManifest is a collection of paths and their file metadata.
 type ZipFileManifest map[string]os.FileInfo
 
+// newZipManifest creates an empty ZipFileManifest.
 func newZipManifest() ZipFileManifest {
 	return make(ZipFileManifest)
 }
 
+// Add a new path and it's file metadata to the collection.
 func (z ZipFileManifest) Add(entry string, info os.FileInfo) {
 	z[entry] = info
 }
 
+// GlobMatch returns the path keys that match the given value(s).
 func (z ZipFileManifest) GlobMatch(patterns ...string) []string {
 	uniqueMatches := internal.NewStringSet()
 
 	for _, pattern := range patterns {
 		for entry := range z {
-			if GlobMatch(pattern, entry) {
+			// We want to match globs as if entries begin with a leading slash (akin to an absolute path)
+			// so that glob logic is consistent inside and outside of ZIP archives
+			normalizedEntry := normalizeZipEntryName(entry)
+
+			if GlobMatch(pattern, normalizedEntry) {
 				uniqueMatches.Add(entry)
 			}
 		}
@@ -38,6 +47,7 @@ func (z ZipFileManifest) GlobMatch(patterns ...string) []string {
 	return results
 }
 
+// NewZipFileManifest creates and returns a new ZipFileManifest populated with path and metadata from the given zip archive path.
 func NewZipFileManifest(archivePath string) (ZipFileManifest, error) {
 	zipReader, err := zip.OpenReader(archivePath)
 	manifest := newZipManifest()
@@ -56,3 +66,12 @@ func NewZipFileManifest(archivePath string) (ZipFileManifest, error) {
 	}
 	return manifest, nil
 }
+
+// normalizeZipEntryName takes the given path entry and ensures it is prefixed with "/".
+func normalizeZipEntryName(entry string) string {
+	if !strings.HasPrefix(entry, "/") {
+		return "/" + entry
+	}
+
+	return entry
+}

internal/file/zip_file_manifest_test.go

@@ -0,0 +1,109 @@
+package file
+
+import (
+	"encoding/json"
+	"os"
+	"path"
+	"testing"
+)
+
+func TestNewZipFileManifest(t *testing.T) {
+	cwd, err := os.Getwd()
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	sourceDirPath := path.Join(cwd, "test-fixtures", "zip-source")
+	err = ensureNestedZipExists(t, sourceDirPath)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	cleanup, archiveFilePath, err := setupZipFileTest(t, sourceDirPath)
+	defer fatalIfError(t, cleanup)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	actual, err := NewZipFileManifest(archiveFilePath)
+	if err != nil {
+		t.Fatalf("unable to extract from unzip archive: %+v", err)
+	}
+
+	if len(expectedZipArchiveEntries) != len(actual) {
+		t.Fatalf("mismatched manifest: %d != %d", len(actual), len(expectedZipArchiveEntries))
+	}
+
+	for _, e := range expectedZipArchiveEntries {
+		_, ok := actual[e]
+		if !ok {
+			t.Errorf("missing path: %s", e)
+		}
+	}
+
+	if t.Failed() {
+		b, err := json.MarshalIndent(actual, "", "  ")
+		if err != nil {
+			t.Fatalf("can't show results: %+v", err)
+		}
+
+		t.Errorf("full result: %s", string(b))
+	}
+}
+
+func TestZipFileManifest_GlobMatch(t *testing.T) {
+	cwd, err := os.Getwd()
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	sourceDirPath := path.Join(cwd, "test-fixtures", "zip-source")
+	err = ensureNestedZipExists(t, sourceDirPath)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	cleanup, archiveFilePath, err := setupZipFileTest(t, sourceDirPath)
+	//goland:noinspection GoNilness
+	defer fatalIfError(t, cleanup)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	z, err := NewZipFileManifest(archiveFilePath)
+	if err != nil {
+		t.Fatalf("unable to extract from unzip archive: %+v", err)
+	}
+
+	cases := []struct {
+		glob     string
+		expected string
+	}{
+		{
+			"/b*",
+			"b-file.txt",
+		},
+		{
+			"*/a-file.txt",
+			"some-dir/a-file.txt",
+		},
+		{
+			"**/*.zip",
+			"nested.zip",
+		},
+	}
+
+	for _, tc := range cases {
+		t.Run(tc.glob, func(t *testing.T) {
+			glob := tc.glob
+
+			results := z.GlobMatch(glob)
+
+			if len(results) == 1 && results[0] == tc.expected {
+				return
+			}
+
+			t.Errorf("unexpected results for glob '%s': %+v", glob, results)
+		})
+	}
+}

internal/file/zip_file_traversal.go

@@ -15,6 +15,7 @@ import (
 )
 
 const (
+	// represents the order of bytes
 	_  = iota
 	KB = 1 << (10 * iota)
 	MB
@@ -33,6 +34,7 @@ func newZipTraverseRequest(paths ...string) zipTraversalRequest {
 	return results
 }
 
+// TraverseFilesInZip enumerates all paths stored within a zip archive using the visitor pattern.
 func TraverseFilesInZip(archivePath string, visitor func(*zip.File) error, paths ...string) error {
 	request := newZipTraverseRequest(paths...)
 
@@ -63,8 +65,9 @@ func TraverseFilesInZip(archivePath string, visitor func(*zip.File) error, paths
 	return nil
 }
 
-func ExtractFromZipToUniqueTempFile(archivePath, dir string, paths ...string) (map[string]io.Reader, error) {
-	results := make(map[string]io.Reader)
+// ExtractFromZipToUniqueTempFile extracts select paths for the given archive to a temporary directory, returning file openers for each file extracted.
+func ExtractFromZipToUniqueTempFile(archivePath, dir string, paths ...string) (map[string]Opener, error) {
+	results := make(map[string]Opener)
 
 	// don't allow for full traversal, only select traversal from given paths
 	if len(paths) == 0 {
@@ -78,11 +81,21 @@ func ExtractFromZipToUniqueTempFile(archivePath, dir string, paths ...string) (m
 		if err != nil {
 			return fmt.Errorf("unable to create temp file: %w", err)
 		}
+		// we shouldn't try and keep the tempfile open as the returned result may have several files, which takes up
+		// resources (leading to "too many open files"). Instead we'll return a file opener to the caller which
+		// provides a ReadCloser. It is up to the caller to handle closing the file explicitly.
+		defer tempFile.Close()
 
 		zippedFile, err := file.Open()
 		if err != nil {
 			return fmt.Errorf("unable to read file=%q from zip=%q: %w", file.Name, archivePath, err)
 		}
+		defer func() {
+			err := zippedFile.Close()
+			if err != nil {
+				log.Errorf("unable to close source file=%q from zip=%q: %+v", file.Name, archivePath, err)
+			}
+		}()
 
 		if file.FileInfo().IsDir() {
 			return fmt.Errorf("unable to extract directories, only files: %s", file.Name)
@@ -103,18 +116,15 @@ func ExtractFromZipToUniqueTempFile(archivePath, dir string, paths ...string) (m
 			return fmt.Errorf("unable to reset file pointer (%s): %w", tempFile.Name(), err)
 		}
 
-		results[file.Name] = tempFile
+		results[file.Name] = Opener{path: tempFile.Name()}
 
-		err = zippedFile.Close()
-		if err != nil {
-			return fmt.Errorf("unable to close source file=%q from zip=%q: %w", file.Name, archivePath, err)
-		}
 		return nil
 	}
 
 	return results, TraverseFilesInZip(archivePath, visitor, paths...)
 }
 
+// ContentsFromZip extracts select paths for the given archive and returns a set of string contents for each path.
 func ContentsFromZip(archivePath string, paths ...string) (map[string]string, error) {
 	results := make(map[string]string)
 
@@ -156,6 +166,7 @@ func ContentsFromZip(archivePath string, paths ...string) (map[string]string, er
 	return results, TraverseFilesInZip(archivePath, visitor, paths...)
 }
 
+// UnzipToDir extracts a zip archive to a target directory.
 func UnzipToDir(archivePath, targetDir string) error {
 	visitor := func(file *zip.File) error {
 		// the zip-slip attack protection is still being erroneously detected

internal/file/zip_file_traversal_test.go

@@ -6,47 +6,14 @@ import (
 	"io"
 	"io/ioutil"
 	"os"
-	"os/exec"
+	"path"
 	"path/filepath"
 	"strings"
-	"syscall"
 	"testing"
 
 	"github.com/go-test/deep"
 )
 
-func generateFixture(t *testing.T, archivePath string) {
-	cwd, err := os.Getwd()
-	if err != nil {
-		t.Errorf("unable to get cwd: %+v", err)
-	}
-
-	cmd := exec.Command("./generate-zip-fixture.sh", archivePath)
-	cmd.Dir = filepath.Join(cwd, "test-fixtures")
-
-	if err := cmd.Start(); err != nil {
-		t.Fatalf("unable to start generate zip fixture script: %+v", err)
-	}
-
-	if err := cmd.Wait(); err != nil {
-		if exiterr, ok := err.(*exec.ExitError); ok {
-			// The program has exited with an exit code != 0
-
-			// This works on both Unix and Windows. Although package
-			// syscall is generally platform dependent, WaitStatus is
-			// defined for both Unix and Windows and in both cases has
-			// an ExitStatus() method with the same signature.
-			if status, ok := exiterr.Sys().(syscall.WaitStatus); ok {
-				if status.ExitStatus() != 0 {
-					t.Fatalf("failed to generate fixture: rc=%d", status.ExitStatus())
-				}
-			}
-		} else {
-			t.Fatalf("unable to get generate fixture script result: %+v", err)
-		}
-	}
-}
-
 func equal(r1, r2 io.Reader) (bool, error) {
 	w1 := sha256.New()
 	w2 := sha256.New()
@@ -67,54 +34,50 @@ func equal(r1, r2 io.Reader) (bool, error) {
 }
 
 func TestUnzipToDir(t *testing.T) {
-	archivePrefix, err := ioutil.TempFile("", "syft-ziputil-archive-TEST-")
+	cwd, err := os.Getwd()
 	if err != nil {
-		t.Fatalf("unable to create tempfile: %+v", err)
+		t.Fatal(err)
 	}
-	defer os.Remove(archivePrefix.Name())
-	// the zip utility will add ".zip" to the end of the given name
-	archivePath := archivePrefix.Name() + ".zip"
-	defer os.Remove(archivePath)
-	t.Logf("archive path: %s", archivePath)
 
-	generateFixture(t, archivePrefix.Name())
+	goldenRootDir := filepath.Join(cwd, "test-fixtures")
+	sourceDirPath := path.Join(goldenRootDir, "zip-source")
+	cleanup, archiveFilePath, err := setupZipFileTest(t, sourceDirPath)
+	defer fatalIfError(t, cleanup)
+	if err != nil {
+		t.Fatal(err)
+	}
 
-	contentsDir, err := ioutil.TempDir("", "syft-ziputil-contents-TEST-")
+	unzipDestinationDir, err := ioutil.TempDir("", "syft-ziputil-contents-TEST-")
+	defer os.RemoveAll(unzipDestinationDir)
 	if err != nil {
 		t.Fatalf("unable to create tempdir: %+v", err)
 	}
-	defer os.RemoveAll(contentsDir)
 
-	t.Logf("content path: %s", contentsDir)
+	t.Logf("content path: %s", unzipDestinationDir)
 
-	cwd, err := os.Getwd()
-	if err != nil {
-		t.Errorf("unable to get cwd: %+v", err)
-	}
-
-	t.Logf("running from: %s", cwd)
-
-	// note: zip utility already includes "zip-source" as a parent dir for all contained files
-	goldenRootDir := filepath.Join(cwd, "test-fixtures")
-	expectedPaths := 4
+	expectedPaths := len(expectedZipArchiveEntries)
 	observedPaths := 0
 
-	err = UnzipToDir(archivePath, contentsDir)
+	err = UnzipToDir(archiveFilePath, unzipDestinationDir)
 	if err != nil {
 		t.Fatalf("unable to unzip archive: %+v", err)
 	}
 
 	// compare the source dir tree and the unzipped tree
-	err = filepath.Walk(filepath.Join(contentsDir, "zip-source"),
+	err = filepath.Walk(unzipDestinationDir,
 		func(path string, info os.FileInfo, err error) error {
-			t.Logf("unzipped path: %s", path)
-			observedPaths++
+			// We don't unzip the root archive dir, since there's no archive entry for it
+			if path != unzipDestinationDir {
+				t.Logf("unzipped path: %s", path)
+				observedPaths++
+			}
+
 			if err != nil {
 				t.Fatalf("this should not happen")
 				return err
 			}
 
-			goldenPath := filepath.Join(goldenRootDir, strings.TrimPrefix(path, contentsDir))
+			goldenPath := filepath.Join(sourceDirPath, strings.TrimPrefix(path, unzipDestinationDir))
 
 			if info.IsDir() {
 				i, err := os.Stat(goldenPath)
@@ -156,12 +119,11 @@ func TestUnzipToDir(t *testing.T) {
 	}
 
 	if observedPaths != expectedPaths {
-		t.Errorf("missed test paths: %d!=%d", observedPaths, expectedPaths)
+		t.Errorf("missed test paths: %d != %d", observedPaths, expectedPaths)
 	}
-
 }
 
-func TestExtractFilesFromZipFile(t *testing.T) {
+func TestContentsFromZip(t *testing.T) {
 	archivePrefix, err := ioutil.TempFile("", "syft-ziputil-archive-TEST-")
 	if err != nil {
 		t.Fatalf("unable to create tempfile: %+v", err)
@@ -172,7 +134,10 @@ func TestExtractFilesFromZipFile(t *testing.T) {
 	defer os.Remove(archivePath)
 	t.Logf("archive path: %s", archivePath)
 
-	generateFixture(t, archivePrefix.Name())
+	err = createZipArchive(t, "zip-source", archivePrefix.Name())
+	if err != nil {
+		t.Fatal(err)
+	}
 
 	cwd, err := os.Getwd()
 	if err != nil {
@@ -181,8 +146,8 @@ func TestExtractFilesFromZipFile(t *testing.T) {
 
 	t.Logf("running from: %s", cwd)
 
-	aFilePath := filepath.Join("zip-source", "some-dir", "a-file.txt")
-	bFilePath := filepath.Join("zip-source", "b-file.txt")
+	aFilePath := filepath.Join("some-dir", "a-file.txt")
+	bFilePath := filepath.Join("b-file.txt")
 
 	expected := map[string]string{
 		aFilePath: "A file! nice!",
@@ -207,60 +172,4 @@ func TestExtractFilesFromZipFile(t *testing.T) {
 
 		t.Errorf("full result: %s", string(b))
 	}
-
-}
-
-func TestZipFileManifest(t *testing.T) {
-	archivePrefix, err := ioutil.TempFile("", "syft-ziputil-archive-TEST-")
-	if err != nil {
-		t.Fatalf("unable to create tempfile: %+v", err)
-	}
-	defer os.Remove(archivePrefix.Name())
-	// the zip utility will add ".zip" to the end of the given name
-	archivePath := archivePrefix.Name() + ".zip"
-	defer os.Remove(archivePath)
-	t.Logf("archive path: %s", archivePath)
-
-	generateFixture(t, archivePrefix.Name())
-
-	cwd, err := os.Getwd()
-	if err != nil {
-		t.Errorf("unable to get cwd: %+v", err)
-	}
-
-	t.Logf("running from: %s", cwd)
-
-	expected := []string{
-		filepath.Join("zip-source") + string(os.PathSeparator),
-		filepath.Join("zip-source", "some-dir") + string(os.PathSeparator),
-		filepath.Join("zip-source", "some-dir", "a-file.txt"),
-		filepath.Join("zip-source", "b-file.txt"),
-	}
-
-	actual, err := NewZipFileManifest(archivePath)
-	if err != nil {
-		t.Fatalf("unable to extract from unzip archive: %+v", err)
-	}
-
-	if len(expected) != len(actual) {
-		t.Fatalf("mismatched manifest: %d != %d", len(actual), len(expected))
-	}
-
-	for _, e := range expected {
-		_, ok := actual[e]
-		if !ok {
-			t.Errorf("missing path: %s", e)
-		}
-	}
-
-	if t.Failed() {
-
-		b, err := json.MarshalIndent(actual, "", "  ")
-		if err != nil {
-			t.Fatalf("can't show results: %+v", err)
-		}
-
-		t.Errorf("full result: %s", string(b))
-	}
-
 }

internal/format/color.go

@@ -1,21 +0,0 @@
-package format
-
-import "fmt"
-
-const (
-	DefaultColor Color = iota + 30
-	Red
-	Green
-	Yellow
-	Blue
-	Magenta
-	Cyan
-	White
-)
-
-type Color uint8
-
-// TODO: not cross platform (windows...)
-func (c Color) Format(s string) string {
-	return fmt.Sprintf("\x1b[%dm%s\x1b[0m", c, s)
-}

internal/log/log.go

@@ -1,25 +1,49 @@
+/*
+Package log contains the singleton object and helper functions for facilitating logging within the syft library.
+*/
 package log
 
 import "github.com/anchore/syft/syft/logger"
 
+// Log is the singleton used to facilitate logging internally within syft
 var Log logger.Logger = &nopLogger{}
 
+// Errorf takes a formatted template string and template arguments for the error logging level.
 func Errorf(format string, args ...interface{}) {
 	Log.Errorf(format, args...)
 }
 
+// Error logs the given arguments at the error logging level.
+func Error(args ...interface{}) {
+	Log.Error(args...)
+}
+
+// Warnf takes a formatted template string and template arguments for the warning logging level.
+func Warnf(format string, args ...interface{}) {
+	Log.Warnf(format, args...)
+}
+
+// Warn logs the given arguments at the warning logging level.
+func Warn(args ...interface{}) {
+	Log.Warn(args...)
+}
+
+// Infof takes a formatted template string and template arguments for the info logging level.
 func Infof(format string, args ...interface{}) {
 	Log.Infof(format, args...)
 }
 
+// Info logs the given arguments at the info logging level.
 func Info(args ...interface{}) {
 	Log.Info(args...)
 }
 
+// Debugf takes a formatted template string and template arguments for the debug logging level.
 func Debugf(format string, args ...interface{}) {
 	Log.Debugf(format, args...)
 }
 
+// Debug logs the given arguments at the debug logging level.
 func Debug(args ...interface{}) {
 	Log.Debug(args...)
 }

internal/log/nop.go

@@ -3,6 +3,9 @@ package log
 type nopLogger struct{}
 
 func (l *nopLogger) Errorf(format string, args ...interface{}) {}
+func (l *nopLogger) Error(args ...interface{})                 {}
+func (l *nopLogger) Warnf(format string, args ...interface{})  {}
+func (l *nopLogger) Warn(args ...interface{})                  {}
 func (l *nopLogger) Infof(format string, args ...interface{})  {}
 func (l *nopLogger) Info(args ...interface{})                  {}
 func (l *nopLogger) Debugf(format string, args ...interface{}) {}

internal/logger/doc.go

@@ -0,0 +1,4 @@
+/*
+Package logger contains implementations for the syft.logger.Logger interface.
+*/
+package logger

internal/logger/logrus.go

@@ -10,6 +10,7 @@ import (
 	prefixed "github.com/x-cray/logrus-prefixed-formatter"
 )
 
+// LogrusConfig contains all configurable values for the Logrus logger
 type LogrusConfig struct {
 	EnableConsole bool
 	EnableFile    bool
@@ -18,16 +19,19 @@ type LogrusConfig struct {
 	FileLocation  string
 }
 
+// LogrusLogger contains all runtime values for using Logrus with the configured output target and input configuration values.
 type LogrusLogger struct {
 	Config LogrusConfig
 	Logger *logrus.Logger
 	Output io.Writer
 }
 
+// LogrusNestedLogger is a wrapper for Logrus to enable nested logging configuration (loggers that always attach key-value pairs to all log entries)
 type LogrusNestedLogger struct {
 	Logger *logrus.Entry
 }
 
+// NewLogrusLogger creates a new LogrusLogger with the given configuration
 func NewLogrusLogger(cfg LogrusConfig) *LogrusLogger {
 	appLogger := logrus.New()
 
@@ -76,42 +80,82 @@ func NewLogrusLogger(cfg LogrusConfig) *LogrusLogger {
 	}
 }
 
+// Debugf takes a formatted template string and template arguments for the debug logging level.
 func (l *LogrusLogger) Debugf(format string, args ...interface{}) {
 	l.Logger.Debugf(format, args...)
 }
 
+// Infof takes a formatted template string and template arguments for the info logging level.
 func (l *LogrusLogger) Infof(format string, args ...interface{}) {
 	l.Logger.Infof(format, args...)
 }
 
-func (l *LogrusLogger) Debug(args ...interface{}) {
-	l.Logger.Debug(args...)
-}
-
-func (l *LogrusLogger) Info(args ...interface{}) {
-	l.Logger.Info(args...)
+// Warnf takes a formatted template string and template arguments for the warning logging level.
+func (l *LogrusLogger) Warnf(format string, args ...interface{}) {
+	l.Logger.Warnf(format, args...)
 }
 
+// Errorf takes a formatted template string and template arguments for the error logging level.
 func (l *LogrusLogger) Errorf(format string, args ...interface{}) {
 	l.Logger.Errorf(format, args...)
 }
 
+// Debug logs the given arguments at the debug logging level.
+func (l *LogrusLogger) Debug(args ...interface{}) {
+	l.Logger.Debug(args...)
+}
+
+// Info logs the given arguments at the info logging level.
+func (l *LogrusLogger) Info(args ...interface{}) {
+	l.Logger.Info(args...)
+}
+
+// Warn logs the given arguments at the warning logging level.
+func (l *LogrusLogger) Warn(args ...interface{}) {
+	l.Logger.Warn(args...)
+}
+
+// Error logs the given arguments at the error logging level.
+func (l *LogrusLogger) Error(args ...interface{}) {
+	l.Logger.Error(args...)
+}
+
+// Debugf takes a formatted template string and template arguments for the debug logging level.
 func (l *LogrusNestedLogger) Debugf(format string, args ...interface{}) {
 	l.Logger.Debugf(format, args...)
 }
 
+// Infof takes a formatted template string and template arguments for the info logging level.
 func (l *LogrusNestedLogger) Infof(format string, args ...interface{}) {
 	l.Logger.Infof(format, args...)
 }
 
+// Warnf takes a formatted template string and template arguments for the warning logging level.
+func (l *LogrusNestedLogger) Warnf(format string, args ...interface{}) {
+	l.Logger.Warnf(format, args...)
+}
+
+// Errorf takes a formatted template string and template arguments for the error logging level.
+func (l *LogrusNestedLogger) Errorf(format string, args ...interface{}) {
+	l.Logger.Errorf(format, args...)
+}
+
+// Debug logs the given arguments at the debug logging level.
 func (l *LogrusNestedLogger) Debug(args ...interface{}) {
 	l.Logger.Debug(args...)
 }
 
+// Info logs the given arguments at the info logging level.
 func (l *LogrusNestedLogger) Info(args ...interface{}) {
 	l.Logger.Info(args...)
 }
 
-func (l *LogrusNestedLogger) Errorf(format string, args ...interface{}) {
-	l.Logger.Errorf(format, args...)
+// Warn logs the given arguments at the warning logging level.
+func (l *LogrusNestedLogger) Warn(args ...interface{}) {
+	l.Logger.Warn(args...)
+}
+
+// Error logs the given arguments at the error logging level.
+func (l *LogrusNestedLogger) Error(args ...interface{}) {
+	l.Logger.Error(args...)
 }

internal/parse.go

@@ -0,0 +1,15 @@
+package internal
+
+import "regexp"
+
+// MatchCaptureGroups takes a regular expression and string and returns all of the named capture group results in a map.
+func MatchCaptureGroups(regEx *regexp.Regexp, str string) map[string]string {
+	match := regEx.FindStringSubmatch(str)
+	results := make(map[string]string)
+	for i, name := range regEx.SubexpNames() {
+		if i > 0 && i <= len(match) {
+			results[name] = match[i]
+		}
+	}
+	return results
+}

internal/stringset.go

@@ -1,11 +1,16 @@
 package internal
 
+import "sort"
+
+// StringSet represents a set of string types.
 type StringSet map[string]struct{}
 
+// NewStringSet creates a new empty StringSet.
 func NewStringSet() StringSet {
 	return make(StringSet)
 }
 
+// NewStringSetFromSlice creates a StringSet populated with values from the given slice.
 func NewStringSetFromSlice(start []string) StringSet {
 	ret := make(StringSet)
 	for _, s := range start {
@@ -14,19 +19,23 @@ func NewStringSetFromSlice(start []string) StringSet {
 	return ret
 }
 
+// Add a string to the set.
 func (s StringSet) Add(i string) {
 	s[i] = struct{}{}
 }
 
+// Remove a string from the set.
 func (s StringSet) Remove(i string) {
 	delete(s, i)
 }
 
+// Contains indicates if the given string is contained within the set.
 func (s StringSet) Contains(i string) bool {
 	_, ok := s[i]
 	return ok
 }
 
+// ToSlice returns a sorted slice of strings that are contained within the set.
 func (s StringSet) ToSlice() []string {
 	ret := make([]string, len(s))
 	idx := 0
@@ -34,5 +43,6 @@ func (s StringSet) ToSlice() []string {
 		ret[idx] = v
 		idx++
 	}
+	sort.Strings(ret)
 	return ret
 }

internal/version/build.go

@@ -1,3 +1,6 @@
+/*
+Package version contains all build time metadata (version, build time, git commit, etc).
+*/
 package version
 
 import (

internal/version/update.go

@@ -18,6 +18,7 @@ var latestAppVersionURL = struct {
 	path: fmt.Sprintf("/%s/releases/latest/VERSION", internal.ApplicationName),
 }
 
+// IsUpdateAvailable indicates if there is a newer application version available, and if so, what the new version is.
 func IsUpdateAvailable() (bool, string, error) {
 	currentVersionStr := FromBuild().Version
 	currentVersion, err := hashiVersion.NewVersion(currentVersionStr)

schema/cyclonedx/bd.xsd

@@ -1,183 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
-CycloneDX BOM Descriptor Extension
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
--->
-<xs:schema xmlns:xs="http://www.w3.org/2001/XMLSchema"
-           xmlns:vc="http://www.w3.org/2007/XMLSchema-versioning"
-           xmlns:bom="http://cyclonedx.org/schema/bom/1.1"
-           xmlns:bd="http://cyclonedx.org/schema/ext/bom-descriptor/1.0"
-           elementFormDefault="qualified"
-           targetNamespace="http://cyclonedx.org/schema/ext/bom-descriptor/1.0"
-           vc:minVersion="1.0"
-           vc:maxVersion="1.1"
-           version="1.0">
-
-  <xs:annotation>
-    <xs:documentation>
-      <name>CycloneDX BOM Descriptor Extension</name>
-      <url>https://cyclonedx.org/ext/bom-descriptor</url>
-      <license uri="http://www.apache.org/licenses/LICENSE-2.0"
-               version="2.0">Apache License, Version 2.0</license>
-      <authors>
-        <author>Steve Springett</author>
-      </authors>
-    </xs:documentation>
-  </xs:annotation>
-
-  <xs:import namespace="http://cyclonedx.org/schema/bom/1.1" schemaLocation="http://cyclonedx.org/schema/bom/1.1"/>
-
-  <xs:complexType name="metadata">
-    <xs:sequence minOccurs="0" maxOccurs="1">
-      <xs:element name="timestamp" type="xs:dateTime" minOccurs="0">
-        <xs:annotation>
-          <xs:documentation>The date and time (timestamp) when the document was created.</xs:documentation>
-        </xs:annotation>
-      </xs:element>
-      <xs:element name="tool" minOccurs="0" type="bd:toolType">
-        <xs:annotation>
-          <xs:documentation>The tool used to create the BOM.</xs:documentation>
-        </xs:annotation>
-      </xs:element>
-      <xs:element name="authors" minOccurs="0" maxOccurs="1">
-        <xs:annotation>
-          <xs:documentation>The person(s) who created the BOM. Authors are common in BOMs created through
-            manual processes. BOMs created through automated means may not have authors.</xs:documentation>
-        </xs:annotation>
-        <xs:complexType>
-          <xs:sequence minOccurs="0" maxOccurs="unbounded">
-            <xs:element name="author" type="bd:organizationalPerson"/>
-          </xs:sequence>
-        </xs:complexType>
-      </xs:element>
-      <xs:element name="component" type="bom:component" minOccurs="0">
-        <xs:annotation>
-          <xs:documentation>The component that the BOM describes.</xs:documentation>
-        </xs:annotation>
-      </xs:element>
-      <xs:element name="manufacture" type="bd:organizationalEntity" minOccurs="0" maxOccurs="unbounded">
-        <xs:annotation>
-          <xs:documentation>The organization that manufactured the component that the BOM describes.</xs:documentation>
-        </xs:annotation>
-      </xs:element>
-      <xs:element name="supplier" type="bd:organizationalEntity" minOccurs="0" maxOccurs="unbounded">
-        <xs:annotation>
-          <xs:documentation>The organization that supplied the component that the BOM describes. The
-            supplier may often be the manufacture, but may also be a distributor or repackager.</xs:documentation>
-        </xs:annotation>
-      </xs:element>
-    </xs:sequence>
-    <xs:anyAttribute namespace="##other" processContents="lax">
-      <xs:annotation>
-        <xs:documentation>User-defined attributes may be used on this element as long as they
-          do not have the same name as an existing attribute used by the schema.</xs:documentation>
-      </xs:annotation>
-    </xs:anyAttribute>
-  </xs:complexType>
-
-  <xs:complexType name="organizationalEntity">
-    <xs:sequence minOccurs="0" maxOccurs="1">
-      <xs:element name="name" type="xs:normalizedString" minOccurs="0" maxOccurs="1">
-        <xs:annotation>
-          <xs:documentation>The name of the organization</xs:documentation>
-        </xs:annotation>
-      </xs:element>
-      <xs:element name="url" type="xs:anyURI" minOccurs="0" maxOccurs="unbounded">
-        <xs:annotation>
-          <xs:documentation>The URL of the organization. Multiple URLs are allowed.</xs:documentation>
-        </xs:annotation>
-      </xs:element>
-      <xs:element name="contact" type="bd:organizationalPerson" minOccurs="0" maxOccurs="unbounded">
-        <xs:annotation>
-          <xs:documentation>A contact person at the organization. Multiple contacts are allowed.</xs:documentation>
-        </xs:annotation>
-      </xs:element>
-    </xs:sequence>
-    <xs:anyAttribute namespace="##other" processContents="lax">
-      <xs:annotation>
-        <xs:documentation>User-defined attributes may be used on this element as long as they
-          do not have the same name as an existing attribute used by the schema.</xs:documentation>
-      </xs:annotation>
-    </xs:anyAttribute>
-  </xs:complexType>
-
-  <xs:complexType name="toolType">
-    <xs:annotation>
-      <xs:documentation>Specifies a tool (manual or automated).</xs:documentation>
-    </xs:annotation>
-    <xs:sequence minOccurs="0" maxOccurs="1">
-      <xs:element name="vendor" minOccurs="0" maxOccurs="1" type="xs:normalizedString">
-        <xs:annotation>
-          <xs:documentation>The vendor of the tool used to create the BOM.</xs:documentation>
-        </xs:annotation>
-      </xs:element>
-      <xs:element name="name" minOccurs="0" maxOccurs="1" type="xs:normalizedString">
-        <xs:annotation>
-          <xs:documentation>The name of the tool used to create the BOM.</xs:documentation>
-        </xs:annotation>
-      </xs:element>
-      <xs:element name="version" minOccurs="0" maxOccurs="1" type="xs:normalizedString">
-        <xs:annotation>
-          <xs:documentation>The version of the tool used to create the BOM.</xs:documentation>
-        </xs:annotation>
-      </xs:element>
-      <xs:element name="hashes" minOccurs="0" maxOccurs="1">
-        <xs:complexType>
-          <xs:sequence minOccurs="0" maxOccurs="unbounded">
-            <xs:element name="hash" type="bom:hashType"/>
-          </xs:sequence>
-        </xs:complexType>
-      </xs:element>
-    </xs:sequence>
-    <xs:anyAttribute namespace="##other" processContents="lax">
-      <xs:annotation>
-        <xs:documentation>User-defined attributes may be used on this element as long as they
-          do not have the same name as an existing attribute used by the schema.</xs:documentation>
-      </xs:annotation>
-    </xs:anyAttribute>
-  </xs:complexType>
-
-  <xs:complexType name="organizationalPerson">
-    <xs:sequence minOccurs="0" maxOccurs="1">
-      <xs:element name="name" type="xs:normalizedString" minOccurs="0" maxOccurs="1">
-        <xs:annotation>
-          <xs:documentation>The name of the person</xs:documentation>
-        </xs:annotation>
-      </xs:element>
-      <xs:element name="email" type="xs:normalizedString" minOccurs="0" maxOccurs="unbounded">
-        <xs:annotation>
-          <xs:documentation>The email address of the person. Multiple email addresses are allowed.</xs:documentation>
-        </xs:annotation>
-      </xs:element>
-      <xs:element name="phone" type="xs:normalizedString" minOccurs="0" maxOccurs="unbounded">
-        <xs:annotation>
-          <xs:documentation>The phone number of the person. Multiple phone numbers are allowed.</xs:documentation>
-        </xs:annotation>
-      </xs:element>
-    </xs:sequence>
-    <xs:anyAttribute namespace="##other" processContents="lax">
-      <xs:annotation>
-        <xs:documentation>User-defined attributes may be used on this element as long as they
-          do not have the same name as an existing attribute used by the schema.</xs:documentation>
-      </xs:annotation>
-    </xs:anyAttribute>
-  </xs:complexType>
-
-  <xs:element name="metadata" type="bd:metadata">
-    <xs:annotation>
-      <xs:documentation>Provides additional information about a BOM.</xs:documentation>
-    </xs:annotation>
-  </xs:element>
-
-</xs:schema>

schema/json/.gitignore

@@ -1 +0,0 @@
-examples/

schema/json/README.md

@@ -0,0 +1,39 @@
+# JSON Schema
+
+This is the JSON schema for output from the JSON presenter (`syft <img> -o json`). The required inputs for defining the JSON schema are as follows:
+
+- the value of `internal.JSONSchemaVersion` that governs the schema filename
+- the `Document` struct definition within `syft/presenters/json/document.go` that governs the overall document shape
+- the `metadataContainer` struct definition within `schema/json/generate.go` that governs the allowable shapes of `pkg.Package.Metadata`
+
+With regard to testing the JSON schema, integration test cases provided by the developer are used as examples to validate that JSON output from Syft is always valid relative to the `schema/json/schema-$VERSION.json` file.
+
+## Versioning
+
+Versioning the JSON schema must be done manually by changing the `JSONSchemaVersion` constant within `internal/constants.go`.
+
+This schema is being versioned based off of the "SchemaVer" guidelines, which slightly diverges from Semantic Versioning to tailor for the purposes of data models. 
+
+Given a version number format `MODEL.REVISION.ADDITION`:
+
+- `MODEL`: increment when you make a breaking schema change which will prevent interaction with any historical data
+- `REVISION`: increment when you make a schema change which may prevent interaction with some historical data
+- `ADDITION`: increment when you make a schema change that is compatible with all historical data
+
+## Adding a New `pkg.*Metadata` Type
+
+When adding a new `pkg.*Metadata` that is assigned to the `pkg.Package.Metadata` struct field it is important that a few things
+are done:
+
+- a new integration test case is added to `test/integration/pkg_cases_test.go` that exercises the new package type with the new metadata
+- the new metadata struct is added to the `metadataContainer` struct within `schema/json/generate.go`
+
+## Generating a New Schema
+
+Create the new schema by running `cd schema/json && go run generate.go` (note you must be in the `schema/json` dir while running this):
+
+- If there is **not** an existing schema for the given version, then the new schema file will be written to `schema/json/schema-$VERSION.json`
+- If there is an existing schema for the given version and the new schema matches the existing schema, no action is taken
+- If there is an existing schema for the given version and the new schema **does not** match the existing schema, an error is shown indicating to increment the version appropriately (see the "Versioning" section)
+
+***Note: never delete a JSON schema and never change an existing JSON schema once it has been published in a release!*** Only add new schemas with a newly incremented version. All previous schema files must be stored in the `schema/json/` directory.

schema/json/generate.go

@@ -0,0 +1,125 @@
+package main
+
+import (
+	"bytes"
+	"encoding/json"
+	"fmt"
+	"io/ioutil"
+	"os"
+	"sort"
+	"strings"
+
+	"github.com/alecthomas/jsonschema"
+	"github.com/anchore/syft/internal"
+	"github.com/anchore/syft/syft/pkg"
+	jsonPresenter "github.com/anchore/syft/syft/presenter/json"
+)
+
+/*
+This method of creating the JSON schema only captures strongly typed fields for the purpose of integrations between syft
+JSON output and integrations. The downside to this approach is that any values and types used on weakly typed fields
+are not captured (empty interfaces). This means that pkg.Package.Metadata is not validated at this time. This approach
+can be extended to include specific package metadata struct shapes in the future.
+*/
+
+// This should represent all possible metadatas represented in the pkg.Package.Metadata field (an interface{}).
+// When a new package metadata definition is created it will need to be manually added here. The variable name does
+// not matter as long as it is exported.
+type metadataContainer struct {
+	Apk    pkg.ApkMetadata
+	Dpkg   pkg.DpkgMetadata
+	Gem    pkg.GemMetadata
+	Java   pkg.JavaMetadata
+	Npm    pkg.NpmPackageJSONMetadata
+	Python pkg.PythonPackageMetadata
+	Rpm    pkg.RpmdbMetadata
+}
+
+// nolint:funlen
+func main() {
+	metadataSchema := jsonschema.Reflect(&metadataContainer{})
+	documentSchema := jsonschema.Reflect(&jsonPresenter.Document{})
+
+	// TODO: inject source definitions
+
+	// inject the definitions of all metadatas into the schema definitions
+
+	var metadataNames []string
+	for name, definition := range metadataSchema.Definitions {
+		if name == "metadataContainer" {
+			// ignore the definition for the fake container
+			continue
+		}
+		documentSchema.Definitions[name] = definition
+		if strings.HasSuffix(name, "Metadata") {
+			metadataNames = append(metadataNames, name)
+		}
+	}
+
+	// ensure the generated list of names is stable between runs
+	sort.Strings(metadataNames)
+
+	var metadataTypes = []map[string]string{
+		// allow for no metadata to be provided
+		{"type": "null"},
+	}
+	for _, name := range metadataNames {
+		metadataTypes = append(metadataTypes, map[string]string{
+			"$ref": fmt.Sprintf("#/definitions/%s", name),
+		})
+	}
+
+	// set the "anyOf" field for Package.Metadata to be a conjunction of several types
+	documentSchema.Definitions["Package"].Properties.Set("metadata", map[string][]map[string]string{
+		"anyOf": metadataTypes,
+	})
+
+	filename := fmt.Sprintf("schema-%s.json", internal.JSONSchemaVersion)
+
+	var newSchemaBuffer = new(bytes.Buffer)
+	enc := json.NewEncoder(newSchemaBuffer)
+	// prevent > and < from being escaped in the payload
+	enc.SetEscapeHTML(false)
+	enc.SetIndent("", "  ")
+	err := enc.Encode(&documentSchema)
+	if err != nil {
+		panic(err)
+	}
+
+	if _, err := os.Stat(filename); !os.IsNotExist(err) {
+		// check if the schema is the same...
+		existingFh, err := os.Open(filename)
+		if err != nil {
+			panic(err)
+		}
+
+		existingSchemaBytes, err := ioutil.ReadAll(existingFh)
+		if err != nil {
+			panic(err)
+		}
+
+		if bytes.Equal(existingSchemaBytes, newSchemaBuffer.Bytes()) {
+			// the generated schema is the same, bail with no error :)
+			fmt.Println("No change to the existing schema!")
+			os.Exit(0)
+		}
+
+		// the generated schema is different, bail with error :(
+		fmt.Printf("Cowardly refusing to overwrite existing schema (%s)!\nSee the scheam/json/README.md for how to increment\n", filename)
+		os.Exit(1)
+	}
+
+	fh, err := os.Create(filename)
+	if err != nil {
+		panic(err)
+	}
+
+	_, err = fh.Write(newSchemaBuffer.Bytes())
+	if err != nil {
+		panic(err)
+	}
+
+	defer fh.Close()
+
+	fmt.Printf("wrote new schema to %q\n", filename)
+}

schema/json/generate.py

@@ -1,30 +0,0 @@
-#!/usr/env/bin python3
-import os
-import glob
-import json
-
-from genson import SchemaBuilder
-
-EXAMPLES_DIR = "examples/"
-OUTPUT = "schema.json"
-
-
-def main():
-    builder = SchemaBuilder()
-
-    print("Generating new Syft json schema...")
-    for filepath in glob.glob(os.path.join(EXAMPLES_DIR, '*.json')):
-        with open(filepath, 'r') as f:
-            print(f"  adding {filepath}")
-            builder.add_object(json.loads(f.read()))
-
-    print("Building schema...")
-    new_schema = builder.to_schema()
-    with open(OUTPUT, 'w') as f:
-        f.write(json.dumps(new_schema, sort_keys=True, indent=4))
-
-    print(f"New schema written to '{OUTPUT}'")
-
-
-if __name__ == "__main__":
-    main()

schema/json/requirements.txt

@@ -1 +0,0 @@
-genson

schema/json/schema-1.0.0.json

@@ -0,0 +1,678 @@
+{
+  "$schema": "http://json-schema.org/draft-04/schema#",
+  "$ref": "#/definitions/Document",
+  "definitions": {
+    "ApkFileRecord": {
+      "required": [
+        "path"
+      ],
+      "properties": {
+        "path": {
+          "type": "string"
+        },
+        "ownerUid": {
+          "type": "string"
+        },
+        "ownerGid": {
+          "type": "string"
+        },
+        "permissions": {
+          "type": "string"
+        },
+        "checksum": {
+          "type": "string"
+        }
+      },
+      "additionalProperties": false,
+      "type": "object"
+    },
+    "ApkMetadata": {
+      "required": [
+        "package",
+        "originPackage",
+        "maintainer",
+        "version",
+        "license",
+        "architecture",
+        "url",
+        "description",
+        "size",
+        "installedSize",
+        "pullDependencies",
+        "pullChecksum",
+        "gitCommitOfApkPort",
+        "files"
+      ],
+      "properties": {
+        "package": {
+          "type": "string"
+        },
+        "originPackage": {
+          "type": "string"
+        },
+        "maintainer": {
+          "type": "string"
+        },
+        "version": {
+          "type": "string"
+        },
+        "license": {
+          "type": "string"
+        },
+        "architecture": {
+          "type": "string"
+        },
+        "url": {
+          "type": "string"
+        },
+        "description": {
+          "type": "string"
+        },
+        "size": {
+          "type": "integer"
+        },
+        "installedSize": {
+          "type": "integer"
+        },
+        "pullDependencies": {
+          "type": "string"
+        },
+        "pullChecksum": {
+          "type": "string"
+        },
+        "gitCommitOfApkPort": {
+          "type": "string"
+        },
+        "files": {
+          "items": {
+            "$schema": "http://json-schema.org/draft-04/schema#",
+            "$ref": "#/definitions/ApkFileRecord"
+          },
+          "type": "array"
+        }
+      },
+      "additionalProperties": false,
+      "type": "object"
+    },
+    "Descriptor": {
+      "required": [
+        "name",
+        "version"
+      ],
+      "properties": {
+        "name": {
+          "type": "string"
+        },
+        "version": {
+          "type": "string"
+        }
+      },
+      "additionalProperties": false,
+      "type": "object"
+    },
+    "Distribution": {
+      "required": [
+        "name",
+        "version",
+        "idLike"
+      ],
+      "properties": {
+        "name": {
+          "type": "string"
+        },
+        "version": {
+          "type": "string"
+        },
+        "idLike": {
+          "type": "string"
+        }
+      },
+      "additionalProperties": false,
+      "type": "object"
+    },
+    "Document": {
+      "required": [
+        "artifacts",
+        "source",
+        "distro",
+        "descriptor",
+        "schema"
+      ],
+      "properties": {
+        "artifacts": {
+          "items": {
+            "$schema": "http://json-schema.org/draft-04/schema#",
+            "$ref": "#/definitions/Package"
+          },
+          "type": "array"
+        },
+        "source": {
+          "$schema": "http://json-schema.org/draft-04/schema#",
+          "$ref": "#/definitions/Source"
+        },
+        "distro": {
+          "$schema": "http://json-schema.org/draft-04/schema#",
+          "$ref": "#/definitions/Distribution"
+        },
+        "descriptor": {
+          "$schema": "http://json-schema.org/draft-04/schema#",
+          "$ref": "#/definitions/Descriptor"
+        },
+        "schema": {
+          "$schema": "http://json-schema.org/draft-04/schema#",
+          "$ref": "#/definitions/Schema"
+        }
+      },
+      "additionalProperties": false,
+      "type": "object"
+    },
+    "DpkgFileRecord": {
+      "required": [
+        "path",
+        "md5"
+      ],
+      "properties": {
+        "path": {
+          "type": "string"
+        },
+        "md5": {
+          "type": "string"
+        }
+      },
+      "additionalProperties": false,
+      "type": "object"
+    },
+    "DpkgMetadata": {
+      "required": [
+        "package",
+        "source",
+        "version",
+        "architecture",
+        "maintainer",
+        "installedSize",
+        "files"
+      ],
+      "properties": {
+        "package": {
+          "type": "string"
+        },
+        "source": {
+          "type": "string"
+        },
+        "version": {
+          "type": "string"
+        },
+        "architecture": {
+          "type": "string"
+        },
+        "maintainer": {
+          "type": "string"
+        },
+        "installedSize": {
+          "type": "integer"
+        },
+        "files": {
+          "items": {
+            "$schema": "http://json-schema.org/draft-04/schema#",
+            "$ref": "#/definitions/DpkgFileRecord"
+          },
+          "type": "array"
+        }
+      },
+      "additionalProperties": false,
+      "type": "object"
+    },
+    "GemMetadata": {
+      "required": [
+        "name",
+        "version"
+      ],
+      "properties": {
+        "name": {
+          "type": "string"
+        },
+        "version": {
+          "type": "string"
+        },
+        "files": {
+          "items": {
+            "type": "string"
+          },
+          "type": "array"
+        },
+        "authors": {
+          "items": {
+            "type": "string"
+          },
+          "type": "array"
+        },
+        "licenses": {
+          "items": {
+            "type": "string"
+          },
+          "type": "array"
+        },
+        "homepage": {
+          "type": "string"
+        }
+      },
+      "additionalProperties": false,
+      "type": "object"
+    },
+    "JavaManifest": {
+      "properties": {
+        "main": {
+          "patternProperties": {
+            ".*": {
+              "type": "string"
+            }
+          },
+          "type": "object"
+        },
+        "namedSections": {
+          "patternProperties": {
+            ".*": {
+              "patternProperties": {
+                ".*": {
+                  "type": "string"
+                }
+              },
+              "type": "object"
+            }
+          },
+          "type": "object"
+        }
+      },
+      "additionalProperties": false,
+      "type": "object"
+    },
+    "JavaMetadata": {
+      "required": [
+        "virtualPath"
+      ],
+      "properties": {
+        "virtualPath": {
+          "type": "string"
+        },
+        "manifest": {
+          "$schema": "http://json-schema.org/draft-04/schema#",
+          "$ref": "#/definitions/JavaManifest"
+        },
+        "pomProperties": {
+          "$schema": "http://json-schema.org/draft-04/schema#",
+          "$ref": "#/definitions/PomProperties"
+        }
+      },
+      "additionalProperties": false,
+      "type": "object"
+    },
+    "Location": {
+      "required": [
+        "path"
+      ],
+      "properties": {
+        "path": {
+          "type": "string"
+        },
+        "layerID": {
+          "type": "string"
+        }
+      },
+      "additionalProperties": false,
+      "type": "object"
+    },
+    "NpmPackageJSONMetadata": {
+      "required": [
+        "author",
+        "licenses",
+        "homepage",
+        "description",
+        "url"
+      ],
+      "properties": {
+        "files": {
+          "items": {
+            "type": "string"
+          },
+          "type": "array"
+        },
+        "author": {
+          "type": "string"
+        },
+        "licenses": {
+          "items": {
+            "type": "string"
+          },
+          "type": "array"
+        },
+        "homepage": {
+          "type": "string"
+        },
+        "description": {
+          "type": "string"
+        },
+        "url": {
+          "type": "string"
+        }
+      },
+      "additionalProperties": false,
+      "type": "object"
+    },
+    "Package": {
+      "required": [
+        "name",
+        "version",
+        "type",
+        "foundBy",
+        "locations",
+        "licenses",
+        "language",
+        "cpes",
+        "purl",
+        "metadataType",
+        "metadata"
+      ],
+      "properties": {
+        "name": {
+          "type": "string"
+        },
+        "version": {
+          "type": "string"
+        },
+        "type": {
+          "type": "string"
+        },
+        "foundBy": {
+          "type": "string"
+        },
+        "locations": {
+          "items": {
+            "$schema": "http://json-schema.org/draft-04/schema#",
+            "$ref": "#/definitions/Location"
+          },
+          "type": "array"
+        },
+        "licenses": {
+          "items": {
+            "type": "string"
+          },
+          "type": "array"
+        },
+        "language": {
+          "type": "string"
+        },
+        "cpes": {
+          "items": {
+            "type": "string"
+          },
+          "type": "array"
+        },
+        "purl": {
+          "type": "string"
+        },
+        "metadataType": {
+          "type": "string"
+        },
+        "metadata": {
+          "anyOf": [
+            {
+              "type": "null"
+            },
+            {
+              "$ref": "#/definitions/ApkMetadata"
+            },
+            {
+              "$ref": "#/definitions/DpkgMetadata"
+            },
+            {
+              "$ref": "#/definitions/GemMetadata"
+            },
+            {
+              "$ref": "#/definitions/JavaMetadata"
+            },
+            {
+              "$ref": "#/definitions/NpmPackageJSONMetadata"
+            },
+            {
+              "$ref": "#/definitions/PythonPackageMetadata"
+            },
+            {
+              "$ref": "#/definitions/RpmdbMetadata"
+            }
+          ]
+        }
+      },
+      "additionalProperties": false,
+      "type": "object"
+    },
+    "PomProperties": {
+      "required": [
+        "path",
+        "name",
+        "groupId",
+        "artifactId",
+        "version",
+        "extraFields"
+      ],
+      "properties": {
+        "path": {
+          "type": "string"
+        },
+        "name": {
+          "type": "string"
+        },
+        "groupId": {
+          "type": "string"
+        },
+        "artifactId": {
+          "type": "string"
+        },
+        "version": {
+          "type": "string"
+        },
+        "extraFields": {
+          "patternProperties": {
+            ".*": {
+              "type": "string"
+            }
+          },
+          "type": "object"
+        }
+      },
+      "additionalProperties": false,
+      "type": "object"
+    },
+    "PythonFileDigest": {
+      "required": [
+        "algorithm",
+        "value"
+      ],
+      "properties": {
+        "algorithm": {
+          "type": "string"
+        },
+        "value": {
+          "type": "string"
+        }
+      },
+      "additionalProperties": false,
+      "type": "object"
+    },
+    "PythonFileRecord": {
+      "required": [
+        "path"
+      ],
+      "properties": {
+        "path": {
+          "type": "string"
+        },
+        "digest": {
+          "$schema": "http://json-schema.org/draft-04/schema#",
+          "$ref": "#/definitions/PythonFileDigest"
+        },
+        "size": {
+          "type": "string"
+        }
+      },
+      "additionalProperties": false,
+      "type": "object"
+    },
+    "PythonPackageMetadata": {
+      "required": [
+        "name",
+        "version",
+        "license",
+        "author",
+        "authorEmail",
+        "platform",
+        "sitePackagesRootPath"
+      ],
+      "properties": {
+        "name": {
+          "type": "string"
+        },
+        "version": {
+          "type": "string"
+        },
+        "license": {
+          "type": "string"
+        },
+        "author": {
+          "type": "string"
+        },
+        "authorEmail": {
+          "type": "string"
+        },
+        "platform": {
+          "type": "string"
+        },
+        "files": {
+          "items": {
+            "$schema": "http://json-schema.org/draft-04/schema#",
+            "$ref": "#/definitions/PythonFileRecord"
+          },
+          "type": "array"
+        },
+        "sitePackagesRootPath": {
+          "type": "string"
+        },
+        "topLevelPackages": {
+          "items": {
+            "type": "string"
+          },
+          "type": "array"
+        }
+      },
+      "additionalProperties": false,
+      "type": "object"
+    },
+    "RpmdbFileRecord": {
+      "required": [
+        "path",
+        "mode",
+        "size",
+        "sha256"
+      ],
+      "properties": {
+        "path": {
+          "type": "string"
+        },
+        "mode": {
+          "type": "integer"
+        },
+        "size": {
+          "type": "integer"
+        },
+        "sha256": {
+          "type": "string"
+        }
+      },
+      "additionalProperties": false,
+      "type": "object"
+    },
+    "RpmdbMetadata": {
+      "required": [
+        "name",
+        "version",
+        "epoch",
+        "architecture",
+        "release",
+        "sourceRpm",
+        "size",
+        "license",
+        "vendor",
+        "files"
+      ],
+      "properties": {
+        "name": {
+          "type": "string"
+        },
+        "version": {
+          "type": "string"
+        },
+        "epoch": {
+          "type": "integer"
+        },
+        "architecture": {
+          "type": "string"
+        },
+        "release": {
+          "type": "string"
+        },
+        "sourceRpm": {
+          "type": "string"
+        },
+        "size": {
+          "type": "integer"
+        },
+        "license": {
+          "type": "string"
+        },
+        "vendor": {
+          "type": "string"
+        },
+        "files": {
+          "items": {
+            "$schema": "http://json-schema.org/draft-04/schema#",
+            "$ref": "#/definitions/RpmdbFileRecord"
+          },
+          "type": "array"
+        }
+      },
+      "additionalProperties": false,
+      "type": "object"
+    },
+    "Schema": {
+      "required": [
+        "version",
+        "url"
+      ],
+      "properties": {
+        "version": {
+          "type": "string"
+        },
+        "url": {
+          "type": "string"
+        }
+      },
+      "additionalProperties": false,
+      "type": "object"
+    },
+    "Source": {
+      "required": [
+        "type",
+        "target"
+      ],
+      "properties": {
+        "type": {
+          "type": "string"
+        },
+        "target": {
+          "additionalProperties": true
+        }
+      },
+      "additionalProperties": false,
+      "type": "object"
+    }
+  }
+}

schema/json/schema-1.0.1.json

@@ -0,0 +1,682 @@
+{
+  "$schema": "http://json-schema.org/draft-04/schema#",
+  "$ref": "#/definitions/Document",
+  "definitions": {
+    "ApkFileRecord": {
+      "required": [
+        "path"
+      ],
+      "properties": {
+        "path": {
+          "type": "string"
+        },
+        "ownerUid": {
+          "type": "string"
+        },
+        "ownerGid": {
+          "type": "string"
+        },
+        "permissions": {
+          "type": "string"
+        },
+        "checksum": {
+          "type": "string"
+        }
+      },
+      "additionalProperties": false,
+      "type": "object"
+    },
+    "ApkMetadata": {
+      "required": [
+        "package",
+        "originPackage",
+        "maintainer",
+        "version",
+        "license",
+        "architecture",
+        "url",
+        "description",
+        "size",
+        "installedSize",
+        "pullDependencies",
+        "pullChecksum",
+        "gitCommitOfApkPort",
+        "files"
+      ],
+      "properties": {
+        "package": {
+          "type": "string"
+        },
+        "originPackage": {
+          "type": "string"
+        },
+        "maintainer": {
+          "type": "string"
+        },
+        "version": {
+          "type": "string"
+        },
+        "license": {
+          "type": "string"
+        },
+        "architecture": {
+          "type": "string"
+        },
+        "url": {
+          "type": "string"
+        },
+        "description": {
+          "type": "string"
+        },
+        "size": {
+          "type": "integer"
+        },
+        "installedSize": {
+          "type": "integer"
+        },
+        "pullDependencies": {
+          "type": "string"
+        },
+        "pullChecksum": {
+          "type": "string"
+        },
+        "gitCommitOfApkPort": {
+          "type": "string"
+        },
+        "files": {
+          "items": {
+            "$schema": "http://json-schema.org/draft-04/schema#",
+            "$ref": "#/definitions/ApkFileRecord"
+          },
+          "type": "array"
+        }
+      },
+      "additionalProperties": false,
+      "type": "object"
+    },
+    "Descriptor": {
+      "required": [
+        "name",
+        "version"
+      ],
+      "properties": {
+        "name": {
+          "type": "string"
+        },
+        "version": {
+          "type": "string"
+        }
+      },
+      "additionalProperties": false,
+      "type": "object"
+    },
+    "Distribution": {
+      "required": [
+        "name",
+        "version",
+        "idLike"
+      ],
+      "properties": {
+        "name": {
+          "type": "string"
+        },
+        "version": {
+          "type": "string"
+        },
+        "idLike": {
+          "type": "string"
+        }
+      },
+      "additionalProperties": false,
+      "type": "object"
+    },
+    "Document": {
+      "required": [
+        "artifacts",
+        "source",
+        "distro",
+        "descriptor",
+        "schema"
+      ],
+      "properties": {
+        "artifacts": {
+          "items": {
+            "$schema": "http://json-schema.org/draft-04/schema#",
+            "$ref": "#/definitions/Package"
+          },
+          "type": "array"
+        },
+        "source": {
+          "$schema": "http://json-schema.org/draft-04/schema#",
+          "$ref": "#/definitions/Source"
+        },
+        "distro": {
+          "$schema": "http://json-schema.org/draft-04/schema#",
+          "$ref": "#/definitions/Distribution"
+        },
+        "descriptor": {
+          "$schema": "http://json-schema.org/draft-04/schema#",
+          "$ref": "#/definitions/Descriptor"
+        },
+        "schema": {
+          "$schema": "http://json-schema.org/draft-04/schema#",
+          "$ref": "#/definitions/Schema"
+        }
+      },
+      "additionalProperties": false,
+      "type": "object"
+    },
+    "DpkgFileRecord": {
+      "required": [
+        "path",
+        "md5"
+      ],
+      "properties": {
+        "path": {
+          "type": "string"
+        },
+        "md5": {
+          "type": "string"
+        }
+      },
+      "additionalProperties": false,
+      "type": "object"
+    },
+    "DpkgMetadata": {
+      "required": [
+        "package",
+        "source",
+        "version",
+        "sourceVersion",
+        "architecture",
+        "maintainer",
+        "installedSize",
+        "files"
+      ],
+      "properties": {
+        "package": {
+          "type": "string"
+        },
+        "source": {
+          "type": "string"
+        },
+        "version": {
+          "type": "string"
+        },
+        "sourceVersion": {
+          "type": "string"
+        },
+        "architecture": {
+          "type": "string"
+        },
+        "maintainer": {
+          "type": "string"
+        },
+        "installedSize": {
+          "type": "integer"
+        },
+        "files": {
+          "items": {
+            "$schema": "http://json-schema.org/draft-04/schema#",
+            "$ref": "#/definitions/DpkgFileRecord"
+          },
+          "type": "array"
+        }
+      },
+      "additionalProperties": false,
+      "type": "object"
+    },
+    "GemMetadata": {
+      "required": [
+        "name",
+        "version"
+      ],
+      "properties": {
+        "name": {
+          "type": "string"
+        },
+        "version": {
+          "type": "string"
+        },
+        "files": {
+          "items": {
+            "type": "string"
+          },
+          "type": "array"
+        },
+        "authors": {
+          "items": {
+            "type": "string"
+          },
+          "type": "array"
+        },
+        "licenses": {
+          "items": {
+            "type": "string"
+          },
+          "type": "array"
+        },
+        "homepage": {
+          "type": "string"
+        }
+      },
+      "additionalProperties": false,
+      "type": "object"
+    },
+    "JavaManifest": {
+      "properties": {
+        "main": {
+          "patternProperties": {
+            ".*": {
+              "type": "string"
+            }
+          },
+          "type": "object"
+        },
+        "namedSections": {
+          "patternProperties": {
+            ".*": {
+              "patternProperties": {
+                ".*": {
+                  "type": "string"
+                }
+              },
+              "type": "object"
+            }
+          },
+          "type": "object"
+        }
+      },
+      "additionalProperties": false,
+      "type": "object"
+    },
+    "JavaMetadata": {
+      "required": [
+        "virtualPath"
+      ],
+      "properties": {
+        "virtualPath": {
+          "type": "string"
+        },
+        "manifest": {
+          "$schema": "http://json-schema.org/draft-04/schema#",
+          "$ref": "#/definitions/JavaManifest"
+        },
+        "pomProperties": {
+          "$schema": "http://json-schema.org/draft-04/schema#",
+          "$ref": "#/definitions/PomProperties"
+        }
+      },
+      "additionalProperties": false,
+      "type": "object"
+    },
+    "Location": {
+      "required": [
+        "path"
+      ],
+      "properties": {
+        "path": {
+          "type": "string"
+        },
+        "layerID": {
+          "type": "string"
+        }
+      },
+      "additionalProperties": false,
+      "type": "object"
+    },
+    "NpmPackageJSONMetadata": {
+      "required": [
+        "author",
+        "licenses",
+        "homepage",
+        "description",
+        "url"
+      ],
+      "properties": {
+        "files": {
+          "items": {
+            "type": "string"
+          },
+          "type": "array"
+        },
+        "author": {
+          "type": "string"
+        },
+        "licenses": {
+          "items": {
+            "type": "string"
+          },
+          "type": "array"
+        },
+        "homepage": {
+          "type": "string"
+        },
+        "description": {
+          "type": "string"
+        },
+        "url": {
+          "type": "string"
+        }
+      },
+      "additionalProperties": false,
+      "type": "object"
+    },
+    "Package": {
+      "required": [
+        "name",
+        "version",
+        "type",
+        "foundBy",
+        "locations",
+        "licenses",
+        "language",
+        "cpes",
+        "purl",
+        "metadataType",
+        "metadata"
+      ],
+      "properties": {
+        "name": {
+          "type": "string"
+        },
+        "version": {
+          "type": "string"
+        },
+        "type": {
+          "type": "string"
+        },
+        "foundBy": {
+          "type": "string"
+        },
+        "locations": {
+          "items": {
+            "$schema": "http://json-schema.org/draft-04/schema#",
+            "$ref": "#/definitions/Location"
+          },
+          "type": "array"
+        },
+        "licenses": {
+          "items": {
+            "type": "string"
+          },
+          "type": "array"
+        },
+        "language": {
+          "type": "string"
+        },
+        "cpes": {
+          "items": {
+            "type": "string"
+          },
+          "type": "array"
+        },
+        "purl": {
+          "type": "string"
+        },
+        "metadataType": {
+          "type": "string"
+        },
+        "metadata": {
+          "anyOf": [
+            {
+              "type": "null"
+            },
+            {
+              "$ref": "#/definitions/ApkMetadata"
+            },
+            {
+              "$ref": "#/definitions/DpkgMetadata"
+            },
+            {
+              "$ref": "#/definitions/GemMetadata"
+            },
+            {
+              "$ref": "#/definitions/JavaMetadata"
+            },
+            {
+              "$ref": "#/definitions/NpmPackageJSONMetadata"
+            },
+            {
+              "$ref": "#/definitions/PythonPackageMetadata"
+            },
+            {
+              "$ref": "#/definitions/RpmdbMetadata"
+            }
+          ]
+        }
+      },
+      "additionalProperties": false,
+      "type": "object"
+    },
+    "PomProperties": {
+      "required": [
+        "path",
+        "name",
+        "groupId",
+        "artifactId",
+        "version",
+        "extraFields"
+      ],
+      "properties": {
+        "path": {
+          "type": "string"
+        },
+        "name": {
+          "type": "string"
+        },
+        "groupId": {
+          "type": "string"
+        },
+        "artifactId": {
+          "type": "string"
+        },
+        "version": {
+          "type": "string"
+        },
+        "extraFields": {
+          "patternProperties": {
+            ".*": {
+              "type": "string"
+            }
+          },
+          "type": "object"
+        }
+      },
+      "additionalProperties": false,
+      "type": "object"
+    },
+    "PythonFileDigest": {
+      "required": [
+        "algorithm",
+        "value"
+      ],
+      "properties": {
+        "algorithm": {
+          "type": "string"
+        },
+        "value": {
+          "type": "string"
+        }
+      },
+      "additionalProperties": false,
+      "type": "object"
+    },
+    "PythonFileRecord": {
+      "required": [
+        "path"
+      ],
+      "properties": {
+        "path": {
+          "type": "string"
+        },
+        "digest": {
+          "$schema": "http://json-schema.org/draft-04/schema#",
+          "$ref": "#/definitions/PythonFileDigest"
+        },
+        "size": {
+          "type": "string"
+        }
+      },
+      "additionalProperties": false,
+      "type": "object"
+    },
+    "PythonPackageMetadata": {
+      "required": [
+        "name",
+        "version",
+        "license",
+        "author",
+        "authorEmail",
+        "platform",
+        "sitePackagesRootPath"
+      ],
+      "properties": {
+        "name": {
+          "type": "string"
+        },
+        "version": {
+          "type": "string"
+        },
+        "license": {
+          "type": "string"
+        },
+        "author": {
+          "type": "string"
+        },
+        "authorEmail": {
+          "type": "string"
+        },
+        "platform": {
+          "type": "string"
+        },
+        "files": {
+          "items": {
+            "$schema": "http://json-schema.org/draft-04/schema#",
+            "$ref": "#/definitions/PythonFileRecord"
+          },
+          "type": "array"
+        },
+        "sitePackagesRootPath": {
+          "type": "string"
+        },
+        "topLevelPackages": {
+          "items": {
+            "type": "string"
+          },
+          "type": "array"
+        }
+      },
+      "additionalProperties": false,
+      "type": "object"
+    },
+    "RpmdbFileRecord": {
+      "required": [
+        "path",
+        "mode",
+        "size",
+        "sha256"
+      ],
+      "properties": {
+        "path": {
+          "type": "string"
+        },
+        "mode": {
+          "type": "integer"
+        },
+        "size": {
+          "type": "integer"
+        },
+        "sha256": {
+          "type": "string"
+        }
+      },
+      "additionalProperties": false,
+      "type": "object"
+    },
+    "RpmdbMetadata": {
+      "required": [
+        "name",
+        "version",
+        "epoch",
+        "architecture",
+        "release",
+        "sourceRpm",
+        "size",
+        "license",
+        "vendor",
+        "files"
+      ],
+      "properties": {
+        "name": {
+          "type": "string"
+        },
+        "version": {
+          "type": "string"
+        },
+        "epoch": {
+          "type": "integer"
+        },
+        "architecture": {
+          "type": "string"
+        },
+        "release": {
+          "type": "string"
+        },
+        "sourceRpm": {
+          "type": "string"
+        },
+        "size": {
+          "type": "integer"
+        },
+        "license": {
+          "type": "string"
+        },
+        "vendor": {
+          "type": "string"
+        },
+        "files": {
+          "items": {
+            "$schema": "http://json-schema.org/draft-04/schema#",
+            "$ref": "#/definitions/RpmdbFileRecord"
+          },
+          "type": "array"
+        }
+      },
+      "additionalProperties": false,
+      "type": "object"
+    },
+    "Schema": {
+      "required": [
+        "version",
+        "url"
+      ],
+      "properties": {
+        "version": {
+          "type": "string"
+        },
+        "url": {
+          "type": "string"
+        }
+      },
+      "additionalProperties": false,
+      "type": "object"
+    },
+    "Source": {
+      "required": [
+        "type",
+        "target"
+      ],
+      "properties": {
+        "type": {
+          "type": "string"
+        },
+        "target": {
+          "additionalProperties": true
+        }
+      },
+      "additionalProperties": false,
+      "type": "object"
+    }
+  }
+}

schema/json/schema.json

@@ -1,576 +0,0 @@
-{
-    "$schema": "http://json-schema.org/schema#",
-    "properties": {
-        "artifacts": {
-            "items": {
-                "properties": {
-                    "foundBy": {
-                        "items": {
-                            "type": "string"
-                        },
-                        "type": "array"
-                    },
-                    "locations": {
-                        "items": {
-                            "anyOf": [
-                                {
-                                    "type": "string"
-                                },
-                                {
-                                    "properties": {
-                                        "layerIndex": {
-                                            "type": "integer"
-                                        },
-                                        "path": {
-                                            "type": "string"
-                                        }
-                                    },
-                                    "required": [
-                                        "layerIndex",
-                                        "path"
-                                    ],
-                                    "type": "object"
-                                }
-                            ]
-                        },
-                        "type": "array"
-                    },
-                    "metadata": {
-                        "properties": {
-                            "architecture": {
-                                "type": "string"
-                            },
-                            "authors": {
-                                "type": "null"
-                            },
-                            "description": {
-                                "type": "string"
-                            },
-                            "epoch": {
-                                "type": "integer"
-                            },
-                            "files": {
-                                "items": {
-                                    "anyOf": [
-                                        {
-                                            "type": "string"
-                                        },
-                                        {
-                                            "properties": {
-                                                "checksum": {
-                                                    "type": "string"
-                                                },
-                                                "ownerGid": {
-                                                    "type": "string"
-                                                },
-                                                "ownerUid": {
-                                                    "type": "string"
-                                                },
-                                                "path": {
-                                                    "type": "string"
-                                                },
-                                                "permissions": {
-                                                    "type": "string"
-                                                }
-                                            },
-                                            "required": [
-                                                "checksum",
-                                                "ownerGid",
-                                                "ownerUid",
-                                                "path",
-                                                "permissions"
-                                            ],
-                                            "type": "object"
-                                        }
-                                    ]
-                                },
-                                "type": "array"
-                            },
-                            "gitCommitOfApkPort": {
-                                "type": "string"
-                            },
-                            "installedSize": {
-                                "type": "integer"
-                            },
-                            "license": {
-                                "type": "string"
-                            },
-                            "licenses": {
-                                "items": {
-                                    "type": "string"
-                                },
-                                "type": "array"
-                            },
-                            "maintainer": {
-                                "type": "string"
-                            },
-                            "manifest": {
-                                "anyOf": [
-                                    {
-                                        "type": "null"
-                                    },
-                                    {
-                                        "properties": {
-                                            "extraFields": {
-                                                "properties": {
-                                                    "Archiver-Version": {
-                                                        "type": "string"
-                                                    },
-                                                    "Build-Jdk": {
-                                                        "type": "string"
-                                                    },
-                                                    "Built-By": {
-                                                        "type": "string"
-                                                    },
-                                                    "Created-By": {
-                                                        "type": "string"
-                                                    },
-                                                    "Extension-Name": {
-                                                        "type": "string"
-                                                    },
-                                                    "Group-Id": {
-                                                        "type": "string"
-                                                    },
-                                                    "Hudson-Version": {
-                                                        "type": "string"
-                                                    },
-                                                    "Jenkins-Version": {
-                                                        "type": "string"
-                                                    },
-                                                    "Long-Name": {
-                                                        "type": "string"
-                                                    },
-                                                    "Main-Class": {
-                                                        "type": "string"
-                                                    },
-                                                    "Minimum-Java-Version": {
-                                                        "type": "string"
-                                                    },
-                                                    "Plugin-Dependencies": {
-                                                        "type": "string"
-                                                    },
-                                                    "Plugin-Developers": {
-                                                        "type": "string"
-                                                    },
-                                                    "Plugin-License-Name": {
-                                                        "type": "string"
-                                                    },
-                                                    "Plugin-License-Url": {
-                                                        "type": "string"
-                                                    },
-                                                    "Plugin-ScmUrl": {
-                                                        "type": "string"
-                                                    },
-                                                    "Plugin-Version": {
-                                                        "type": "string"
-                                                    },
-                                                    "Short-Name": {
-                                                        "type": "string"
-                                                    }
-                                                },
-                                                "required": [
-                                                    "Archiver-Version",
-                                                    "Build-Jdk",
-                                                    "Built-By",
-                                                    "Created-By"
-                                                ],
-                                                "type": "object"
-                                            },
-                                            "implementationTitle": {
-                                                "type": "string"
-                                            },
-                                            "implementationVendor": {
-                                                "type": "string"
-                                            },
-                                            "implementationVersion": {
-                                                "type": "string"
-                                            },
-                                            "manifestVersion": {
-                                                "type": "string"
-                                            },
-                                            "name": {
-                                                "type": "string"
-                                            },
-                                            "specificationTitle": {
-                                                "type": "string"
-                                            },
-                                            "specificationVendor": {
-                                                "type": "string"
-                                            },
-                                            "specificationVersion": {
-                                                "type": "string"
-                                            }
-                                        },
-                                        "required": [
-                                            "extraFields",
-                                            "implementationTitle",
-                                            "implementationVendor",
-                                            "implementationVersion",
-                                            "manifestVersion",
-                                            "name",
-                                            "specificationTitle",
-                                            "specificationVendor",
-                                            "specificationVersion"
-                                        ],
-                                        "type": "object"
-                                    }
-                                ]
-                            },
-                            "name": {
-                                "type": "string"
-                            },
-                            "originPackage": {
-                                "type": "string"
-                            },
-                            "package": {
-                                "type": "string"
-                            },
-                            "parentPackage": {
-                                "anyOf": [
-                                    {
-                                        "type": "null"
-                                    },
-                                    {
-                                        "properties": {
-                                            "foundBy": {
-                                                "type": "string"
-                                            },
-                                            "language": {
-                                                "type": "integer"
-                                            },
-                                            "licenses": {
-                                                "type": "null"
-                                            },
-                                            "manifest": {
-                                                "type": "string"
-                                            },
-                                            "metadata": {
-                                                "properties": {
-                                                    "manifest": {
-                                                        "properties": {
-                                                            "extraFields": {
-                                                                "properties": {
-                                                                    "Archiver-Version": {
-                                                                        "type": "string"
-                                                                    },
-                                                                    "Build-Jdk": {
-                                                                        "type": "string"
-                                                                    },
-                                                                    "Built-By": {
-                                                                        "type": "string"
-                                                                    },
-                                                                    "Created-By": {
-                                                                        "type": "string"
-                                                                    },
-                                                                    "Extension-Name": {
-                                                                        "type": "string"
-                                                                    },
-                                                                    "Group-Id": {
-                                                                        "type": "string"
-                                                                    },
-                                                                    "Hudson-Version": {
-                                                                        "type": "string"
-                                                                    },
-                                                                    "Jenkins-Version": {
-                                                                        "type": "string"
-                                                                    },
-                                                                    "Long-Name": {
-                                                                        "type": "string"
-                                                                    },
-                                                                    "Main-Class": {
-                                                                        "type": "string"
-                                                                    },
-                                                                    "Minimum-Java-Version": {
-                                                                        "type": "string"
-                                                                    },
-                                                                    "Plugin-Dependencies": {
-                                                                        "type": "string"
-                                                                    },
-                                                                    "Plugin-Developers": {
-                                                                        "type": "string"
-                                                                    },
-                                                                    "Plugin-License-Name": {
-                                                                        "type": "string"
-                                                                    },
-                                                                    "Plugin-License-Url": {
-                                                                        "type": "string"
-                                                                    },
-                                                                    "Plugin-ScmUrl": {
-                                                                        "type": "string"
-                                                                    },
-                                                                    "Plugin-Version": {
-                                                                        "type": "string"
-                                                                    },
-                                                                    "Short-Name": {
-                                                                        "type": "string"
-                                                                    }
-                                                                },
-                                                                "required": [
-                                                                    "Archiver-Version",
-                                                                    "Build-Jdk",
-                                                                    "Built-By",
-                                                                    "Created-By"
-                                                                ],
-                                                                "type": "object"
-                                                            },
-                                                            "implementationTitle": {
-                                                                "type": "string"
-                                                            },
-                                                            "implementationVendor": {
-                                                                "type": "string"
-                                                            },
-                                                            "implementationVersion": {
-                                                                "type": "string"
-                                                            },
-                                                            "manifestVersion": {
-                                                                "type": "string"
-                                                            },
-                                                            "name": {
-                                                                "type": "string"
-                                                            },
-                                                            "specificationTitle": {
-                                                                "type": "string"
-                                                            },
-                                                            "specificationVendor": {
-                                                                "type": "string"
-                                                            },
-                                                            "specificationVersion": {
-                                                                "type": "string"
-                                                            }
-                                                        },
-                                                        "required": [
-                                                            "extraFields",
-                                                            "implementationTitle",
-                                                            "implementationVendor",
-                                                            "implementationVersion",
-                                                            "manifestVersion",
-                                                            "name",
-                                                            "specificationTitle",
-                                                            "specificationVendor",
-                                                            "specificationVersion"
-                                                        ],
-                                                        "type": "object"
-                                                    },
-                                                    "parentPackage": {
-                                                        "type": "null"
-                                                    },
-                                                    "pomProperties": {
-                                                        "properties": {
-                                                            "Path": {
-                                                                "type": "string"
-                                                            },
-                                                            "artifactId": {
-                                                                "type": "string"
-                                                            },
-                                                            "extraFields": {
-                                                                "type": "null"
-                                                            },
-                                                            "groupId": {
-                                                                "type": "string"
-                                                            },
-                                                            "name": {
-                                                                "type": "string"
-                                                            },
-                                                            "version": {
-                                                                "type": "string"
-                                                            }
-                                                        },
-                                                        "required": [
-                                                            "Path",
-                                                            "artifactId",
-                                                            "extraFields",
-                                                            "groupId",
-                                                            "name",
-                                                            "version"
-                                                        ],
-                                                        "type": "object"
-                                                    }
-                                                },
-                                                "required": [
-                                                    "manifest",
-                                                    "parentPackage",
-                                                    "pomProperties"
-                                                ],
-                                                "type": "object"
-                                            },
-                                            "sources": {
-                                                "type": "null"
-                                            },
-                                            "type": {
-                                                "type": "string"
-                                            },
-                                            "version": {
-                                                "type": "string"
-                                            }
-                                        },
-                                        "required": [
-                                            "foundBy",
-                                            "language",
-                                            "licenses",
-                                            "manifest",
-                                            "metadata",
-                                            "sources",
-                                            "type",
-                                            "version"
-                                        ],
-                                        "type": "object"
-                                    }
-                                ]
-                            },
-                            "pomProperties": {
-                                "properties": {
-                                    "Path": {
-                                        "type": "string"
-                                    },
-                                    "artifactId": {
-                                        "type": "string"
-                                    },
-                                    "extraFields": {
-                                        "type": "null"
-                                    },
-                                    "groupId": {
-                                        "type": "string"
-                                    },
-                                    "name": {
-                                        "type": "string"
-                                    },
-                                    "version": {
-                                        "type": "string"
-                                    }
-                                },
-                                "required": [
-                                    "Path",
-                                    "artifactId",
-                                    "extraFields",
-                                    "groupId",
-                                    "name",
-                                    "version"
-                                ],
-                                "type": "object"
-                            },
-                            "pullChecksum": {
-                                "type": "string"
-                            },
-                            "pullDependencies": {
-                                "type": "string"
-                            },
-                            "release": {
-                                "type": "string"
-                            },
-                            "size": {
-                                "type": "integer"
-                            },
-                            "source": {
-                                "type": "string"
-                            },
-                            "sourceRpm": {
-                                "type": "string"
-                            },
-                            "url": {
-                                "type": "string"
-                            },
-                            "vendor": {
-                                "type": "string"
-                            },
-                            "version": {
-                                "type": "string"
-                            }
-                        },
-                        "type": "object"
-                    },
-                    "name": {
-                        "type": "string"
-                    },
-                    "type": {
-                        "type": "string"
-                    },
-                    "version": {
-                        "type": "string"
-                    }
-                },
-                "required": [
-                    "foundBy",
-                    "locations",
-                    "name",
-                    "type",
-                    "version"
-                ],
-                "type": "object"
-            },
-            "type": "array"
-        },
-        "source": {
-            "properties": {
-                "target": {
-                    "anyOf": [
-                        {
-                            "type": "string"
-                        },
-                        {
-                            "properties": {
-                                "digest": {
-                                    "type": "string"
-                                },
-                                "layers": {
-                                    "items": {
-                                        "properties": {
-                                            "digest": {
-                                                "type": "string"
-                                            },
-                                            "mediaType": {
-                                                "type": "string"
-                                            },
-                                            "size": {
-                                                "type": "integer"
-                                            }
-                                        },
-                                        "required": [
-                                            "digest",
-                                            "mediaType",
-                                            "size"
-                                        ],
-                                        "type": "object"
-                                    },
-                                    "type": "array"
-                                },
-                                "mediaType": {
-                                    "type": "string"
-                                },
-                                "size": {
-                                    "type": "integer"
-                                },
-                                "tags": {
-                                    "items": {
-                                        "type": "string"
-                                    },
-                                    "type": "array"
-                                }
-                            },
-                            "required": [
-                                "digest",
-                                "layers",
-                                "mediaType",
-                                "size",
-                                "tags"
-                            ],
-                            "type": "object"
-                        }
-                    ]
-                },
-                "type": {
-                    "type": "string"
-                }
-            },
-            "required": [
-                "target",
-                "type"
-            ],
-            "type": "object"
-        }
-    },
-    "required": [
-        "artifacts",
-        "source"
-    ],
-    "type": "object"
-}

syft/cataloger/apkdb/parse_apk_db.go

@@ -20,9 +20,14 @@ var _ common.ParserFn = parseApkDB
 // parseApkDb parses individual packages from a given Alpine DB file. For more information on specific fields
 // see https://wiki.alpinelinux.org/wiki/Apk_spec .
 func parseApkDB(_ string, reader io.Reader) ([]pkg.Package, error) {
+	// larger capacity for the scanner.
+	const maxScannerCapacity = 1024 * 1024
+	// a new larger buffer for the scanner
+	bufScan := make([]byte, maxScannerCapacity)
 	packages := make([]pkg.Package, 0)
 
 	scanner := bufio.NewScanner(reader)
+	scanner.Buffer(bufScan, maxScannerCapacity)
 	onDoubleLF := func(data []byte, atEOF bool) (advance int, token []byte, err error) {
 		for i := 0; i < len(data); i++ {
 			if i > 0 && data[i-1] == '\n' && data[i] == '\n' {
@@ -44,11 +49,12 @@ func parseApkDB(_ string, reader io.Reader) ([]pkg.Package, error) {
 		}
 		if metadata != nil {
 			packages = append(packages, pkg.Package{
-				Name:     metadata.Package,
-				Version:  metadata.Version,
-				Licenses: strings.Split(metadata.License, " "),
-				Type:     pkg.ApkPkg,
-				Metadata: *metadata,
+				Name:         metadata.Package,
+				Version:      metadata.Version,
+				Licenses:     strings.Split(metadata.License, " "),
+				Type:         pkg.ApkPkg,
+				MetadataType: pkg.ApkMetadataType,
+				Metadata:     *metadata,
 			})
 		}
 	}
@@ -83,7 +89,16 @@ func parseApkDBEntry(reader io.Reader) (*pkg.ApkMetadata, error) {
 
 		switch key {
 		case "F":
-			lastFile = "/" + value
+			currentFile := "/" + value
+
+			newFileRecord := pkg.ApkFileRecord{
+				Path: currentFile,
+			}
+			files = append(files, newFileRecord)
+			fileRecord = &files[len(files)-1]
+
+			// future aux references are relative to previous "F" records
+			lastFile = currentFile
 			continue
 		case "R":
 			newFileRecord := pkg.ApkFileRecord{
@@ -91,24 +106,24 @@ func parseApkDBEntry(reader io.Reader) (*pkg.ApkMetadata, error) {
 			}
 			files = append(files, newFileRecord)
 			fileRecord = &files[len(files)-1]
-		case "a":
+		case "a", "M":
 			ownershipFields := strings.Split(value, ":")
 			if len(ownershipFields) < 3 {
-				log.Errorf("unexpected APK ownership field: %q", value)
+				log.Warnf("unexpected APK ownership field: %q", value)
 				continue
 			}
 			if fileRecord == nil {
-				log.Errorf("ownership field with no parent record: %q", value)
+				log.Warnf("ownership field with no parent record: %q", value)
 				continue
 			}
 			fileRecord.OwnerUID = ownershipFields[0]
-			fileRecord.OwnerGUI = ownershipFields[1]
+			fileRecord.OwnerGID = ownershipFields[1]
 			fileRecord.Permissions = ownershipFields[2]
 			// note: there are more optional fields available that we are not capturing, e.g.:
 			// "0:0:755:Q1JaDEHQHBbizhEzoWK1YxuraNU/4="
 		case "Z":
 			if fileRecord == nil {
-				log.Errorf("checksum field with no parent record: %q", value)
+				log.Warnf("checksum field with no parent record: %q", value)
 				continue
 			}
 			fileRecord.Checksum = value

syft/cataloger/apkdb/parse_apk_db_test.go

@@ -19,10 +19,25 @@ func TestExtraFileAttributes(t *testing.T) {
 			name: "test extra file attributes (checksum) are ignored",
 			expected: pkg.ApkMetadata{
 				Files: []pkg.ApkFileRecord{
+					{
+						Path: "/usr",
+					},
+					{
+						Path: "/usr/lib",
+					},
+					{
+						Path: "/usr/lib/jvm",
+					},
+					{
+						Path: "/usr/lib/jvm/java-1.8-openjdk",
+					},
+					{
+						Path: "/usr/lib/jvm/java-1.8-openjdk/bin",
+					},
 					{
 						Path:        "/usr/lib/jvm/java-1.8-openjdk/bin/policytool",
 						OwnerUID:    "0",
-						OwnerGUI:    "0",
+						OwnerGID:    "0",
 						Permissions: "755",
 						Checksum:    "Q1M0C9qfC/+kdRiOodeihG2GMRtkE=",
 					},
@@ -60,13 +75,13 @@ func TestExtraFileAttributes(t *testing.T) {
 	}
 }
 
-func TestSinglePackage(t *testing.T) {
+func TestSinglePackageDetails(t *testing.T) {
 	tests := []struct {
-		name     string
+		fixture  string
 		expected pkg.ApkMetadata
 	}{
 		{
-			name: "Test Single Package",
+			fixture: "test-fixtures/single",
 			expected: pkg.ApkMetadata{
 				Package:          "musl-utils",
 				OriginPackage:    "musl",
@@ -82,51 +97,428 @@ func TestSinglePackage(t *testing.T) {
 				PullChecksum:     "Q1bTtF5526tETKfL+lnigzIDvm+2o=",
 				GitCommitOfAport: "4024cc3b29ad4c65544ad068b8f59172b5494306",
 				Files: []pkg.ApkFileRecord{
+					{
+						Path: "/sbin",
+					},
 					{
 						Path:        "/sbin/ldconfig",
 						OwnerUID:    "0",
-						OwnerGUI:    "0",
+						OwnerGID:    "0",
 						Permissions: "755",
 						Checksum:    "Q1Kja2+POZKxEkUOZqwSjC6kmaED4=",
 					},
+					{
+						Path: "/usr",
+					},
+					{
+						Path: "/usr/bin",
+					},
 					{
 						Path:        "/usr/bin/iconv",
 						OwnerUID:    "0",
-						OwnerGUI:    "0",
+						OwnerGID:    "0",
 						Permissions: "755",
 						Checksum:    "Q1CVmFbdY+Hv6/jAHl1gec2Kbx1EY=",
 					},
 					{
 						Path:        "/usr/bin/ldd",
 						OwnerUID:    "0",
-						OwnerGUI:    "0",
+						OwnerGID:    "0",
 						Permissions: "755",
 						Checksum:    "Q1yFAhGggmL7ERgbIA7KQxyTzf3ks=",
 					},
 					{
 						Path:        "/usr/bin/getconf",
 						OwnerUID:    "0",
-						OwnerGUI:    "0",
+						OwnerGID:    "0",
 						Permissions: "755",
 						Checksum:    "Q1dAdYK8M/INibRQF5B3Rw7cmNDDA=",
 					},
 					{
 						Path:        "/usr/bin/getent",
 						OwnerUID:    "0",
-						OwnerGUI:    "0",
+						OwnerGID:    "0",
 						Permissions: "755",
 						Checksum:    "Q1eR2Dz/WylabgbWMTkd2+hGmEya4=",
 					},
 				},
 			},
 		},
+		{
+			fixture: "test-fixtures/base",
+			expected: pkg.ApkMetadata{
+				Package:          "alpine-baselayout",
+				OriginPackage:    "alpine-baselayout",
+				Version:          "3.2.0-r6",
+				Description:      "Alpine base dir structure and init scripts",
+				Maintainer:       "Natanael Copa <ncopa@alpinelinux.org>",
+				License:          "GPL-2.0-only",
+				Architecture:     "x86_64",
+				URL:              "https://git.alpinelinux.org/cgit/aports/tree/main/alpine-baselayout",
+				Size:             19917,
+				InstalledSize:    409600,
+				PullDependencies: "/bin/sh so:libc.musl-x86_64.so.1",
+				PullChecksum:     "Q1myMNfd7u5v5UTgNHeq1e31qTjZU=",
+				GitCommitOfAport: "e1c51734fa96fa4bac92e9f14a474324c67916fc",
+				Files: []pkg.ApkFileRecord{
+					{
+						Path: "/dev",
+					},
+					{
+						Path: "/dev/pts",
+					},
+					{
+						Path: "/dev/shm",
+					},
+					{
+						Path: "/etc",
+					},
+					{
+						Path:     "/etc/fstab",
+						Checksum: "Q11Q7hNe8QpDS531guqCdrXBzoA/o=",
+					},
+					{
+						Path:     "/etc/group",
+						Checksum: "Q1oJ16xWudgKOrXIEquEDzlF2Lsm4=",
+					},
+					{
+						Path:     "/etc/hostname",
+						Checksum: "Q16nVwYVXP/tChvUPdukVD2ifXOmc=",
+					},
+					{
+						Path:     "/etc/hosts",
+						Checksum: "Q1BD6zJKZTRWyqGnPi4tSfd3krsMU=",
+					},
+					{
+						Path:     "/etc/inittab",
+						Checksum: "Q1TsthbhW7QzWRe1E/NKwTOuD4pHc=",
+					},
+					{
+						Path:     "/etc/modules",
+						Checksum: "Q1toogjUipHGcMgECgPJX64SwUT1M=",
+					},
+					{
+						Path:     "/etc/motd",
+						Checksum: "Q1XmduVVNURHQ27TvYp1Lr5TMtFcA=",
+					},
+					{
+						Path:        "/etc/mtab",
+						OwnerUID:    "0",
+						OwnerGID:    "0",
+						Permissions: "777",
+						Checksum:    "Q1kiljhXXH1LlQroHsEJIkPZg2eiw=",
+					},
+					{
+						Path:     "/etc/passwd",
+						Checksum: "Q1TchuuLUfur0izvfZQZxgN/LJhB8=",
+					},
+					{
+						Path:     "/etc/profile",
+						Checksum: "Q1KpFb8kl5LvwXWlY3e58FNsjrI34=",
+					},
+					{
+						Path:     "/etc/protocols",
+						Checksum: "Q13FqXUnvuOpMDrH/6rehxuYAEE34=",
+					},
+					{
+						Path:     "/etc/services",
+						Checksum: "Q1C6HJNgQvLWqt5VY+n7MZJ1rsDuY=",
+					},
+					{
+						Path:        "/etc/shadow",
+						OwnerUID:    "0",
+						OwnerGID:    "42",
+						Permissions: "640",
+						Checksum:    "Q1ltrPIAW2zHeDiajsex2Bdmq3uqA=",
+					},
+					{
+						Path:     "/etc/shells",
+						Checksum: "Q1ojm2YdpCJ6B/apGDaZ/Sdb2xJkA=",
+					},
+					{
+						Path:     "/etc/sysctl.conf",
+						Checksum: "Q14upz3tfnNxZkIEsUhWn7Xoiw96g=",
+					},
+					{
+						Path: "/etc/apk",
+					},
+					{
+						Path: "/etc/conf.d",
+					},
+					{
+						Path: "/etc/crontabs",
+					},
+					{
+						Path:        "/etc/crontabs/root",
+						OwnerUID:    "0",
+						OwnerGID:    "0",
+						Permissions: "600",
+						Checksum:    "Q1vfk1apUWI4yLJGhhNRd0kJixfvY=",
+					},
+					{
+						Path: "/etc/init.d",
+					},
+					{
+						Path: "/etc/modprobe.d",
+					},
+					{
+						Path:     "/etc/modprobe.d/aliases.conf",
+						Checksum: "Q1WUbh6TBYNVK7e4Y+uUvLs/7viqk=",
+					},
+					{
+						Path:     "/etc/modprobe.d/blacklist.conf",
+						Checksum: "Q1xxYGU6S6TLQvb7ervPrWWwAWqMg=",
+					},
+					{
+						Path:     "/etc/modprobe.d/i386.conf",
+						Checksum: "Q1pnay/njn6ol9cCssL7KiZZ8etlc=",
+					},
+					{
+						Path:     "/etc/modprobe.d/kms.conf",
+						Checksum: "Q1ynbLn3GYDpvajba/ldp1niayeog=",
+					},
+					{
+						Path: "/etc/modules-load.d",
+					},
+					{
+						Path: "/etc/network",
+					},
+					{
+						Path: "/etc/network/if-down.d",
+					},
+					{
+						Path: "/etc/network/if-post-down.d",
+					},
+					{
+						Path: "/etc/network/if-pre-up.d",
+					},
+					{
+						Path: "/etc/network/if-up.d",
+					},
+					{
+						Path: "/etc/opt",
+					},
+					{
+						Path: "/etc/periodic",
+					},
+					{
+						Path: "/etc/periodic/15min",
+					},
+					{
+						Path: "/etc/periodic/daily",
+					},
+					{
+						Path: "/etc/periodic/hourly",
+					},
+					{
+						Path: "/etc/periodic/monthly",
+					},
+					{
+						Path: "/etc/periodic/weekly",
+					},
+					{
+						Path: "/etc/profile.d",
+					},
+					{
+						Path:     "/etc/profile.d/color_prompt",
+						Checksum: "Q10wL23GuSCVfumMRgakabUI6EsSk=",
+					},
+					{
+						Path:     "/etc/profile.d/locale",
+						Checksum: "Q1R4bIEpnKxxOSrlnZy9AoawqZ5DU=",
+					},
+					{
+						Path: "/etc/sysctl.d",
+					},
+					{
+						Path: "/home",
+					},
+					{
+						Path: "/lib",
+					},
+					{
+						Path: "/lib/firmware",
+					},
+					{
+						Path: "/lib/mdev",
+					},
+					{
+						Path: "/lib/modules-load.d",
+					},
+					{
+						Path: "/lib/sysctl.d",
+					},
+					{
+						Path:     "/lib/sysctl.d/00-alpine.conf",
+						Checksum: "Q1HpElzW1xEgmKfERtTy7oommnq6c=",
+					},
+					{
+						Path: "/media",
+					},
+					{
+						Path: "/media/cdrom",
+					},
+					{
+						Path: "/media/floppy",
+					},
+					{
+						Path: "/media/usb",
+					},
+					{
+						Path: "/mnt",
+					},
+					{
+						Path: "/opt",
+					},
+					{
+						Path: "/proc",
+					},
+					{
+						Path:        "/root",
+						OwnerUID:    "0",
+						OwnerGID:    "0",
+						Permissions: "700",
+					},
+					{
+						Path: "/run",
+					},
+					{
+						Path: "/sbin",
+					},
+					{
+						Path:        "/sbin/mkmntdirs",
+						OwnerUID:    "0",
+						OwnerGID:    "0",
+						Permissions: "755",
+						Checksum:    "Q1YeuSmC7iDbEWrusPzA/zUQF6YSg=",
+					},
+					{
+						Path: "/srv",
+					},
+					{
+						Path: "/sys",
+					},
+					{
+						Path:        "/tmp",
+						OwnerUID:    "0",
+						OwnerGID:    "0",
+						Permissions: "1777",
+					},
+					{
+						Path: "/usr",
+					},
+					{
+						Path: "/usr/lib",
+					},
+					{
+						Path: "/usr/lib/modules-load.d",
+					},
+					{
+						Path: "/usr/local",
+					},
+					{
+						Path: "/usr/local/bin",
+					},
+					{
+						Path: "/usr/local/lib",
+					},
+					{
+						Path: "/usr/local/share",
+					},
+					{
+						Path: "/usr/sbin",
+					},
+					{
+						Path: "/usr/share",
+					},
+					{
+						Path: "/usr/share/man",
+					},
+					{
+						Path: "/usr/share/misc",
+					},
+					{
+						Path: "/var",
+					},
+					{
+						Path:        "/var/run",
+						OwnerUID:    "0",
+						OwnerGID:    "0",
+						Permissions: "777",
+						Checksum:    "Q11/SNZz/8cK2dSKK+cJpVrZIuF4Q=",
+					},
+					{
+						Path: "/var/cache",
+					},
+					{
+						Path: "/var/cache/misc",
+					},
+					{
+						Path:        "/var/empty",
+						OwnerUID:    "0",
+						OwnerGID:    "0",
+						Permissions: "555",
+					},
+					{
+						Path: "/var/lib",
+					},
+					{
+						Path: "/var/lib/misc",
+					},
+					{
+						Path: "/var/local",
+					},
+					{
+						Path: "/var/lock",
+					},
+					{
+						Path: "/var/lock/subsys",
+					},
+					{
+						Path: "/var/log",
+					},
+					{
+						Path: "/var/mail",
+					},
+					{
+						Path: "/var/opt",
+					},
+					{
+						Path: "/var/spool",
+					},
+					{
+						Path:        "/var/spool/mail",
+						OwnerUID:    "0",
+						OwnerGID:    "0",
+						Permissions: "777",
+						Checksum:    "Q1dzbdazYZA2nTzSIG3YyNw7d4Juc=",
+					},
+					{
+						Path: "/var/spool/cron",
+					},
+					{
+						Path:        "/var/spool/cron/crontabs",
+						OwnerUID:    "0",
+						OwnerGID:    "0",
+						Permissions: "777",
+						Checksum:    "Q1OFZt+ZMp7j0Gny0rqSKuWJyqYmA=",
+					},
+					{
+						Path:        "/var/tmp",
+						OwnerUID:    "0",
+						OwnerGID:    "0",
+						Permissions: "1777",
+					},
+				},
+			},
+		},
 	}
 
 	for _, test := range tests {
-		t.Run(test.name, func(t *testing.T) {
-			file, err := os.Open("test-fixtures/single")
+		t.Run(test.fixture, func(t *testing.T) {
+			file, err := os.Open(test.fixture)
 			if err != nil {
-				t.Fatal("Unable to read test_fixtures/single: ", err)
+				t.Fatal("Unable to read fixture: ", err)
 			}
 			defer func() {
 				err := file.Close()
@@ -160,10 +552,11 @@ func TestMultiplePackages(t *testing.T) {
 			fixture: "test-fixtures/multiple",
 			expected: []pkg.Package{
 				{
-					Name:     "libc-utils",
-					Version:  "0.7.2-r0",
-					Licenses: []string{"BSD"},
-					Type:     pkg.ApkPkg,
+					Name:         "libc-utils",
+					Version:      "0.7.2-r0",
+					Licenses:     []string{"BSD"},
+					Type:         pkg.ApkPkg,
+					MetadataType: pkg.ApkMetadataType,
 					Metadata: pkg.ApkMetadata{
 						Package:          "libc-utils",
 						OriginPackage:    "libc-dev",
@@ -182,10 +575,11 @@ func TestMultiplePackages(t *testing.T) {
 					},
 				},
 				{
-					Name:     "musl-utils",
-					Version:  "1.1.24-r2",
-					Licenses: []string{"MIT", "BSD", "GPL2+"},
-					Type:     pkg.ApkPkg,
+					Name:         "musl-utils",
+					Version:      "1.1.24-r2",
+					Licenses:     []string{"MIT", "BSD", "GPL2+"},
+					Type:         pkg.ApkPkg,
+					MetadataType: pkg.ApkMetadataType,
 					Metadata: pkg.ApkMetadata{
 						Package:          "musl-utils",
 						OriginPackage:    "musl",
@@ -201,38 +595,47 @@ func TestMultiplePackages(t *testing.T) {
 						PullChecksum:     "Q1bTtF5526tETKfL+lnigzIDvm+2o=",
 						GitCommitOfAport: "4024cc3b29ad4c65544ad068b8f59172b5494306",
 						Files: []pkg.ApkFileRecord{
+							{
+								Path: "/sbin",
+							},
 							{
 								Path:        "/sbin/ldconfig",
 								OwnerUID:    "0",
-								OwnerGUI:    "0",
+								OwnerGID:    "0",
 								Permissions: "755",
 								Checksum:    "Q1Kja2+POZKxEkUOZqwSjC6kmaED4=",
 							},
+							{
+								Path: "/usr",
+							},
+							{
+								Path: "/usr/bin",
+							},
 							{
 								Path:        "/usr/bin/iconv",
 								OwnerUID:    "0",
-								OwnerGUI:    "0",
+								OwnerGID:    "0",
 								Permissions: "755",
 								Checksum:    "Q1CVmFbdY+Hv6/jAHl1gec2Kbx1EY=",
 							},
 							{
 								Path:        "/usr/bin/ldd",
 								OwnerUID:    "0",
-								OwnerGUI:    "0",
+								OwnerGID:    "0",
 								Permissions: "755",
 								Checksum:    "Q1yFAhGggmL7ERgbIA7KQxyTzf3ks=",
 							},
 							{
 								Path:        "/usr/bin/getconf",
 								OwnerUID:    "0",
-								OwnerGUI:    "0",
+								OwnerGID:    "0",
 								Permissions: "755",
 								Checksum:    "Q1dAdYK8M/INibRQF5B3Rw7cmNDDA=",
 							},
 							{
 								Path:        "/usr/bin/getent",
 								OwnerUID:    "0",
-								OwnerGUI:    "0",
+								OwnerGID:    "0",
 								Permissions: "755",
 								Checksum:    "Q1eR2Dz/WylabgbWMTkd2+hGmEya4=",
 							},

syft/cataloger/apkdb/test-fixtures/base

@@ -0,0 +1,149 @@
+C:Q1myMNfd7u5v5UTgNHeq1e31qTjZU=
+P:alpine-baselayout
+V:3.2.0-r6
+A:x86_64
+S:19917
+I:409600
+T:Alpine base dir structure and init scripts
+U:https://git.alpinelinux.org/cgit/aports/tree/main/alpine-baselayout
+L:GPL-2.0-only
+o:alpine-baselayout
+m:Natanael Copa <ncopa@alpinelinux.org>
+t:1590679557
+c:e1c51734fa96fa4bac92e9f14a474324c67916fc
+D:/bin/sh so:libc.musl-x86_64.so.1
+p:cmd:mkmntdirs
+F:dev
+F:dev/pts
+F:dev/shm
+F:etc
+R:fstab
+Z:Q11Q7hNe8QpDS531guqCdrXBzoA/o=
+R:group
+Z:Q1oJ16xWudgKOrXIEquEDzlF2Lsm4=
+R:hostname
+Z:Q16nVwYVXP/tChvUPdukVD2ifXOmc=
+R:hosts
+Z:Q1BD6zJKZTRWyqGnPi4tSfd3krsMU=
+R:inittab
+Z:Q1TsthbhW7QzWRe1E/NKwTOuD4pHc=
+R:modules
+Z:Q1toogjUipHGcMgECgPJX64SwUT1M=
+R:motd
+Z:Q1XmduVVNURHQ27TvYp1Lr5TMtFcA=
+R:mtab
+a:0:0:777
+Z:Q1kiljhXXH1LlQroHsEJIkPZg2eiw=
+R:passwd
+Z:Q1TchuuLUfur0izvfZQZxgN/LJhB8=
+R:profile
+Z:Q1KpFb8kl5LvwXWlY3e58FNsjrI34=
+R:protocols
+Z:Q13FqXUnvuOpMDrH/6rehxuYAEE34=
+R:services
+Z:Q1C6HJNgQvLWqt5VY+n7MZJ1rsDuY=
+R:shadow
+a:0:42:640
+Z:Q1ltrPIAW2zHeDiajsex2Bdmq3uqA=
+R:shells
+Z:Q1ojm2YdpCJ6B/apGDaZ/Sdb2xJkA=
+R:sysctl.conf
+Z:Q14upz3tfnNxZkIEsUhWn7Xoiw96g=
+F:etc/apk
+F:etc/conf.d
+F:etc/crontabs
+R:root
+a:0:0:600
+Z:Q1vfk1apUWI4yLJGhhNRd0kJixfvY=
+F:etc/init.d
+F:etc/modprobe.d
+R:aliases.conf
+Z:Q1WUbh6TBYNVK7e4Y+uUvLs/7viqk=
+R:blacklist.conf
+Z:Q1xxYGU6S6TLQvb7ervPrWWwAWqMg=
+R:i386.conf
+Z:Q1pnay/njn6ol9cCssL7KiZZ8etlc=
+R:kms.conf
+Z:Q1ynbLn3GYDpvajba/ldp1niayeog=
+F:etc/modules-load.d
+F:etc/network
+F:etc/network/if-down.d
+F:etc/network/if-post-down.d
+F:etc/network/if-pre-up.d
+F:etc/network/if-up.d
+F:etc/opt
+F:etc/periodic
+F:etc/periodic/15min
+F:etc/periodic/daily
+F:etc/periodic/hourly
+F:etc/periodic/monthly
+F:etc/periodic/weekly
+F:etc/profile.d
+R:color_prompt
+Z:Q10wL23GuSCVfumMRgakabUI6EsSk=
+R:locale
+Z:Q1R4bIEpnKxxOSrlnZy9AoawqZ5DU=
+F:etc/sysctl.d
+F:home
+F:lib
+F:lib/firmware
+F:lib/mdev
+F:lib/modules-load.d
+F:lib/sysctl.d
+R:00-alpine.conf
+Z:Q1HpElzW1xEgmKfERtTy7oommnq6c=
+F:media
+F:media/cdrom
+F:media/floppy
+F:media/usb
+F:mnt
+F:opt
+F:proc
+F:root
+M:0:0:700
+F:run
+F:sbin
+R:mkmntdirs
+a:0:0:755
+Z:Q1YeuSmC7iDbEWrusPzA/zUQF6YSg=
+F:srv
+F:sys
+F:tmp
+M:0:0:1777
+F:usr
+F:usr/lib
+F:usr/lib/modules-load.d
+F:usr/local
+F:usr/local/bin
+F:usr/local/lib
+F:usr/local/share
+F:usr/sbin
+F:usr/share
+F:usr/share/man
+F:usr/share/misc
+F:var
+R:run
+a:0:0:777
+Z:Q11/SNZz/8cK2dSKK+cJpVrZIuF4Q=
+F:var/cache
+F:var/cache/misc
+F:var/empty
+M:0:0:555
+F:var/lib
+F:var/lib/misc
+F:var/local
+F:var/lock
+F:var/lock/subsys
+F:var/log
+F:var/mail
+F:var/opt
+F:var/spool
+R:mail
+a:0:0:777
+Z:Q1dzbdazYZA2nTzSIG3YyNw7d4Juc=
+F:var/spool/cron
+R:crontabs
+a:0:0:777
+Z:Q1OFZt+ZMp7j0Gny0rqSKuWJyqYmA=
+F:var/tmp
+M:0:0:1777

syft/cataloger/catalog.go

@@ -1,12 +1,12 @@
 package cataloger
 
 import (
-	"github.com/anchore/stereoscope/pkg/file"
 	"github.com/anchore/syft/internal/bus"
 	"github.com/anchore/syft/internal/log"
+	"github.com/anchore/syft/syft/distro"
 	"github.com/anchore/syft/syft/event"
 	"github.com/anchore/syft/syft/pkg"
-	"github.com/anchore/syft/syft/scope"
+	"github.com/anchore/syft/syft/source"
 	"github.com/hashicorp/go-multierror"
 	"github.com/wagoodman/go-partybus"
 	"github.com/wagoodman/go-progress"
@@ -33,44 +33,37 @@ func newMonitor() (*progress.Manual, *progress.Manual) {
 	return &filesProcessed, &packagesDiscovered
 }
 
-// Catalog a given scope (container image or filesystem) with the given catalogers, returning all discovered packages.
+// Catalog a given source (container image or filesystem) with the given catalogers, returning all discovered packages.
 // In order to efficiently retrieve contents from a underlying container image the content fetch requests are
 // done in bulk. Specifically, all files of interest are collected from each catalogers and accumulated into a single
 // request.
-func Catalog(resolver scope.Resolver, catalogers ...Cataloger) (*pkg.Catalog, error) {
+func Catalog(resolver source.Resolver, theDistro *distro.Distro, catalogers ...Cataloger) (*pkg.Catalog, error) {
 	catalog := pkg.NewCatalog()
-	fileSelection := make([]file.Reference, 0)
-
 	filesProcessed, packagesDiscovered := newMonitor()
 
-	// ask catalogers for files to extract from the image tar
-	for _, a := range catalogers {
-		fileSelection = append(fileSelection, a.SelectFiles(resolver)...)
-		log.Debugf("cataloger '%s' selected '%d' files", a.Name(), len(fileSelection))
-		filesProcessed.N += int64(len(fileSelection))
-	}
-
-	// fetch contents for requested selection by catalogers
-	// TODO: we should consider refactoring to return a set of io.Readers instead of the full contents themselves (allow for optional buffering).
-	contents, err := resolver.MultipleFileContentsByRef(fileSelection...)
-	if err != nil {
-		return nil, err
-	}
-
 	// perform analysis, accumulating errors for each failed analysis
 	var errs error
-	for _, a := range catalogers {
-		// TODO: check for multiple rounds of analyses by Iterate error
-		packages, err := a.Catalog(contents)
+	for _, theCataloger := range catalogers {
+		// find packages from the underlying raw data
+		packages, err := theCataloger.Catalog(resolver)
 		if err != nil {
 			errs = multierror.Append(errs, err)
 			continue
 		}
 
-		log.Debugf("cataloger '%s' discovered '%d' packages", a.Name(), len(packages))
-		packagesDiscovered.N += int64(len(packages))
+		catalogedPackages := len(packages)
+
+		log.Debugf("cataloger '%s' discovered '%d' packages", theCataloger.Name(), catalogedPackages)
+		packagesDiscovered.N += int64(catalogedPackages)
 
 		for _, p := range packages {
+			// generate CPEs
+			p.CPEs = generatePackageCPEs(p)
+
+			// generate PURL
+			p.PURL = generatePackageURL(p, theDistro)
+
+			// add to catalog
 			catalog.Add(p)
 		}
 	}

syft/cataloger/cataloger.go

@@ -6,7 +6,6 @@ catalogers defined in child packages as well as the interface definition to impl
 package cataloger
 
 import (
-	"github.com/anchore/stereoscope/pkg/file"
 	"github.com/anchore/syft/syft/cataloger/apkdb"
 	"github.com/anchore/syft/syft/cataloger/deb"
 	"github.com/anchore/syft/syft/cataloger/golang"
@@ -16,7 +15,7 @@ import (
 	"github.com/anchore/syft/syft/cataloger/rpmdb"
 	"github.com/anchore/syft/syft/cataloger/ruby"
 	"github.com/anchore/syft/syft/pkg"
-	"github.com/anchore/syft/syft/scope"
+	"github.com/anchore/syft/syft/source"
 )
 
 // Cataloger describes behavior for an object to participate in parsing container image or file system
@@ -25,20 +24,16 @@ import (
 type Cataloger interface {
 	// Name returns a string that uniquely describes a cataloger
 	Name() string
-	// SelectFiles discovers and returns specific files that the cataloger would like to inspect the contents of.
-	SelectFiles(scope.FileResolver) []file.Reference
-	// Catalog is given the file contents and should return any discovered Packages after analyzing the contents.
-	Catalog(map[file.Reference]string) ([]pkg.Package, error)
-	// TODO: add "IterationNeeded" error to indicate to the driver to continue with another Select/Catalog pass
-	// TODO: we should consider refactoring to return a set of io.Readers instead of the full contents themselves (allow for optional buffering).
+	// Catalog is given an object to resolve file references and content, this function returns any discovered Packages after analyzing the catalog source.
+	Catalog(resolver source.Resolver) ([]pkg.Package, error)
 }
 
 // ImageCatalogers returns a slice of locally implemented catalogers that are fit for detecting installations of packages.
 func ImageCatalogers() []Cataloger {
 	return []Cataloger{
 		ruby.NewGemSpecCataloger(),
-		python.NewPythonCataloger(),         // TODO: split and replace me
-		javascript.NewJavascriptCataloger(), // TODO: split and replace me
+		python.NewPythonPackageCataloger(),
+		javascript.NewJavascriptPackageCataloger(),
 		deb.NewDpkgdbCataloger(),
 		rpmdb.NewRpmdbCataloger(),
 		java.NewJavaCataloger(),
@@ -51,8 +46,9 @@ func ImageCatalogers() []Cataloger {
 func DirectoryCatalogers() []Cataloger {
 	return []Cataloger{
 		ruby.NewGemFileLockCataloger(),
-		python.NewPythonCataloger(),         // TODO: split and replace me
-		javascript.NewJavascriptCataloger(), // TODO: split and replace me
+		python.NewPythonIndexCataloger(),
+		python.NewPythonPackageCataloger(),
+		javascript.NewJavascriptLockCataloger(),
 		deb.NewDpkgdbCataloger(),
 		rpmdb.NewRpmdbCataloger(),
 		java.NewJavaCataloger(),

syft/cataloger/common/generic_cataloger.go

@@ -6,10 +6,9 @@ package common
 import (
 	"strings"
 
-	"github.com/anchore/stereoscope/pkg/file"
 	"github.com/anchore/syft/internal/log"
 	"github.com/anchore/syft/syft/pkg"
-	"github.com/anchore/syft/syft/scope"
+	"github.com/anchore/syft/syft/source"
 )
 
 // GenericCataloger implements the Catalog interface and is responsible for dispatching the proper parser function for
@@ -17,8 +16,8 @@ import (
 type GenericCataloger struct {
 	globParsers       map[string]ParserFn
 	pathParsers       map[string]ParserFn
-	selectedFiles     []file.Reference
-	parsers           map[file.Reference]ParserFn
+	selectedFiles     []source.Location
+	parsers           map[source.Location]ParserFn
 	upstreamCataloger string
 }
 
@@ -27,81 +26,91 @@ func NewGenericCataloger(pathParsers map[string]ParserFn, globParsers map[string
 	return &GenericCataloger{
 		globParsers:       globParsers,
 		pathParsers:       pathParsers,
-		selectedFiles:     make([]file.Reference, 0),
-		parsers:           make(map[file.Reference]ParserFn),
+		selectedFiles:     make([]source.Location, 0),
+		parsers:           make(map[source.Location]ParserFn),
 		upstreamCataloger: upstreamCataloger,
 	}
 }
 
 // Name returns a string that uniquely describes the upstream cataloger that this Generic Cataloger represents.
-func (a *GenericCataloger) Name() string {
-	return a.upstreamCataloger
+func (c *GenericCataloger) Name() string {
+	return c.upstreamCataloger
 }
 
 // register pairs a set of file references with a parser function for future cataloging (when the file contents are resolved)
-func (a *GenericCataloger) register(files []file.Reference, parser ParserFn) {
-	a.selectedFiles = append(a.selectedFiles, files...)
+func (c *GenericCataloger) register(files []source.Location, parser ParserFn) {
+	c.selectedFiles = append(c.selectedFiles, files...)
 	for _, f := range files {
-		a.parsers[f] = parser
+		c.parsers[f] = parser
 	}
 }
 
 // clear deletes all registered file-reference-to-parser-function pairings from former SelectFiles() and register() calls
-func (a *GenericCataloger) clear() {
-	a.selectedFiles = make([]file.Reference, 0)
-	a.parsers = make(map[file.Reference]ParserFn)
+func (c *GenericCataloger) clear() {
+	c.selectedFiles = make([]source.Location, 0)
+	c.parsers = make(map[source.Location]ParserFn)
+}
+
+// Catalog is given an object to resolve file references and content, this function returns any discovered Packages after analyzing the catalog source.
+func (c *GenericCataloger) Catalog(resolver source.Resolver) ([]pkg.Package, error) {
+	fileSelection := c.selectFiles(resolver)
+	contents, err := resolver.MultipleFileContentsByLocation(fileSelection)
+	if err != nil {
+		return nil, err
+	}
+	return c.catalog(contents)
 }
 
 // SelectFiles takes a set of file trees and resolves and file references of interest for future cataloging
-func (a *GenericCataloger) SelectFiles(resolver scope.FileResolver) []file.Reference {
+func (c *GenericCataloger) selectFiles(resolver source.FileResolver) []source.Location {
 	// select by exact path
-	for path, parser := range a.pathParsers {
-		files, err := resolver.FilesByPath(file.Path(path))
+	for path, parser := range c.pathParsers {
+		files, err := resolver.FilesByPath(path)
 		if err != nil {
-			log.Errorf("cataloger failed to select files by path: %+v", err)
+			log.Warnf("cataloger failed to select files by path: %+v", err)
 		}
 		if files != nil {
-			a.register(files, parser)
+			c.register(files, parser)
 		}
 	}
 
 	// select by glob pattern
-	for globPattern, parser := range a.globParsers {
+	for globPattern, parser := range c.globParsers {
 		fileMatches, err := resolver.FilesByGlob(globPattern)
 		if err != nil {
-			log.Errorf("failed to find files by glob: %s", globPattern)
+			log.Warnf("failed to find files by glob: %s", globPattern)
 		}
 		if fileMatches != nil {
-			a.register(fileMatches, parser)
+			c.register(fileMatches, parser)
 		}
 	}
 
-	return a.selectedFiles
+	return c.selectedFiles
 }
 
-// Catalog takes a set of file contents and uses any configured parser functions to resolve and return discovered packages
-func (a *GenericCataloger) Catalog(contents map[file.Reference]string) ([]pkg.Package, error) {
-	defer a.clear()
+// catalog takes a set of file contents and uses any configured parser functions to resolve and return discovered packages
+func (c *GenericCataloger) catalog(contents map[source.Location]string) ([]pkg.Package, error) {
+	defer c.clear()
 
 	packages := make([]pkg.Package, 0)
 
-	for reference, parser := range a.parsers {
-		content, ok := contents[reference]
+	for location, parser := range c.parsers {
+		content, ok := contents[location]
 		if !ok {
-			log.Errorf("cataloger '%s' missing file content: %+v", a.upstreamCataloger, reference)
+			log.Warnf("cataloger '%s' missing file content: %+v", c.upstreamCataloger, location)
 			continue
 		}
 
-		entries, err := parser(string(reference.Path), strings.NewReader(content))
+		entries, err := parser(location.Path, strings.NewReader(content))
 		if err != nil {
 			// TODO: should we fail? or only log?
-			log.Errorf("cataloger '%s' failed to parse entries (reference=%+v): %+v", a.upstreamCataloger, reference, err)
+			log.Warnf("cataloger '%s' failed to parse entries (location=%+v): %+v", c.upstreamCataloger, location, err)
 			continue
 		}
 
 		for _, entry := range entries {
-			entry.FoundBy = a.upstreamCataloger
-			entry.Source = []file.Reference{reference}
+			entry.FoundBy = c.upstreamCataloger
+			entry.Locations = []source.Location{location}
 
 			packages = append(packages, entry)
 		}

syft/cataloger/common/generic_cataloger_test.go

@@ -6,37 +6,50 @@ import (
 	"io/ioutil"
 	"testing"
 
-	"github.com/anchore/stereoscope/pkg/file"
-	"github.com/anchore/syft/internal"
+	"github.com/anchore/syft/syft/source"
+
 	"github.com/anchore/syft/syft/pkg"
 )
 
-type testResolver struct {
-	contents map[file.Reference]string
+type testResolverMock struct {
+	contents map[source.Location]string
 }
 
-func newTestResolver() *testResolver {
-	return &testResolver{
-		contents: make(map[file.Reference]string),
+func newTestResolver() *testResolverMock {
+	return &testResolverMock{
+		contents: make(map[source.Location]string),
 	}
 }
 
-func (r *testResolver) FilesByPath(paths ...file.Path) ([]file.Reference, error) {
-	results := make([]file.Reference, len(paths))
+func (r *testResolverMock) FileContentsByLocation(_ source.Location) (string, error) {
+	return "", fmt.Errorf("not implemented")
+}
+
+func (r *testResolverMock) MultipleFileContentsByLocation([]source.Location) (map[source.Location]string, error) {
+	return r.contents, nil
+}
+
+func (r *testResolverMock) FilesByPath(paths ...string) ([]source.Location, error) {
+	results := make([]source.Location, len(paths))
 
 	for idx, p := range paths {
-		results[idx] = file.NewFileReference(p)
+		results[idx] = source.NewLocation(p)
 		r.contents[results[idx]] = fmt.Sprintf("%s file contents!", p)
 	}
 
 	return results, nil
 }
 
-func (r *testResolver) FilesByGlob(patterns ...string) ([]file.Reference, error) {
+func (r *testResolverMock) FilesByGlob(_ ...string) ([]source.Location, error) {
 	path := "/a-path.txt"
-	ref := file.NewFileReference(file.Path(path))
-	r.contents[ref] = fmt.Sprintf("%s file contents!", path)
-	return []file.Reference{ref}, nil
+	location := source.NewLocation(path)
+	r.contents[location] = fmt.Sprintf("%s file contents!", path)
+	return []source.Location{location}, nil
+}
+
+func (r *testResolverMock) RelativeFileByPath(_ source.Location, _ string) *source.Location {
+	panic(fmt.Errorf("not implemented"))
+	return nil
 }
 
 func parser(_ string, reader io.Reader) ([]pkg.Package, error) {
@@ -64,31 +77,16 @@ func TestGenericCataloger(t *testing.T) {
 	resolver := newTestResolver()
 	cataloger := NewGenericCataloger(pathParsers, globParsers, upstream)
 
-	selected := cataloger.SelectFiles(resolver)
-
-	if len(selected) != 3 {
-		t.Fatalf("unexpected selection length: %d", len(selected))
-	}
-
-	expectedSelection := internal.NewStringSetFromSlice([]string{"/last/path.txt", "/another-path.txt", "/a-path.txt"})
-	selectionByPath := make(map[string]file.Reference)
-	for _, s := range selected {
-		if !expectedSelection.Contains(string(s.Path)) {
-			t.Errorf("unexpected selection path: %+v", s.Path)
-		}
-		selectionByPath[string(s.Path)] = s
-	}
-
-	expectedPkgs := make(map[file.Reference]pkg.Package)
-	for path, ref := range selectionByPath {
-		expectedPkgs[ref] = pkg.Package{
+	expectedSelection := []string{"/last/path.txt", "/another-path.txt", "/a-path.txt"}
+	expectedPkgs := make(map[string]pkg.Package)
+	for _, path := range expectedSelection {
+		expectedPkgs[path] = pkg.Package{
 			FoundBy: upstream,
-			Source:  []file.Reference{ref},
 			Name:    fmt.Sprintf("%s file contents!", path),
 		}
 	}
 
-	actualPkgs, err := cataloger.Catalog(resolver.contents)
+	actualPkgs, err := cataloger.Catalog(resolver)
 	if err != nil {
 		t.Fatalf("cataloger catalog action failed: %+v", err)
 	}
@@ -98,8 +96,8 @@ func TestGenericCataloger(t *testing.T) {
 	}
 
 	for _, p := range actualPkgs {
-		ref := p.Source[0]
-		exP, ok := expectedPkgs[ref]
+		ref := p.Locations[0]
+		exP, ok := expectedPkgs[ref.Path]
 		if !ok {
 			t.Errorf("missing expected pkg: ref=%+v", ref)
 			continue
@@ -110,7 +108,7 @@ func TestGenericCataloger(t *testing.T) {
 		}
 
 		if exP.Name != p.Name {
-			t.Errorf("bad contents mapping: %+v", p.Source)
+			t.Errorf("bad contents mapping: %+v", p.Locations)
 		}
 	}
 }

syft/cataloger/cpe.go

@@ -0,0 +1,118 @@
+package cataloger
+
+import (
+	"fmt"
+	"sort"
+	"strings"
+
+	"github.com/anchore/syft/internal"
+	"github.com/anchore/syft/syft/pkg"
+	"github.com/facebookincubator/nvdtools/wfn"
+)
+
+// this is functionally equivalent to "*" and consistent with no input given (thus easier to test)
+const any = ""
+
+func newCPE(product, vendor, version, targetSW string) wfn.Attributes {
+	cpe := *(wfn.NewAttributesWithAny())
+	cpe.Part = "a"
+	cpe.Product = product
+	cpe.Vendor = vendor
+	cpe.Version = version
+	cpe.TargetSW = targetSW
+
+	return cpe
+}
+
+// generatePackageCPEs Create a list of CPEs, trying to guess the vendor, product tuple and setting TargetSoftware if possible
+func generatePackageCPEs(p pkg.Package) []pkg.CPE {
+	targetSws := candidateTargetSoftwareAttrs(p)
+	vendors := candidateVendors(p)
+	products := candidateProducts(p)
+
+	keys := internal.NewStringSet()
+	cpes := make([]pkg.CPE, 0)
+	for _, product := range products {
+		for _, vendor := range append([]string{any}, vendors...) {
+			for _, targetSw := range append([]string{any}, targetSws...) {
+				// prevent duplicate entries...
+				key := fmt.Sprintf("%s|%s|%s|%s", product, vendor, p.Version, targetSw)
+				if keys.Contains(key) {
+					continue
+				}
+				keys.Add(key)
+
+				// add a new entry...
+				c := newCPE(product, vendor, p.Version, targetSw)
+				cpes = append(cpes, c)
+			}
+		}
+	}
+
+	sort.Sort(ByCPESpecificity(cpes))
+
+	return cpes
+}
+
+func candidateTargetSoftwareAttrs(p pkg.Package) []string {
+	// TODO: would be great to allow these to be overridden by user data/config
+	var targetSw []string
+	switch p.Language {
+	case pkg.Java:
+		targetSw = append(targetSw, "java", "maven")
+	case pkg.JavaScript:
+		targetSw = append(targetSw, "node.js", "nodejs")
+	case pkg.Ruby:
+		targetSw = append(targetSw, "ruby", "rails")
+	case pkg.Python:
+		targetSw = append(targetSw, "python")
+	}
+
+	if p.Type == pkg.JenkinsPluginPkg {
+		targetSw = append(targetSw, "jenkins", "cloudbees_jenkins")
+	}
+
+	return targetSw
+}
+
+func candidateVendors(p pkg.Package) []string {
+	vendors := candidateProducts(p)
+	switch p.Language {
+	case pkg.Python:
+		vendors = append(vendors, fmt.Sprintf("python-%s", p.Name))
+	case pkg.Java:
+		if p.MetadataType == pkg.JavaMetadataType {
+			if metadata, ok := p.Metadata.(pkg.JavaMetadata); ok && metadata.PomProperties != nil {
+				// derive the vendor from the groupID (e.g. org.sonatype.nexus --> sonatype)
+				if strings.HasPrefix(metadata.PomProperties.GroupID, "org.") || strings.HasPrefix(metadata.PomProperties.GroupID, "com.") {
+					fields := strings.Split(metadata.PomProperties.GroupID, ".")
+					if len(fields) >= 3 {
+						vendors = append(vendors, fields[1])
+					}
+				}
+			}
+		}
+	}
+	return vendors
+}
+
+func candidateProducts(p pkg.Package) []string {
+	var products = []string{p.Name}
+	switch p.Language {
+	case pkg.Java:
+		if p.MetadataType == pkg.JavaMetadataType {
+			if metadata, ok := p.Metadata.(pkg.JavaMetadata); ok && metadata.PomProperties != nil {
+				// derive the product from the groupID (e.g. org.sonatype.nexus --> nexus)
+				if strings.HasPrefix(metadata.PomProperties.GroupID, "org.") || strings.HasPrefix(metadata.PomProperties.GroupID, "com.") {
+					fields := strings.Split(metadata.PomProperties.GroupID, ".")
+					if len(fields) >= 3 {
+						products = append(products, fields[2])
+					}
+				}
+			}
+		}
+	default:
+		return products
+	}
+	return products
+}

syft/cataloger/cpe_specificity.go

@@ -0,0 +1,31 @@
+package cataloger
+
+import "github.com/facebookincubator/nvdtools/wfn"
+
+type ByCPESpecificity []wfn.Attributes
+
+// Implementing sort.Interface
+func (c ByCPESpecificity) Len() int      { return len(c) }
+func (c ByCPESpecificity) Swap(i, j int) { c[i], c[j] = c[j], c[i] }
+func (c ByCPESpecificity) Less(i, j int) bool {
+	return countSpecifiedFields(c[i]) > countSpecifiedFields(c[j])
+}
+
+func countSpecifiedFields(cpe wfn.Attributes) int {
+	checksForSpecifiedField := []func(cpe wfn.Attributes) bool{
+		func(cpe wfn.Attributes) bool { return cpe.Part != "" },
+		func(cpe wfn.Attributes) bool { return cpe.Product != "" },
+		func(cpe wfn.Attributes) bool { return cpe.Vendor != "" },
+		func(cpe wfn.Attributes) bool { return cpe.Version != "" },
+		func(cpe wfn.Attributes) bool { return cpe.TargetSW != "" },
+	}
+
+	count := 0
+	for _, fieldIsSpecified := range checksForSpecifiedField {
+		if fieldIsSpecified(cpe) {
+			count++
+		}
+	}
+
+	return count
+}

syft/cataloger/cpe_test.go

@@ -0,0 +1,182 @@
+package cataloger
+
+import (
+	"sort"
+	"testing"
+
+	"github.com/scylladb/go-set/strset"
+
+	"github.com/scylladb/go-set"
+
+	"github.com/anchore/syft/syft/pkg"
+)
+
+func TestGenerate(t *testing.T) {
+	tests := []struct {
+		name     string
+		p        pkg.Package
+		expected []string
+	}{
+		{
+			name: "python language",
+			p: pkg.Package{
+				Name:     "name",
+				Version:  "3.2",
+				FoundBy:  "some-analyzer",
+				Language: pkg.Python,
+				Type:     pkg.DebPkg,
+			},
+			expected: []string{
+				"cpe:2.3:a:*:name:3.2:*:*:*:*:*:*:*",
+				"cpe:2.3:a:*:name:3.2:*:*:*:*:python:*:*",
+				"cpe:2.3:a:name:name:3.2:*:*:*:*:*:*:*",
+				"cpe:2.3:a:name:name:3.2:*:*:*:*:python:*:*",
+				"cpe:2.3:a:python-name:name:3.2:*:*:*:*:*:*:*",
+				"cpe:2.3:a:python-name:name:3.2:*:*:*:*:python:*:*",
+			},
+		},
+		{
+			name: "javascript language",
+			p: pkg.Package{
+				Name:     "name",
+				Version:  "3.2",
+				FoundBy:  "some-analyzer",
+				Language: pkg.JavaScript,
+				Type:     pkg.DebPkg,
+			},
+			expected: []string{
+				"cpe:2.3:a:*:name:3.2:*:*:*:*:*:*:*",
+				"cpe:2.3:a:*:name:3.2:*:*:*:*:node.js:*:*",
+				"cpe:2.3:a:*:name:3.2:*:*:*:*:nodejs:*:*",
+				"cpe:2.3:a:name:name:3.2:*:*:*:*:*:*:*",
+				"cpe:2.3:a:name:name:3.2:*:*:*:*:node.js:*:*",
+				"cpe:2.3:a:name:name:3.2:*:*:*:*:nodejs:*:*",
+			},
+		},
+		{
+			name: "ruby language",
+			p: pkg.Package{
+				Name:     "name",
+				Version:  "3.2",
+				FoundBy:  "some-analyzer",
+				Language: pkg.Ruby,
+				Type:     pkg.DebPkg,
+			},
+			expected: []string{
+				"cpe:2.3:a:*:name:3.2:*:*:*:*:*:*:*",
+				"cpe:2.3:a:*:name:3.2:*:*:*:*:ruby:*:*",
+				"cpe:2.3:a:*:name:3.2:*:*:*:*:rails:*:*",
+				"cpe:2.3:a:name:name:3.2:*:*:*:*:*:*:*",
+				"cpe:2.3:a:name:name:3.2:*:*:*:*:ruby:*:*",
+				"cpe:2.3:a:name:name:3.2:*:*:*:*:rails:*:*",
+			},
+		},
+		{
+			name: "java language",
+			p: pkg.Package{
+				Name:     "name",
+				Version:  "3.2",
+				FoundBy:  "some-analyzer",
+				Language: pkg.Java,
+				Type:     pkg.DebPkg,
+			},
+			expected: []string{
+				"cpe:2.3:a:*:name:3.2:*:*:*:*:*:*:*",
+				"cpe:2.3:a:*:name:3.2:*:*:*:*:java:*:*",
+				"cpe:2.3:a:*:name:3.2:*:*:*:*:maven:*:*",
+				"cpe:2.3:a:name:name:3.2:*:*:*:*:*:*:*",
+				"cpe:2.3:a:name:name:3.2:*:*:*:*:java:*:*",
+				"cpe:2.3:a:name:name:3.2:*:*:*:*:maven:*:*",
+			},
+		},
+		{
+			name: "java language with groupID",
+			p: pkg.Package{
+				Name:         "name",
+				Version:      "3.2",
+				FoundBy:      "some-analyzer",
+				Language:     pkg.Java,
+				Type:         pkg.JavaPkg,
+				MetadataType: pkg.JavaMetadataType,
+				Metadata: pkg.JavaMetadata{
+					PomProperties: &pkg.PomProperties{
+						GroupID: "org.sonatype.nexus",
+					},
+				},
+			},
+			expected: []string{
+				"cpe:2.3:a:*:name:3.2:*:*:*:*:*:*:*",
+				"cpe:2.3:a:*:name:3.2:*:*:*:*:java:*:*",
+				"cpe:2.3:a:*:name:3.2:*:*:*:*:maven:*:*",
+				"cpe:2.3:a:name:name:3.2:*:*:*:*:*:*:*",
+				"cpe:2.3:a:name:name:3.2:*:*:*:*:java:*:*",
+				"cpe:2.3:a:name:name:3.2:*:*:*:*:maven:*:*",
+				"cpe:2.3:a:sonatype:name:3.2:*:*:*:*:*:*:*",
+				"cpe:2.3:a:sonatype:name:3.2:*:*:*:*:java:*:*",
+				"cpe:2.3:a:sonatype:name:3.2:*:*:*:*:maven:*:*",
+				"cpe:2.3:a:*:nexus:3.2:*:*:*:*:*:*:*",
+				"cpe:2.3:a:*:nexus:3.2:*:*:*:*:java:*:*",
+				"cpe:2.3:a:*:nexus:3.2:*:*:*:*:maven:*:*",
+				"cpe:2.3:a:sonatype:nexus:3.2:*:*:*:*:*:*:*",
+				"cpe:2.3:a:sonatype:nexus:3.2:*:*:*:*:java:*:*",
+				"cpe:2.3:a:sonatype:nexus:3.2:*:*:*:*:maven:*:*",
+				"cpe:2.3:a:nexus:name:3.2:*:*:*:*:*:*:*",
+				"cpe:2.3:a:nexus:name:3.2:*:*:*:*:java:*:*",
+				"cpe:2.3:a:nexus:name:3.2:*:*:*:*:maven:*:*",
+				"cpe:2.3:a:name:nexus:3.2:*:*:*:*:*:*:*",
+				"cpe:2.3:a:name:nexus:3.2:*:*:*:*:java:*:*",
+				"cpe:2.3:a:name:nexus:3.2:*:*:*:*:maven:*:*",
+				"cpe:2.3:a:nexus:nexus:3.2:*:*:*:*:*:*:*",
+				"cpe:2.3:a:nexus:nexus:3.2:*:*:*:*:java:*:*",
+				"cpe:2.3:a:nexus:nexus:3.2:*:*:*:*:maven:*:*",
+			},
+		},
+		{
+			name: "jenkins package",
+			p: pkg.Package{
+				Name:     "name",
+				Version:  "3.2",
+				FoundBy:  "some-analyzer",
+				Language: pkg.Java,
+				Type:     pkg.JenkinsPluginPkg,
+			},
+			expected: []string{
+				"cpe:2.3:a:*:name:3.2:*:*:*:*:*:*:*",
+				"cpe:2.3:a:*:name:3.2:*:*:*:*:java:*:*",
+				"cpe:2.3:a:*:name:3.2:*:*:*:*:maven:*:*",
+				"cpe:2.3:a:*:name:3.2:*:*:*:*:jenkins:*:*",
+				"cpe:2.3:a:*:name:3.2:*:*:*:*:cloudbees_jenkins:*:*",
+				"cpe:2.3:a:name:name:3.2:*:*:*:*:*:*:*",
+				"cpe:2.3:a:name:name:3.2:*:*:*:*:java:*:*",
+				"cpe:2.3:a:name:name:3.2:*:*:*:*:maven:*:*",
+				"cpe:2.3:a:name:name:3.2:*:*:*:*:jenkins:*:*",
+				"cpe:2.3:a:name:name:3.2:*:*:*:*:cloudbees_jenkins:*:*",
+			},
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			actual := generatePackageCPEs(test.p)
+
+			expectedCpeSet := set.NewStringSet(test.expected...)
+			actualCpeSet := set.NewStringSet()
+			for _, a := range actual {
+				actualCpeSet.Add(a.BindToFmtString())
+			}
+
+			extra := strset.Difference(expectedCpeSet, actualCpeSet).List()
+			sort.Strings(extra)
+			for _, d := range extra {
+				t.Errorf("extra CPE: %+v", d)
+			}
+
+			missing := strset.Difference(actualCpeSet, expectedCpeSet).List()
+			sort.Strings(missing)
+			for _, d := range missing {
+				t.Errorf("missing CPE: %+v", d)
+			}
+
+		})
+	}
+}

syft/cataloger/deb/cataloger.go

@@ -4,14 +4,189 @@ Package dpkg provides a concrete Cataloger implementation for Debian package DB
 package deb
 
 import (
-	"github.com/anchore/syft/syft/cataloger/common"
+	"fmt"
+	"io"
+	"path"
+	"path/filepath"
+	"strings"
+
+	"github.com/anchore/syft/syft/pkg"
+	"github.com/anchore/syft/syft/source"
 )
 
+const (
+	dpkgStatusGlob = "**/var/lib/dpkg/status"
+	md5sumsExt     = ".md5sums"
+	docsPath       = "/usr/share/doc"
+)
+
+type Cataloger struct{}
+
 // NewDpkgdbCataloger returns a new Deb package cataloger object.
-func NewDpkgdbCataloger() *common.GenericCataloger {
-	globParsers := map[string]common.ParserFn{
-		"**/var/lib/dpkg/status": parseDpkgStatus,
+func NewDpkgdbCataloger() *Cataloger {
+	return &Cataloger{}
+}
+
+// Name returns a string that uniquely describes a cataloger
+func (c *Cataloger) Name() string {
+	return "dpkgdb-cataloger"
+}
+
+// Catalog is given an object to resolve file references and content, this function returns any discovered Packages after analyzing dpkg support files.
+// nolint:funlen
+func (c *Cataloger) Catalog(resolver source.Resolver) ([]pkg.Package, error) {
+	dbFileMatches, err := resolver.FilesByGlob(dpkgStatusGlob)
+	if err != nil {
+		return nil, fmt.Errorf("failed to find dpkg status files's by glob: %w", err)
 	}
 
-	return common.NewGenericCataloger(nil, globParsers, "dpkgdb-cataloger")
+	var pkgs []pkg.Package
+	for _, dbLocation := range dbFileMatches {
+		dbContents, err := resolver.FileContentsByLocation(dbLocation)
+		if err != nil {
+			return nil, err
+		}
+
+		pkgs, err = parseDpkgStatus(strings.NewReader(dbContents))
+		if err != nil {
+			return nil, fmt.Errorf("unable to catalog dpkg package=%+v: %w", dbLocation.Path, err)
+		}
+
+		md5ContentsByName, md5RefsByName, err := fetchMd5Contents(resolver, dbLocation, pkgs)
+		if err != nil {
+			return nil, fmt.Errorf("unable to find dpkg md5 contents: %w", err)
+		}
+
+		copyrightContentsByName, copyrightRefsByName, err := fetchCopyrightContents(resolver, dbLocation, pkgs)
+		if err != nil {
+			return nil, fmt.Errorf("unable to find dpkg copyright contents: %w", err)
+		}
+
+		for i := range pkgs {
+			p := &pkgs[i]
+			p.FoundBy = c.Name()
+			p.Locations = []source.Location{dbLocation}
+
+			metadata := p.Metadata.(pkg.DpkgMetadata)
+
+			if md5Reader, ok := md5ContentsByName[md5Key(*p)]; ok {
+				// attach the file list
+				metadata.Files = parseDpkgMD5Info(md5Reader)
+
+				// keep a record of the file where this was discovered
+				if ref, ok := md5RefsByName[md5Key(*p)]; ok {
+					p.Locations = append(p.Locations, ref)
+				}
+			} else {
+				// ensure the file list is an empty collection (not nil)
+				metadata.Files = make([]pkg.DpkgFileRecord, 0)
+			}
+
+			// persist alterations
+			p.Metadata = metadata
+
+			copyrightReader, ok := copyrightContentsByName[p.Name]
+			if ok {
+				// attach the licenses
+				p.Licenses = parseLicensesFromCopyright(copyrightReader)
+
+				// keep a record of the file where this was discovered
+				if ref, ok := copyrightRefsByName[p.Name]; ok {
+					p.Locations = append(p.Locations, ref)
+				}
+			}
+		}
+	}
+	return pkgs, nil
+}
+
+func fetchMd5Contents(resolver source.Resolver, dbLocation source.Location, pkgs []pkg.Package) (map[string]io.Reader, map[string]source.Location, error) {
+	// fetch all MD5 file contents. This approach is more efficient than fetching each MD5 file one at a time
+
+	var md5FileMatches []source.Location
+	var nameByRef = make(map[source.Location]string)
+	parentPath := filepath.Dir(dbLocation.Path)
+
+	for _, p := range pkgs {
+		// look for /var/lib/dpkg/info/NAME:ARCH.md5sums
+		name := md5Key(p)
+		md5sumPath := path.Join(parentPath, "info", name+md5sumsExt)
+		md5SumLocation := resolver.RelativeFileByPath(dbLocation, md5sumPath)
+
+		if md5SumLocation == nil {
+			// the most specific key did not work, fallback to just the name
+			// look for /var/lib/dpkg/info/NAME.md5sums
+			name := p.Name
+			md5sumPath := path.Join(parentPath, "info", name+md5sumsExt)
+			md5SumLocation = resolver.RelativeFileByPath(dbLocation, md5sumPath)
+		}
+		// we should have at least one reference
+		if md5SumLocation != nil {
+			md5FileMatches = append(md5FileMatches, *md5SumLocation)
+			nameByRef[*md5SumLocation] = name
+		}
+	}
+
+	// fetch the md5 contents
+	md5ContentsByLocation, err := resolver.MultipleFileContentsByLocation(md5FileMatches)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	// organize content results and refs by a combination of name and architecture
+	var contentsByName = make(map[string]io.Reader)
+	var refsByName = make(map[string]source.Location)
+	for location, contents := range md5ContentsByLocation {
+		name := nameByRef[location]
+		contentsByName[name] = strings.NewReader(contents)
+		refsByName[name] = location
+	}
+
+	return contentsByName, refsByName, nil
+}
+
+func fetchCopyrightContents(resolver source.Resolver, dbLocation source.Location, pkgs []pkg.Package) (map[string]io.Reader, map[string]source.Location, error) {
+	// fetch all copyright file contents. This approach is more efficient than fetching each copyright file one at a time
+
+	var copyrightFileMatches []source.Location
+	var nameByLocation = make(map[source.Location]string)
+	for _, p := range pkgs {
+		// look for /usr/share/docs/NAME/copyright files
+		name := p.Name
+		copyrightPath := path.Join(docsPath, name, "copyright")
+		copyrightLocation := resolver.RelativeFileByPath(dbLocation, copyrightPath)
+
+		// we may not have a copyright file for each package, ignore missing files
+		if copyrightLocation != nil {
+			copyrightFileMatches = append(copyrightFileMatches, *copyrightLocation)
+			nameByLocation[*copyrightLocation] = name
+		}
+	}
+
+	// fetch the copyright contents
+	copyrightContentsByLocation, err := resolver.MultipleFileContentsByLocation(copyrightFileMatches)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	// organize content results and refs by package name
+	var contentsByName = make(map[string]io.Reader)
+	var refsByName = make(map[string]source.Location)
+	for location, contents := range copyrightContentsByLocation {
+		name := nameByLocation[location]
+		contentsByName[name] = strings.NewReader(contents)
+		refsByName[name] = location
+	}
+
+	return contentsByName, refsByName, nil
+}
+
+func md5Key(p pkg.Package) string {
+	metadata := p.Metadata.(pkg.DpkgMetadata)
+
+	contentKey := p.Name
+	if metadata.Architecture != "" && metadata.Architecture != "all" {
+		contentKey = contentKey + ":" + metadata.Architecture
+	}
+	return contentKey
 }

syft/cataloger/deb/cataloger_test.go

@@ -0,0 +1,99 @@
+package deb
+
+import (
+	"testing"
+
+	"github.com/anchore/stereoscope/pkg/imagetest"
+	"github.com/anchore/syft/syft/pkg"
+	"github.com/anchore/syft/syft/source"
+	"github.com/go-test/deep"
+)
+
+func TestDpkgCataloger(t *testing.T) {
+
+	tests := []struct {
+		name     string
+		sources  map[string][]string
+		expected []pkg.Package
+	}{
+		{
+			name: "go-case",
+			sources: map[string][]string{
+				"libpam-runtime": {"/var/lib/dpkg/status", "/var/lib/dpkg/info/libpam-runtime.md5sums", "/usr/share/doc/libpam-runtime/copyright"},
+			},
+			expected: []pkg.Package{
+				{
+					Name:         "libpam-runtime",
+					Version:      "1.1.8-3.6",
+					FoundBy:      "dpkgdb-cataloger",
+					Licenses:     []string{"GPL-2", "LGPL-2.1"},
+					Type:         pkg.DebPkg,
+					MetadataType: pkg.DpkgMetadataType,
+					Metadata: pkg.DpkgMetadata{
+						Package:       "libpam-runtime",
+						Source:        "pam",
+						Version:       "1.1.8-3.6",
+						Architecture:  "all",
+						Maintainer:    "Steve Langasek <vorlon@debian.org>",
+						InstalledSize: 1016,
+						Files: []pkg.DpkgFileRecord{
+							{Path: "/lib/x86_64-linux-gnu/libz.so.1.2.11", MD5: "55f905631797551d4d936a34c7e73474"},
+							{Path: "/usr/share/doc/zlib1g/changelog.Debian.gz", MD5: "cede84bda30d2380217f97753c8ccf3a"},
+							{Path: "/usr/share/doc/zlib1g/changelog.gz", MD5: "f3c9dafa6da7992c47328b4464f6d122"},
+							{Path: "/usr/share/doc/zlib1g/copyright", MD5: "a4fae96070439a5209a62ae5b8017ab2"},
+						},
+					},
+				},
+			},
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+
+			img, cleanup := imagetest.GetFixtureImage(t, "docker-archive", "image-dpkg")
+			defer cleanup()
+
+			s, err := source.NewFromImage(img, source.AllLayersScope, "")
+			if err != nil {
+				t.Fatal(err)
+			}
+
+			c := NewDpkgdbCataloger()
+
+			actual, err := c.Catalog(s.Resolver)
+			if err != nil {
+				t.Fatalf("failed to catalog: %+v", err)
+			}
+
+			if len(actual) != len(test.expected) {
+				for _, a := range actual {
+					t.Logf("   %+v", a)
+				}
+				t.Fatalf("unexpected package count: %d!=%d", len(actual), len(test.expected))
+			}
+
+			// test sources...
+			for idx := range actual {
+				a := &actual[idx]
+				// we will test the sources separately
+				var sourcesList = make([]string, len(a.Locations))
+				for i, s := range a.Locations {
+					sourcesList[i] = s.Path
+				}
+				a.Locations = nil
+
+				for _, d := range deep.Equal(sourcesList, test.sources[a.Name]) {
+					t.Errorf("diff: %+v", d)
+				}
+			}
+
+			// test remaining fields...
+			for _, d := range deep.Equal(actual, test.expected) {
+				t.Errorf("diff: %+v", d)
+			}
+
+		})
+	}
+
+}

syft/cataloger/deb/parse_copyright.go

@@ -0,0 +1,47 @@
+package deb
+
+import (
+	"bufio"
+	"io"
+	"regexp"
+	"sort"
+	"strings"
+
+	"github.com/anchore/syft/internal"
+)
+
+// For more information see: https://www.debian.org/doc/packaging-manuals/copyright-format/1.0/#license-syntax
+
+var licensePattern = regexp.MustCompile(`^License: (?P<license>\S*)`)
+
+func parseLicensesFromCopyright(reader io.Reader) []string {
+	findings := internal.NewStringSet()
+	scanner := bufio.NewScanner(reader)
+
+	for scanner.Scan() {
+		line := scanner.Text()
+
+		matchesByGroup := internal.MatchCaptureGroups(licensePattern, line)
+		if len(matchesByGroup) > 0 {
+			candidate, ok := matchesByGroup["license"]
+			if !ok {
+				continue
+			}
+
+			candidate = strings.TrimSpace(candidate)
+			if strings.Contains(candidate, " or ") || strings.Contains(candidate, " and ") {
+				// this is a multi-license summary, ignore this as other recurrent license lines should cover this
+				continue
+			}
+			if candidate != "" && strings.ToLower(candidate) != "none" {
+				findings.Add(candidate)
+			}
+		}
+	}
+
+	results := findings.ToSlice()
+
+	sort.Strings(results)
+
+	return results
+}

syft/cataloger/deb/parse_copyright_test.go

@@ -0,0 +1,63 @@
+package deb
+
+import (
+	"os"
+	"testing"
+
+	"github.com/go-test/deep"
+)
+
+func TestParseLicensesFromCopyright(t *testing.T) {
+	tests := []struct {
+		fixture  string
+		expected []string
+	}{
+		{
+			fixture:  "test-fixtures/copyright/trilicense",
+			expected: []string{"GPL-2", "LGPL-2.1", "MPL-1.1"},
+		},
+		{
+			fixture:  "test-fixtures/copyright/liblzma5",
+			expected: []string{"Autoconf", "GPL-2", "GPL-2+", "LGPL-2.1+", "PD", "PD-debian", "config-h", "noderivs", "permissive-fsf", "permissive-nowarranty", "probably-PD"},
+		},
+		{
+			fixture:  "test-fixtures/copyright/libaudit-common",
+			expected: []string{"GPL-2", "LGPL-2.1"},
+		},
+		{
+			fixture: "test-fixtures/copyright/python",
+			// note: this should not capture #, Permission, This, see ... however it's not clear how to fix this (this is probably good enough)
+			expected: []string{"#", "Apache", "Apache-2", "Apache-2.0", "Expat", "ISC", "LGPL-2.1+", "PSF-2", "Permission", "Python", "This", "see"},
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.fixture, func(t *testing.T) {
+			file, err := os.Open(test.fixture)
+			if err != nil {
+				t.Fatal("Unable to read: ", err)
+			}
+			defer func() {
+				err := file.Close()
+				if err != nil {
+					t.Fatal("closing file failed:", err)
+				}
+			}()
+
+			actual := parseLicensesFromCopyright(file)
+
+			if len(actual) != len(test.expected) {
+				for _, a := range actual {
+					t.Logf("   %+v", a)
+				}
+				t.Fatalf("unexpected package count: %d!=%d", len(actual), len(test.expected))
+			}
+
+			diffs := deep.Equal(actual, test.expected)
+			for _, d := range diffs {
+				t.Errorf("diff: %+v", d)
+			}
+
+		})
+	}
+}

syft/cataloger/deb/parse_dpkg_info_files.go

@@ -0,0 +1,31 @@
+package deb
+
+import (
+	"bufio"
+	"io"
+	"strings"
+
+	"github.com/anchore/syft/syft/pkg"
+)
+
+func parseDpkgMD5Info(reader io.Reader) []pkg.DpkgFileRecord {
+	// we must preallocate to ensure the resulting struct does not have null
+	var findings = make([]pkg.DpkgFileRecord, 0)
+	scanner := bufio.NewScanner(reader)
+
+	for scanner.Scan() {
+		line := scanner.Text()
+		fields := strings.SplitN(line, " ", 2)
+		if len(fields) == 2 {
+			path := strings.TrimSpace(fields[1])
+			if !strings.HasPrefix(path, "/") {
+				path = "/" + path
+			}
+			findings = append(findings, pkg.DpkgFileRecord{
+				Path: path,
+				MD5:  strings.TrimSpace(fields[0]),
+			})
+		}
+	}
+	return findings
+}

syft/cataloger/deb/parse_dpkg_info_files_test.go

@@ -0,0 +1,57 @@
+package deb
+
+import (
+	"os"
+	"testing"
+
+	"github.com/go-test/deep"
+
+	"github.com/anchore/syft/syft/pkg"
+)
+
+func TestMD5SumInfoParsing(t *testing.T) {
+	tests := []struct {
+		fixture  string
+		expected []pkg.DpkgFileRecord
+	}{
+		{
+			fixture: "test-fixtures/info/zlib1g.md5sums",
+			expected: []pkg.DpkgFileRecord{
+				{Path: "/lib/x86_64-linux-gnu/libz.so.1.2.11", MD5: "55f905631797551d4d936a34c7e73474"},
+				{Path: "/usr/share/doc/zlib1g/changelog.Debian.gz", MD5: "cede84bda30d2380217f97753c8ccf3a"},
+				{Path: "/usr/share/doc/zlib1g/changelog.gz", MD5: "f3c9dafa6da7992c47328b4464f6d122"},
+				{Path: "/usr/share/doc/zlib1g/copyright", MD5: "a4fae96070439a5209a62ae5b8017ab2"},
+			},
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.fixture, func(t *testing.T) {
+			file, err := os.Open(test.fixture)
+			if err != nil {
+				t.Fatal("Unable to read: ", err)
+			}
+			defer func() {
+				err := file.Close()
+				if err != nil {
+					t.Fatal("closing file failed:", err)
+				}
+			}()
+
+			actual := parseDpkgMD5Info(file)
+
+			if len(actual) != len(test.expected) {
+				for _, a := range actual {
+					t.Logf("   %+v", a)
+				}
+				t.Fatalf("unexpected package count: %d!=%d", len(actual), len(test.expected))
+			}
+
+			diffs := deep.Equal(actual, test.expected)
+			for _, d := range diffs {
+				t.Errorf("diff: %+v", d)
+			}
+
+		})
+	}
+}

syft/cataloger/deb/parse_dpkg_status.go

@@ -2,54 +2,64 @@ package deb
 
 import (
 	"bufio"
+	"errors"
 	"fmt"
 	"io"
+	"regexp"
+	"strconv"
 	"strings"
 
-	"github.com/anchore/syft/syft/cataloger/common"
+	"github.com/anchore/syft/internal"
+
 	"github.com/anchore/syft/syft/pkg"
 	"github.com/mitchellh/mapstructure"
 )
 
-// integrity check
-var _ common.ParserFn = parseDpkgStatus
-
 var errEndOfPackages = fmt.Errorf("no more packages to read")
 
 // parseDpkgStatus is a parser function for Debian DB status contents, returning all Debian packages listed.
-func parseDpkgStatus(_ string, reader io.Reader) ([]pkg.Package, error) {
+func parseDpkgStatus(reader io.Reader) ([]pkg.Package, error) {
 	buffedReader := bufio.NewReader(reader)
 	var packages = make([]pkg.Package, 0)
 
-	for {
+	continueProcessing := true
+	for continueProcessing {
 		entry, err := parseDpkgStatusEntry(buffedReader)
 		if err != nil {
-			if err == errEndOfPackages {
-				break
+			if errors.Is(err, errEndOfPackages) {
+				continueProcessing = false
+			} else {
+				return nil, err
 			}
-			return nil, err
 		}
-		packages = append(packages, pkg.Package{
-			Name:     entry.Package,
-			Version:  entry.Version,
-			Type:     pkg.DebPkg,
-			Metadata: entry,
-		})
+
+		if entry.Package != "" {
+			packages = append(packages, pkg.Package{
+				Name:         entry.Package,
+				Version:      entry.Version,
+				Type:         pkg.DebPkg,
+				MetadataType: pkg.DpkgMetadataType,
+				Metadata:     entry,
+			})
+		}
 	}
 
 	return packages, nil
 }
 
 // parseDpkgStatusEntry returns an individual Dpkg entry, or returns errEndOfPackages if there are no more packages to parse from the reader.
+// nolint:funlen
 func parseDpkgStatusEntry(reader *bufio.Reader) (entry pkg.DpkgMetadata, err error) {
-	dpkgFields := make(map[string]string)
+	dpkgFields := make(map[string]interface{})
+	var retErr error
 	var key string
 
 	for {
 		line, err := reader.ReadString('\n')
 		if err != nil {
 			if err == io.EOF {
-				return pkg.DpkgMetadata{}, errEndOfPackages
+				retErr = errEndOfPackages
+				break
 			}
 			return pkg.DpkgMetadata{}, err
 		}
@@ -81,18 +91,16 @@ func parseDpkgStatusEntry(reader *bufio.Reader) (entry pkg.DpkgMetadata, err err
 			dpkgFields[key] = val
 		default:
 			// parse a new key
-			if i := strings.Index(line, ":"); i > 0 {
-				key = strings.TrimSpace(line[0:i])
-				val := strings.TrimSpace(line[i+1:])
-
-				if _, ok := dpkgFields[key]; ok {
-					return pkg.DpkgMetadata{}, fmt.Errorf("duplicate key discovered: %s", key)
-				}
-
-				dpkgFields[key] = val
-			} else {
-				return pkg.DpkgMetadata{}, fmt.Errorf("cannot parse field from line: '%s'", line)
+			var val interface{}
+			key, val, err = handleNewKeyValue(line)
+			if err != nil {
+				return pkg.DpkgMetadata{}, err
 			}
+
+			if _, ok := dpkgFields[key]; ok {
+				return pkg.DpkgMetadata{}, fmt.Errorf("duplicate key discovered: %s", key)
+			}
+			dpkgFields[key] = val
 		}
 	}
 
@@ -101,5 +109,49 @@ func parseDpkgStatusEntry(reader *bufio.Reader) (entry pkg.DpkgMetadata, err err
 		return pkg.DpkgMetadata{}, err
 	}
 
-	return entry, nil
+	name, version := extractSourceVersion(entry.Source)
+	if version != "" {
+		entry.SourceVersion = version
+		entry.Source = name
+	}
+
+	return entry, retErr
+}
+
+// match examples:
+// "a-thing (1.2.3)"     name="a-thing" version="1.2.3"
+// "a-thing"             name="a-thing" version=""
+// ""                    name=""        version=""
+var sourceRegexp = regexp.MustCompile(`(?P<name>\S+)( \((?P<version>.*)\))?`)
+
+// If the source entry string is of the form "<name> (<version>)" then parse and return the components, if
+// of the "<name>" form, then return name and nil
+func extractSourceVersion(source string) (string, string) {
+	// special handling for the Source field since it has formatted data
+	match := internal.MatchCaptureGroups(sourceRegexp, source)
+	return match["name"], match["version"]
+}
+
+// handleNewKeyValue parse a new key-value pair from the given unprocessed line
+func handleNewKeyValue(line string) (string, interface{}, error) {
+	if i := strings.Index(line, ":"); i > 0 {
+		var key = strings.TrimSpace(line[0:i])
+		// mapstruct cant handle "-"
+		key = strings.ReplaceAll(key, "-", "")
+		val := strings.TrimSpace(line[i+1:])
+
+		// further processing of values based on the key that was discovered
+		switch key {
+		case "InstalledSize":
+			numVal, err := strconv.Atoi(val)
+			if err != nil {
+				return "", nil, fmt.Errorf("bad installed-size value=%q: %w", val, err)
+			}
+			return key, numVal, nil
+		default:
+			return key, val, nil
+		}
+	}
+
+	return "", nil, fmt.Errorf("cannot parse field from line: '%s'", line)
 }

syft/cataloger/deb/parse_dpkg_status_test.go

@@ -24,17 +24,19 @@ func TestSinglePackage(t *testing.T) {
 		{
 			name: "Test Single Package",
 			expected: pkg.DpkgMetadata{
-				Package:      "apt",
-				Source:       "apt-dev",
-				Version:      "1.8.2",
-				Architecture: "amd64",
+				Package:       "apt",
+				Source:        "apt-dev",
+				Version:       "1.8.2",
+				Architecture:  "amd64",
+				InstalledSize: 4064,
+				Maintainer:    "APT Development Team <deity@lists.debian.org>",
 			},
 		},
 	}
 
 	for _, test := range tests {
 		t.Run(test.name, func(t *testing.T) {
-			file, err := os.Open("test-fixtures/single")
+			file, err := os.Open("test-fixtures/status/single")
 			if err != nil {
 				t.Fatal("Unable to read test_fixtures/single: ", err)
 			}
@@ -66,15 +68,19 @@ func TestMultiplePackages(t *testing.T) {
 			name: "Test Multiple Package",
 			expected: []pkg.DpkgMetadata{
 				{
-					Package:      "tzdata",
-					Version:      "2020a-0+deb10u1",
-					Source:       "tzdata-dev",
-					Architecture: "all",
+					Package:       "tzdata",
+					Version:       "2020a-0+deb10u1",
+					Source:        "tzdata-dev",
+					Architecture:  "all",
+					InstalledSize: 3036,
+					Maintainer:    "GNU Libc Maintainers <debian-glibc@lists.debian.org>",
 				},
 				{
-					Package:      "util-linux",
-					Version:      "2.33.1-0.1",
-					Architecture: "amd64",
+					Package:       "util-linux",
+					Version:       "2.33.1-0.1",
+					Architecture:  "amd64",
+					InstalledSize: 4327,
+					Maintainer:    "LaMont Jones <lamont@debian.org>",
 				},
 			},
 		},
@@ -82,7 +88,7 @@ func TestMultiplePackages(t *testing.T) {
 
 	for _, test := range tests {
 		t.Run(test.name, func(t *testing.T) {
-			file, err := os.Open("test-fixtures/multiple")
+			file, err := os.Open("test-fixtures/status/multiple")
 			if err != nil {
 				t.Fatal("Unable to read: ", err)
 			}
@@ -93,7 +99,7 @@ func TestMultiplePackages(t *testing.T) {
 				}
 			}()
 
-			pkgs, err := parseDpkgStatus(file.Name(), file)
+			pkgs, err := parseDpkgStatus(file)
 			if err != nil {
 				t.Fatal("Unable to read file contents: ", err)
 			}
@@ -109,3 +115,43 @@ func TestMultiplePackages(t *testing.T) {
 		})
 	}
 }
+
+func TestSourceVersionExtract(t *testing.T) {
+
+	tests := []struct {
+		name     string
+		input    string
+		expected []string
+	}{
+		{
+			name:     "name and version",
+			input:    "test (1.2.3)",
+			expected: []string{"test", "1.2.3"},
+		},
+		{
+			name:     "only name",
+			input:    "test",
+			expected: []string{"test", ""},
+		},
+		{
+			name:     "empty",
+			input:    "",
+			expected: []string{"", ""},
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			name, version := extractSourceVersion(test.input)
+
+			if name != test.expected[0] {
+				t.Errorf("mismatch name for %q : %q!=%q", test.input, name, test.expected[0])
+			}
+
+			if version != test.expected[1] {
+				t.Errorf("mismatch version for %q : %q!=%q", test.input, version, test.expected[1])
+			}
+
+		})
+	}
+}

syft/cataloger/deb/test-fixtures/copyright/libaudit-common

@@ -0,0 +1,43 @@
+Format: http://www.debian.org/doc/packaging-manuals/copyright-format/1.0/
+Upstream-Name: auditd
+Source: https://people.redhat.com/sgrubb/audit/
+
+Files: *
+Copyright: 2012-2016 Steve Grubb <sgrubb@redhat.com>
+           2006-2012 Rik Faith
+License: GPL-2
+
+Files: src/libev/*
+Copyright: 2007-2009 Marc Alexamder Lehmann
+License: GPL-2
+
+Files: lib/*
+Copyright: 2005-2008 Steve Grubb <sgrubb@redhat.com>
+License: LGPL-2.1
+  The audit daemon's library libaudit.* is released under LGPL
+  so that it may be linked with 3rd party software.
+  .
+  On Debian systems, refer to /usr/share/common-licenses/LGPL-2.1
+  for the complete text of the GNU Lesser General Public License.
+
+Files: debian/*
+Copyright: 2007-2011 Philipp Matthias Hahn <pmhahn@debian.org>
+           2012-2016 Laurent Bigonville <bigon@debian.org>
+License: GPL-2
+
+License: GPL-2
+ This package is free software; you can redistribute it and/or modify
+ it under the terms of the GNU General Public License version 2,
+ as published by the Free Software Foundation.
+ .
+ This package is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ GNU General Public License for more details.
+ .
+ You should have received a copy of the GNU General Public License
+ along with this package; if not, write to the Free Software
+ Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301 USA
+ .
+ On Debian systems, the complete text of the GNU General
+ Public License can be found in `/usr/share/common-licenses/GPL-1'.

syft/cataloger/deb/test-fixtures/copyright/liblzma5

@@ -0,0 +1,383 @@
+Format: http://www.debian.org/doc/packaging-manuals/copyright-format/1.0/
+Upstream-Name: XZ Utils
+Upstream-Contact:
+ Lasse Collin <lasse.collin@tukaani.org>
+ http://tukaani.org/xz/lists.html
+Source:
+ http://tukaani.org/xz
+ http://git.tukaani.org/xz.git
+Comment:
+ XZ Utils is developed and maintained upstream by Lasse Collin.  Major
+ portions are based on code by other authors; see AUTHORS for details.
+ Most of the source has been put into the public domain, but some files
+ have not (details below).
+ .
+ This file describes the source package.  The binary packages contain
+ some files derived from other works: for example, images in the API
+ documentation come from Doxygen.
+License:
+ Different licenses apply to different files in this package. Here
+ is a rough summary of which licenses apply to which parts of this
+ package (but check the individual files to be sure!):
+ .
+   - liblzma is in the public domain.
+ .
+   - xz, xzdec, and lzmadec command line tools are in the public
+     domain unless GNU getopt_long had to be compiled and linked
+     in from the lib directory. The getopt_long code is under
+     GNU LGPLv2.1+.
+ .
+   - The scripts to grep, diff, and view compressed files have been
+     adapted from gzip. These scripts and their documentation are
+     under GNU GPLv2+.
+ .
+   - All the documentation in the doc directory and most of the
+     XZ Utils specific documentation files in other directories
+     are in the public domain.
+ .
+   - Translated messages are in the public domain.
+ .
+   - The build system contains public domain files, and files that
+     are under GNU GPLv2+ or GNU GPLv3+. None of these files end up
+     in the binaries being built.
+ .
+   - Test files and test code in the tests directory, and debugging
+     utilities in the debug directory are in the public domain.
+ .
+   - The extra directory may contain public domain files, and files
+     that are under various free software licenses.
+ .
+ You can do whatever you want with the files that have been put into
+ the public domain. If you find public domain legally problematic,
+ take the previous sentence as a license grant. If you still find
+ the lack of copyright legally problematic, you have too many
+ lawyers.
+ .
+ As usual, this software is provided "as is", without any warranty.
+ .
+ If you copy significant amounts of public domain code from XZ Utils
+ into your project, acknowledging this somewhere in your software is
+ polite (especially if it is proprietary, non-free software), but
+ naturally it is not legally required. Here is an example of a good
+ notice to put into "about box" or into documentation:
+ .
+     This software includes code from XZ Utils <http://tukaani.org/xz/>.
+ .
+ The following license texts are included in the following files:
+   - COPYING.LGPLv2.1: GNU Lesser General Public License version 2.1
+   - COPYING.GPLv2: GNU General Public License version 2
+   - COPYING.GPLv3: GNU General Public License version 3
+ .
+ Note that the toolchain (compiler, linker etc.) may add some code
+ pieces that are copyrighted. Thus, it is possible that e.g. liblzma
+ binary wouldn't actually be in the public domain in its entirety
+ even though it contains no copyrighted code from the XZ Utils source
+ package.
+ .
+ If you have questions, don't hesitate to ask the author(s) for more
+ information.
+
+Files: *
+Copyright: 2006-2012, Lasse Collin
+           1999-2008, Igor Pavlov
+           2006, Ville Koskinen
+           1998, Steve Reid
+           2000, Wei Dai
+           2003, Kevin Springle
+           2009, Jonathan Nieder
+           2010, Anders F Björklund
+License: PD
+ This file has been put in the public domain.
+ You can do whatever you want with this file.
+Comment:
+  From: Lasse Collin <lasse.collin@tukaani.org>
+  To: Jonathan Nieder <jrnieder@gmail.com>
+  Subject: Re: XZ utils for Debian
+  Date: Sun, 19 Jul 2009 13:28:23 +0300
+  Message-Id: <200907191328.23816.lasse.collin@tukaani.org>
+ .
+ [...]
+ .
+  > AUTHORS, ChangeLog, COPYING, README, THANKS, TODO,
+  > dos/README, windows/README
+ .
+  COPYING says that most docs are in the public domain. Maybe that's not
+  clear enough, but on the other hand it looks a bit stupid to put
+  copyright information in tiny and relatively small docs like README.
+ .
+  I don't dare to say that _all_ XZ Utils specific docs are in the public
+  domain unless otherwise mentioned in the file. I'm including PDF files
+  generated by groff + ps2pdf, and some day I might include Doxygen-
+  generated HTML docs too. Those don't include any copyright notices, but
+  it seems likely that groff + ps2pdf or at least Doxygen put some
+  copyrighted content into the generated files.
+
+Files: INSTALL NEWS PACKAGERS
+ windows/README-Windows.txt
+ windows/INSTALL-Windows.txt
+Copyright: 2009-2010, Lasse Collin
+License: probably-PD
+ See the note on AUTHORS, README, and so on above.
+
+Files: src/scripts/* lib/* extra/scanlzma/scanlzma.c
+Copyright: © 1993, Jean-loup Gailly
+           © 1989-1994, 1996-1999, 2001-2007, Free Software Foundation, Inc.
+           © 2006 Timo Lindfors
+           2005, Charles Levert
+           2005, 2009, Lasse Collin
+           2009, Andrew Dudman
+Other-Authors: Paul Eggert, Ulrich Drepper
+License: GPL-2+
+
+Files: src/scripts/Makefile.am src/scripts/xzless.1
+Copyright: 2009, Andrew Dudman
+           2009, Lasse Collin
+License: PD
+ This file has been put in the public domain.
+ You can do whatever you want with this file.
+
+Files: doc/examples/xz_pipe_comp.c doc/examples/xz_pipe_decomp.c
+Copyright: 2010, Daniel Mealha Cabrita
+License: PD
+ Not copyrighted -- provided to the public domain.
+
+Files: lib/getopt.c lib/getopt1.c lib/getopt.in.h
+Copyright: © 1987-2007 Free Software Foundation, Inc.
+Other-Authors: Ulrich Drepper
+License: LGPL-2.1+
+
+Files: m4/getopt.m4 m4/posix-shell.m4
+Copyright: © 2002-2006, 2008 Free Software Foundation, Inc.
+           © 2007-2008 Free Software Foundation, Inc.
+Other-Authors: Bruno Haible, Paul Eggert
+License: permissive-fsf
+
+Files: m4/acx_pthread.m4
+Copyright: © 2008, Steven G. Johnson <stevenj@alum.mit.edu>
+License: Autoconf
+
+Files: Doxyfile.in
+Copyright: © 1997-2007 by Dimitri van Heesch
+Origin: Doxygen 1.4.7
+License: GPL-2
+
+Files: src/liblzma/check/crc32_table_?e.h
+ src/liblzma/check/crc64_table_?e.h
+ src/liblzma/lzma/fastpos_table.c
+ src/liblzma/rangecoder/price_table.c
+Copyright: none, automatically generated data
+Generated-With:
+ src/liblzma/check/crc32_tablegen.c
+ src/liblzma/check/crc64_tablegen.c
+ src/liblzma/lzma/fastpos_tablegen.c
+ src/liblzma/rangecoder/price_tablegen.c
+License: none
+ No copyright to license.
+
+Files: .gitignore m4/.gitignore po/.gitignore po/LINGUAS po/POTFILES.in
+Copyright: none; these are just short lists.
+License: none
+ No copyright to license.
+
+Files: tests/compress_prepared_bcj_*
+Copyright: 2008-2009, Lasse Collin
+Source-Code: tests/bcj_test.c
+License: PD
+ This file has been put into the public domain.
+ You can do whatever you want with this file.
+Comment:
+ changelog.gz (commit 975d8fd) explains:
+ .
+ Recreated the BCJ test files for x86 and SPARC. The old files
+ were linked with crt*.o, which are copyrighted, and thus the
+ old test files were not in the public domain as a whole. They
+ are freely distributable though, but it is better to be careful
+ and avoid including any copyrighted pieces in the test files.
+ The new files are just compiled and assembled object files,
+ and thus don't contain any copyrighted code.
+
+Files: po/cs.po po/de.po po/fr.po
+Copyright: 2010, Marek Černocký
+           2010, Andre Noll
+           2011, Adrien Nader
+License: PD
+ This file is put in the public domain.
+
+Files: po/it.po po/pl.po
+Copyright: 2009, 2010, Gruppo traduzione italiano di Ubuntu-it
+           2010, Lorenzo De Liso
+           2009, 2010, 2011, Milo Casagrande
+           2011, Jakub Bogusz
+License: PD
+ This file is in the public domain
+
+Files: INSTALL.generic
+Copyright: © 1994, 1995, 1996, 1999, 2000, 2001, 2002, 2004, 2005,
+             2006, 2007, 2008, 2009, 2010 Free Software Foundation, Inc.
+License: permissive-nowarranty
+
+Files: dos/config.h
+Copyright: © 1992, 1993, 1994, 1999, 2000, 2001, 2002, 2005
+             Free Software Foundation, Inc.
+           2007-2010, Lasse Collin
+Other-Authors: Roland McGrath, Akim Demaille, Paul Eggert,
+               David Mackenzie, Bruno Haible, and many others.
+Origin: configure.ac from XZ Utils,
+        visibility.m4 serial 1 (gettext-0.15),
+        Autoconf 2.52g
+License: config-h
+ configure.ac:
+ .
+  # Author: Lasse Collin
+  #
+  # This file has been put into the public domain.
+  # You can do whatever you want with this file.
+ .
+ visibility.m4:
+ .
+  dnl Copyright (C) 2005 Free Software Foundation, Inc.
+  dnl This file is free software; the Free Software Foundation
+  dnl gives unlimited permission to copy and/or distribute it,
+  dnl with or without modifications, as long as this notice is preserved.
+ .
+ dnl From Bruno Haible.
+ .
+ comments from Autoconf 2.52g:
+ .
+  # Copyright 1992, 1993, 1994, 1999, 2000, 2001, 2002
+  # Free Software Foundation, Inc.
+ .
+ [...]
+ .
+  # As a special exception, the Free Software Foundation gives unlimited
+  # permission to copy, distribute and modify the configure scripts that
+  # are the output of Autoconf.  You need not follow the terms of the GNU
+  # General Public License when using or distributing such scripts, even
+  # though portions of the text of Autoconf appear in them.  The GNU
+  # General Public License (GPL) does govern all other use of the material
+  # that constitutes the Autoconf program.
+ .
+ On Debian systems, the complete text of the GNU General Public
+ License version 2 can be found in ‘/usr/share/common-licenses/GPL-2’.
+ dos/config.h was generated with autoheader, which tells Autoconf to
+ output a script to generate a config.h file and then runs it.
+
+Files: po/Makevars
+Origin: gettext-runtime/po/Makevars (gettext-0.12)
+Copyright: © 2003 Free Software Foundation, Inc.
+Authors: Bruno Haible
+License: LGPL-2.1+
+ The gettext-runtime package is under the LGPL, see files intl/COPYING.LIB-2.0
+ and intl/COPYING.LIB-2.1.
+ .
+ On Debian systems, the complete text of intl/COPYING.LIB-2.0 from
+ gettext-runtime 0.12 can be found in ‘/usr/share/common-licenses/LGPL-2’
+ and the text of intl/COPYING.LIB-2.1 can be found in
+ ‘/usr/share/common-licenses/LGPL-2.1’.
+ .
+ po/Makevars consists mostly of helpful comments and does not contain a
+ copyright and license notice.
+
+Files: COPYING.GPLv2 COPYING.GPLv3 COPYING.LGPLv2.1
+Copyright: © 1989, 1991, 1999, 2007 Free Software Foundation, Inc.
+License: noderivs
+ Everyone is permitted to copy and distribute verbatim copies
+ of this license document, but changing it is not allowed.
+
+Files: debian/*
+Copyright: 2009-2012, Jonathan Nieder
+License: PD-debian
+ The Debian packaging files are in the public domain.
+ You may freely use, modify, distribute, and relicense them.
+
+License: LGPL-2.1+
+ This program is free software; you can redistribute it and/or modify
+ it under the terms of the GNU Lesser General Public License as published by
+ the Free Software Foundation; either version 2.1, or (at your option)
+ any later version.
+ .
+ This program is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ GNU Lesser General Public License for more details.
+ .
+ You should have received a copy of the GNU Lesser General Public License along
+ with this program; if not, write to the Free Software Foundation,
+ Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+ .
+ On Debian systems, the complete text of the GNU Lesser General Public
+ License version 2.1 can be found in ‘/usr/share/common-licenses/LGPL-2.1’.
+
+License: GPL-2
+ Permission to use, copy, modify, and distribute this software and its
+ documentation under the terms of the GNU General Public License is
+ hereby granted. No representations are made about the suitability of
+ this software for any purpose. It is provided "as is" without express
+ or implied warranty. See the GNU General Public License for more
+ details.
+ .
+ Documents produced by doxygen are derivative works derived from the
+ input used in their production; they are not affected by this license.
+ .
+ On Debian systems, the complete text of the version of the GNU General
+ Public License distributed with Doxygen can be found in
+ ‘/usr/share/common-licenses/GPL-2’.
+
+License: GPL-2+
+ This program is free software; you can redistribute it and/or modify
+ it under the terms of the GNU General Public License as published by
+ the Free Software Foundation; either version 2, or (at your option)
+ any later version.
+ .
+ This program is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ GNU General Public License for more details.
+ .
+ You should have received a copy of the GNU General Public License along
+ with this program; if not, write to the Free Software Foundation,
+ Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+ .
+ On Debian systems, the complete text of the GNU General Public License
+ version 2 can be found in ‘/usr/share/common-licenses/GPL-2’.
+
+License: Autoconf
+ This program is free software: you can redistribute it and/or modify it
+ under the terms of the GNU General Public License as published by the
+ Free Software Foundation, either version 3 of the License, or (at your
+ option) any later version.
+ .
+ This program is distributed in the hope that it will be useful, but
+ WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General
+ Public License for more details.
+ .
+ You should have received a copy of the GNU General Public License along
+ with this program. If not, see <http://www.gnu.org/licenses/>.
+ .
+ As a special exception, the respective Autoconf Macro's copyright owner
+ gives unlimited permission to copy, distribute and modify the configure
+ scripts that are the output of Autoconf when processing the Macro. You
+ need not follow the terms of the GNU General Public License when using
+ or distributing such scripts, even though portions of the text of the
+ Macro appear in them. The GNU General Public License (GPL) does govern
+ all other use of the material that constitutes the Autoconf Macro.
+ .
+ This special exception to the GPL applies to versions of the Autoconf
+ Macro released by the Autoconf Archive. When you make and distribute a
+ modified version of the Autoconf Macro, you may extend this special
+ exception to the GPL to apply to your modified version as well.
+ .
+ On Debian systems, the complete text of the GNU General Public
+ License version 3 can be found in ‘/usr/share/common-licenses/GPL-3’.
+
+License: permissive-fsf
+ This file is free software; the Free Software Foundation
+ gives unlimited permission to copy and/or distribute it,
+ with or without modifications, as long as this notice is preserved.
+
+License: permissive-nowarranty
+ Copying and distribution of this file, with or without modification,
+ are permitted in any medium without royalty provided the copyright
+ notice and this notice are preserved.  This file is offered as-is,
+ without warranty of any kind.

syft/cataloger/deb/test-fixtures/copyright/python

@@ -0,0 +1,958 @@
+This package was put together by Klee Dienes <klee@debian.org> from 
+sources from ftp.python.org:/pub/python, based on the Debianization by 
+the previous maintainers Bernd S. Brentrup <bsb@uni-muenster.de> and 
+Bruce Perens. Current maintainer is Matthias Klose <doko@debian.org>. 
+
+It was downloaded from http://python.org/
+
+Copyright:
+
+Upstream Author: Guido van Rossum <guido@cwi.nl> and others.
+
+License:
+
+The following text includes the Python license and licenses and
+acknowledgements for incorporated software. The licenses can be read
+in the HTML and texinfo versions of the documentation as well, after
+installing the pythonx.y-doc package. Licenses for files not licensed
+under the Python Licenses are found at the end of this file.
+
+
+Python License
+==============
+
+A. HISTORY OF THE SOFTWARE
+==========================
+
+Python was created in the early 1990s by Guido van Rossum at Stichting
+Mathematisch Centrum (CWI, see http://www.cwi.nl) in the Netherlands
+as a successor of a language called ABC.  Guido remains Python's
+principal author, although it includes many contributions from others.
+
+In 1995, Guido continued his work on Python at the Corporation for
+National Research Initiatives (CNRI, see http://www.cnri.reston.va.us)
+in Reston, Virginia where he released several versions of the
+software.
+
+In May 2000, Guido and the Python core development team moved to
+BeOpen.com to form the BeOpen PythonLabs team.  In October of the same
+year, the PythonLabs team moved to Digital Creations (now Zope
+Corporation, see http://www.zope.com).  In 2001, the Python Software
+Foundation (PSF, see http://www.python.org/psf/) was formed, a
+non-profit organization created specifically to own Python-related
+Intellectual Property.  Zope Corporation is a sponsoring member of
+the PSF.
+
+All Python releases are Open Source (see http://www.opensource.org for
+the Open Source Definition).  Historically, most, but not all, Python
+releases have also been GPL-compatible; the table below summarizes
+the various releases.
+
+    Release         Derived     Year        Owner       GPL-
+                    from                                compatible? (1)
+
+    0.9.0 thru 1.2              1991-1995   CWI         yes
+    1.3 thru 1.5.2  1.2         1995-1999   CNRI        yes
+    1.6             1.5.2       2000        CNRI        no
+    2.0             1.6         2000        BeOpen.com  no
+    1.6.1           1.6         2001        CNRI        yes (2)
+    2.1             2.0+1.6.1   2001        PSF         no
+    2.0.1           2.0+1.6.1   2001        PSF         yes
+    2.1.1           2.1+2.0.1   2001        PSF         yes
+    2.2             2.1.1       2001        PSF         yes
+    2.1.2           2.1.1       2002        PSF         yes
+    2.1.3           2.1.2       2002        PSF         yes
+    2.2.1           2.2         2002        PSF         yes
+    2.2.2           2.2.1       2002        PSF         yes
+    2.2.3           2.2.2       2003        PSF         yes
+    2.3             2.2.2       2002-2003   PSF         yes
+    2.3.1           2.3         2002-2003   PSF         yes
+    2.3.2           2.3.1       2002-2003   PSF         yes
+    2.3.3           2.3.2       2002-2003   PSF         yes
+    2.3.4           2.3.3       2004        PSF         yes
+    2.3.5           2.3.4       2005        PSF         yes
+    2.4             2.3         2004        PSF         yes
+    2.4.1           2.4         2005        PSF         yes
+    2.4.2           2.4.1       2005        PSF         yes
+    2.4.3           2.4.2       2006        PSF         yes
+    2.5             2.4         2006        PSF         yes
+    2.5.1           2.5         2007        PSF         yes
+    2.5.2           2.5.1       2008        PSF         yes
+    2.5.3           2.5.2       2008        PSF         yes
+    2.6             2.5         2008        PSF         yes
+    2.6.1           2.6         2008        PSF         yes
+    2.6.2           2.6.1       2009        PSF         yes
+    2.6.3           2.6.2       2009        PSF         yes
+    2.6.4           2.6.3       2009        PSF         yes
+    2.6.5           2.6.4       2010        PSF         yes
+    2.7             2.6         2010        PSF         yes
+
+Footnotes:
+
+(1) GPL-compatible doesn't mean that we're distributing Python under
+    the GPL.  All Python licenses, unlike the GPL, let you distribute
+    a modified version without making your changes open source.  The
+    GPL-compatible licenses make it possible to combine Python with
+    other software that is released under the GPL; the others don't.
+
+(2) According to Richard Stallman, 1.6.1 is not GPL-compatible,
+    because its license has a choice of law clause.  According to
+    CNRI, however, Stallman's lawyer has told CNRI's lawyer that 1.6.1
+    is "not incompatible" with the GPL.
+
+Thanks to the many outside volunteers who have worked under Guido's
+direction to make these releases possible.
+
+
+B. TERMS AND CONDITIONS FOR ACCESSING OR OTHERWISE USING PYTHON
+===============================================================
+
+PYTHON SOFTWARE FOUNDATION LICENSE VERSION 2
+--------------------------------------------
+
+1. This LICENSE AGREEMENT is between the Python Software Foundation
+("PSF"), and the Individual or Organization ("Licensee") accessing and
+otherwise using this software ("Python") in source or binary form and
+its associated documentation.
+
+2. Subject to the terms and conditions of this License Agreement, PSF
+hereby grants Licensee a nonexclusive, royalty-free, world-wide
+license to reproduce, analyze, test, perform and/or display publicly,
+prepare derivative works, distribute, and otherwise use Python
+alone or in any derivative version, provided, however, that PSF's
+License Agreement and PSF's notice of copyright, i.e., "Copyright (c)
+2001, 2002, 2003, 2004, 2005, 2006, 2007 Python Software Foundation; 
+All Rights Reserved" are retained in Python alone or in any derivative 
+version prepared by Licensee.
+
+3. In the event Licensee prepares a derivative work that is based on
+or incorporates Python or any part thereof, and wants to make
+the derivative work available to others as provided herein, then
+Licensee hereby agrees to include in any such work a brief summary of
+the changes made to Python.
+
+4. PSF is making Python available to Licensee on an "AS IS"
+basis.  PSF MAKES NO REPRESENTATIONS OR WARRANTIES, EXPRESS OR
+IMPLIED.  BY WAY OF EXAMPLE, BUT NOT LIMITATION, PSF MAKES NO AND
+DISCLAIMS ANY REPRESENTATION OR WARRANTY OF MERCHANTABILITY OR FITNESS
+FOR ANY PARTICULAR PURPOSE OR THAT THE USE OF PYTHON WILL NOT
+INFRINGE ANY THIRD PARTY RIGHTS.
+
+5. PSF SHALL NOT BE LIABLE TO LICENSEE OR ANY OTHER USERS OF PYTHON
+FOR ANY INCIDENTAL, SPECIAL, OR CONSEQUENTIAL DAMAGES OR LOSS AS
+A RESULT OF MODIFYING, DISTRIBUTING, OR OTHERWISE USING PYTHON,
+OR ANY DERIVATIVE THEREOF, EVEN IF ADVISED OF THE POSSIBILITY THEREOF.
+
+6. This License Agreement will automatically terminate upon a material
+breach of its terms and conditions.
+
+7. Nothing in this License Agreement shall be deemed to create any
+relationship of agency, partnership, or joint venture between PSF and
+Licensee.  This License Agreement does not grant permission to use PSF
+trademarks or trade name in a trademark sense to endorse or promote
+products or services of Licensee, or any third party.
+
+8. By copying, installing or otherwise using Python, Licensee
+agrees to be bound by the terms and conditions of this License
+Agreement.
+
+
+BEOPEN.COM LICENSE AGREEMENT FOR PYTHON 2.0
+-------------------------------------------
+
+BEOPEN PYTHON OPEN SOURCE LICENSE AGREEMENT VERSION 1
+
+1. This LICENSE AGREEMENT is between BeOpen.com ("BeOpen"), having an
+office at 160 Saratoga Avenue, Santa Clara, CA 95051, and the
+Individual or Organization ("Licensee") accessing and otherwise using
+this software in source or binary form and its associated
+documentation ("the Software").
+
+2. Subject to the terms and conditions of this BeOpen Python License
+Agreement, BeOpen hereby grants Licensee a non-exclusive,
+royalty-free, world-wide license to reproduce, analyze, test, perform
+and/or display publicly, prepare derivative works, distribute, and
+otherwise use the Software alone or in any derivative version,
+provided, however, that the BeOpen Python License is retained in the
+Software, alone or in any derivative version prepared by Licensee.
+
+3. BeOpen is making the Software available to Licensee on an "AS IS"
+basis.  BEOPEN MAKES NO REPRESENTATIONS OR WARRANTIES, EXPRESS OR
+IMPLIED.  BY WAY OF EXAMPLE, BUT NOT LIMITATION, BEOPEN MAKES NO AND
+DISCLAIMS ANY REPRESENTATION OR WARRANTY OF MERCHANTABILITY OR FITNESS
+FOR ANY PARTICULAR PURPOSE OR THAT THE USE OF THE SOFTWARE WILL NOT
+INFRINGE ANY THIRD PARTY RIGHTS.
+
+4. BEOPEN SHALL NOT BE LIABLE TO LICENSEE OR ANY OTHER USERS OF THE
+SOFTWARE FOR ANY INCIDENTAL, SPECIAL, OR CONSEQUENTIAL DAMAGES OR LOSS
+AS A RESULT OF USING, MODIFYING OR DISTRIBUTING THE SOFTWARE, OR ANY
+DERIVATIVE THEREOF, EVEN IF ADVISED OF THE POSSIBILITY THEREOF.
+
+5. This License Agreement will automatically terminate upon a material
+breach of its terms and conditions.
+
+6. This License Agreement shall be governed by and interpreted in all
+respects by the law of the State of California, excluding conflict of
+law provisions.  Nothing in this License Agreement shall be deemed to
+create any relationship of agency, partnership, or joint venture
+between BeOpen and Licensee.  This License Agreement does not grant
+permission to use BeOpen trademarks or trade names in a trademark
+sense to endorse or promote products or services of Licensee, or any
+third party.  As an exception, the "BeOpen Python" logos available at
+http://www.pythonlabs.com/logos.html may be used according to the
+permissions granted on that web page.
+
+7. By copying, installing or otherwise using the software, Licensee
+agrees to be bound by the terms and conditions of this License
+Agreement.
+
+
+CNRI LICENSE AGREEMENT FOR PYTHON 1.6.1
+---------------------------------------
+
+1. This LICENSE AGREEMENT is between the Corporation for National
+Research Initiatives, having an office at 1895 Preston White Drive,
+Reston, VA 20191 ("CNRI"), and the Individual or Organization
+("Licensee") accessing and otherwise using Python 1.6.1 software in
+source or binary form and its associated documentation.
+
+2. Subject to the terms and conditions of this License Agreement, CNRI
+hereby grants Licensee a nonexclusive, royalty-free, world-wide
+license to reproduce, analyze, test, perform and/or display publicly,
+prepare derivative works, distribute, and otherwise use Python 1.6.1
+alone or in any derivative version, provided, however, that CNRI's
+License Agreement and CNRI's notice of copyright, i.e., "Copyright (c)
+1995-2001 Corporation for National Research Initiatives; All Rights
+Reserved" are retained in Python 1.6.1 alone or in any derivative
+version prepared by Licensee.  Alternately, in lieu of CNRI's License
+Agreement, Licensee may substitute the following text (omitting the
+quotes): "Python 1.6.1 is made available subject to the terms and
+conditions in CNRI's License Agreement.  This Agreement together with
+Python 1.6.1 may be located on the Internet using the following
+unique, persistent identifier (known as a handle): 1895.22/1013.  This
+Agreement may also be obtained from a proxy server on the Internet
+using the following URL: http://hdl.handle.net/1895.22/1013".
+
+3. In the event Licensee prepares a derivative work that is based on
+or incorporates Python 1.6.1 or any part thereof, and wants to make
+the derivative work available to others as provided herein, then
+Licensee hereby agrees to include in any such work a brief summary of
+the changes made to Python 1.6.1.
+
+4. CNRI is making Python 1.6.1 available to Licensee on an "AS IS"
+basis.  CNRI MAKES NO REPRESENTATIONS OR WARRANTIES, EXPRESS OR
+IMPLIED.  BY WAY OF EXAMPLE, BUT NOT LIMITATION, CNRI MAKES NO AND
+DISCLAIMS ANY REPRESENTATION OR WARRANTY OF MERCHANTABILITY OR FITNESS
+FOR ANY PARTICULAR PURPOSE OR THAT THE USE OF PYTHON 1.6.1 WILL NOT
+INFRINGE ANY THIRD PARTY RIGHTS.
+
+5. CNRI SHALL NOT BE LIABLE TO LICENSEE OR ANY OTHER USERS OF PYTHON
+1.6.1 FOR ANY INCIDENTAL, SPECIAL, OR CONSEQUENTIAL DAMAGES OR LOSS AS
+A RESULT OF MODIFYING, DISTRIBUTING, OR OTHERWISE USING PYTHON 1.6.1,
+OR ANY DERIVATIVE THEREOF, EVEN IF ADVISED OF THE POSSIBILITY THEREOF.
+
+6. This License Agreement will automatically terminate upon a material
+breach of its terms and conditions.
+
+7. This License Agreement shall be governed by the federal
+intellectual property law of the United States, including without
+limitation the federal copyright law, and, to the extent such
+U.S. federal law does not apply, by the law of the Commonwealth of
+Virginia, excluding Virginia's conflict of law provisions.
+Notwithstanding the foregoing, with regard to derivative works based
+on Python 1.6.1 that incorporate non-separable material that was
+previously distributed under the GNU General Public License (GPL), the
+law of the Commonwealth of Virginia shall govern this License
+Agreement only as to issues arising under or with respect to
+Paragraphs 4, 5, and 7 of this License Agreement.  Nothing in this
+License Agreement shall be deemed to create any relationship of
+agency, partnership, or joint venture between CNRI and Licensee.  This
+License Agreement does not grant permission to use CNRI trademarks or
+trade name in a trademark sense to endorse or promote products or
+services of Licensee, or any third party.
+
+8. By clicking on the "ACCEPT" button where indicated, or by copying,
+installing or otherwise using Python 1.6.1, Licensee agrees to be
+bound by the terms and conditions of this License Agreement.
+
+        ACCEPT
+
+
+CWI LICENSE AGREEMENT FOR PYTHON 0.9.0 THROUGH 1.2
+--------------------------------------------------
+
+Copyright (c) 1991 - 1995, Stichting Mathematisch Centrum Amsterdam,
+The Netherlands.  All rights reserved.
+
+Permission to use, copy, modify, and distribute this software and its
+documentation for any purpose and without fee is hereby granted,
+provided that the above copyright notice appear in all copies and that
+both that copyright notice and this permission notice appear in
+supporting documentation, and that the name of Stichting Mathematisch
+Centrum or CWI not be used in advertising or publicity pertaining to
+distribution of the software without specific, written prior
+permission.
+
+STICHTING MATHEMATISCH CENTRUM DISCLAIMS ALL WARRANTIES WITH REGARD TO
+THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND
+FITNESS, IN NO EVENT SHALL STICHTING MATHEMATISCH CENTRUM BE LIABLE
+FOR ANY SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
+ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT
+OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+
+
+Licenses and Acknowledgements for Incorporated Software
+=======================================================
+
+Mersenne Twister
+----------------
+
+The `_random' module includes code based on a download from
+`http://www.math.keio.ac.jp/~matumoto/MT2002/emt19937ar.html'.  The
+following are the verbatim comments from the original code:
+
+     A C-program for MT19937, with initialization improved 2002/1/26.
+     Coded by Takuji Nishimura and Makoto Matsumoto.
+
+     Before using, initialize the state by using init_genrand(seed)
+     or init_by_array(init_key, key_length).
+
+     Copyright (C) 1997 - 2002, Makoto Matsumoto and Takuji Nishimura,
+     All rights reserved.
+
+     Redistribution and use in source and binary forms, with or without
+     modification, are permitted provided that the following conditions
+     are met:
+
+      1. Redistributions of source code must retain the above copyright
+         notice, this list of conditions and the following disclaimer.
+
+      2. Redistributions in binary form must reproduce the above copyright
+         notice, this list of conditions and the following disclaimer in the
+         documentation and/or other materials provided with the distribution.
+
+      3. The names of its contributors may not be used to endorse or promote
+         products derived from this software without specific prior written
+         permission.
+
+     THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+     "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+     LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+     A PARTICULAR PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE COPYRIGHT
+     OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+     SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED
+     TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+     PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
+     LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
+     NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+     SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+     Any feedback is very welcome.
+     http://www.math.keio.ac.jp/matumoto/emt.html
+     email: matumoto@math.keio.ac.jp
+
+
+Sockets
+-------
+
+The `socket' module uses the functions, `getaddrinfo', and
+`getnameinfo', which are coded in separate source files from the WIDE
+Project, `http://www.wide.ad.jp/about/index.html'.
+
+     Copyright (C) 1995, 1996, 1997, and 1998 WIDE Project.
+     All rights reserved.
+
+     Redistribution and use in source and binary forms, with or without
+     modification, are permitted provided that the following conditions
+     are met:
+     1. Redistributions of source code must retain the above copyright
+        notice, this list of conditions and the following disclaimer.
+     2. Redistributions in binary form must reproduce the above copyright
+        notice, this list of conditions and the following disclaimer in the
+        documentation and/or other materials provided with the distribution.
+     3. Neither the name of the project nor the names of its contributors
+        may be used to endorse or promote products derived from this software
+        without specific prior written permission.
+
+     THIS SOFTWARE IS PROVIDED BY THE PROJECT AND CONTRIBUTORS ``AS IS'' AND
+     GAI_ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+     IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+     ARE DISCLAIMED.  IN NO EVENT SHALL THE PROJECT OR CONTRIBUTORS BE LIABLE
+     FOR GAI_ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+     CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+     SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+     INTERRUPTION) HOWEVER CAUSED AND ON GAI_ANY THEORY OF LIABILITY, WHETHER
+     IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+     ARISING IN GAI_ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED
+     OF THE POSSIBILITY OF SUCH DAMAGE.
+
+
+Floating point exception control
+--------------------------------
+
+The source for the `fpectl' module includes the following notice:
+
+     ---------------------------------------------------------------------  
+    /                       Copyright (c) 1996.                           \ 
+   |          The Regents of the University of California.                 |
+   |                        All rights reserved.                           |
+   |                                                                       |
+   |   Permission to use, copy, modify, and distribute this software for   |
+   |   any purpose without fee is hereby granted, provided that this en-   |
+   |   tire notice is included in all copies of any software which is or   |
+   |   includes  a  copy  or  modification  of  this software and in all   |
+   |   copies of the supporting documentation for such software.           |
+   |                                                                       |
+   |   This  work was produced at the University of California, Lawrence   |
+   |   Livermore National Laboratory under  contract  no.  W-7405-ENG-48   |
+   |   between  the  U.S.  Department  of  Energy and The Regents of the   |
+   |   University of California for the operation of UC LLNL.              |
+   |                                                                       |
+   |                              DISCLAIMER                               |
+   |                                                                       |
+   |   This  software was prepared as an account of work sponsored by an   |
+   |   agency of the United States Government. Neither the United States   |
+   |   Government  nor the University of California nor any of their em-   |
+   |   ployees, makes any warranty, express or implied, or  assumes  any   |
+   |   liability  or  responsibility  for the accuracy, completeness, or   |
+   |   usefulness of any information,  apparatus,  product,  or  process   |
+   |   disclosed,   or  represents  that  its  use  would  not  infringe   |
+   |   privately-owned rights. Reference herein to any specific  commer-   |
+   |   cial  products,  process,  or  service  by trade name, trademark,   |
+   |   manufacturer, or otherwise, does not  necessarily  constitute  or   |
+   |   imply  its endorsement, recommendation, or favoring by the United   |
+   |   States Government or the University of California. The views  and   |
+   |   opinions  of authors expressed herein do not necessarily state or   |
+   |   reflect those of the United States Government or  the  University   |
+   |   of  California,  and shall not be used for advertising or product   |
+    \  endorsement purposes.                                              / 
+     ---------------------------------------------------------------------
+
+
+Cookie management
+-----------------
+
+The `Cookie' module contains the following notice:
+
+      Copyright 2000 by Timothy O'Malley <timo@alum.mit.edu>
+
+                     All Rights Reserved
+
+      Permission to use, copy, modify, and distribute this software
+      and its documentation for any purpose and without fee is hereby
+      granted, provided that the above copyright notice appear in all
+      copies and that both that copyright notice and this permission
+      notice appear in supporting documentation, and that the name of
+      Timothy O'Malley  not be used in advertising or publicity
+      pertaining to distribution of the software without specific, written
+      prior permission.
+
+      Timothy O'Malley DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS
+      SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
+      AND FITNESS, IN NO EVENT SHALL Timothy O'Malley BE LIABLE FOR
+      ANY SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+      WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS,
+      WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS
+      ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
+      PERFORMANCE OF THIS SOFTWARE.
+
+
+Execution tracing
+-----------------
+
+The `trace' module contains the following notice:
+
+      portions copyright 2001, Autonomous Zones Industries, Inc., all rights...
+      err...  reserved and offered to the public under the terms of the
+      Python 2.2 license.
+      Author: Zooko O'Whielacronx
+      http://zooko.com/
+      mailto:zooko@zooko.com
+
+      Copyright 2000, Mojam Media, Inc., all rights reserved.
+      Author: Skip Montanaro
+
+      Copyright 1999, Bioreason, Inc., all rights reserved.
+      Author: Andrew Dalke
+
+      Copyright 1995-1997, Automatrix, Inc., all rights reserved.
+      Author: Skip Montanaro
+
+      Copyright 1991-1995, Stichting Mathematisch Centrum, all rights reserved.
+
+      Permission to use, copy, modify, and distribute this Python software and
+      its associated documentation for any purpose without fee is hereby
+      granted, provided that the above copyright notice appears in all copies,
+      and that both that copyright notice and this permission notice appear in
+      supporting documentation, and that the name of neither Automatrix,
+      Bioreason or Mojam Media be used in advertising or publicity pertaining
+      to distribution of the software without specific, written prior
+      permission.
+
+
+UUencode and UUdecode functions
+-------------------------------
+
+The `uu' module contains the following notice:
+
+      Copyright 1994 by Lance Ellinghouse
+      Cathedral City, California Republic, United States of America.
+                             All Rights Reserved
+      Permission to use, copy, modify, and distribute this software and its
+      documentation for any purpose and without fee is hereby granted,
+      provided that the above copyright notice appear in all copies and that
+      both that copyright notice and this permission notice appear in
+      supporting documentation, and that the name of Lance Ellinghouse
+      not be used in advertising or publicity pertaining to distribution
+      of the software without specific, written prior permission.
+      LANCE ELLINGHOUSE DISCLAIMS ALL WARRANTIES WITH REGARD TO
+      THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND
+      FITNESS, IN NO EVENT SHALL LANCE ELLINGHOUSE CENTRUM BE LIABLE
+      FOR ANY SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+      WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
+      ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT
+      OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+
+      Modified by Jack Jansen, CWI, July 1995:
+      - Use binascii module to do the actual line-by-line conversion
+        between ascii and binary. This results in a 1000-fold speedup. The C
+        version is still 5 times faster, though.
+      - Arguments more compliant with python standard
+
+
+XML Remote Procedure Calls
+--------------------------
+
+The `xmlrpclib' module contains the following notice:
+
+          The XML-RPC client interface is
+
+      Copyright (c) 1999-2002 by Secret Labs AB
+      Copyright (c) 1999-2002 by Fredrik Lundh
+
+      By obtaining, using, and/or copying this software and/or its
+      associated documentation, you agree that you have read, understood,
+      and will comply with the following terms and conditions:
+
+      Permission to use, copy, modify, and distribute this software and
+      its associated documentation for any purpose and without fee is
+      hereby granted, provided that the above copyright notice appears in
+      all copies, and that both that copyright notice and this permission
+      notice appear in supporting documentation, and that the name of
+      Secret Labs AB or the author not be used in advertising or publicity
+      pertaining to distribution of the software without specific, written
+      prior permission.
+
+      SECRET LABS AB AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH REGARD
+      TO THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANT-
+      ABILITY AND FITNESS.  IN NO EVENT SHALL SECRET LABS AB OR THE AUTHOR
+      BE LIABLE FOR ANY SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY
+      DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS,
+      WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS
+      ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE
+      OF THIS SOFTWARE.
+
+Licenses for Software linked to
+===============================
+
+Note that the choice of GPL compatibility outlined above doesn't extend
+to modules linked to particular libraries, since they change the
+effective License of the module binary.
+
+
+GNU Readline
+------------
+
+The 'readline' module makes use of GNU Readline.
+
+      The GNU Readline Library is free software; you can redistribute it
+      and/or modify it under the terms of the GNU General Public License as
+      published by the Free Software Foundation; either version 2, or (at
+      your option) any later version.
+
+      On Debian systems, you can find the complete statement in
+      /usr/share/doc/readline-common/copyright'. A copy of the GNU General
+      Public License is available in /usr/share/common-licenses/GPL-2'.
+
+
+OpenSSL
+-------
+
+The '_ssl' module makes use of OpenSSL.
+
+      The OpenSSL toolkit stays under a dual license, i.e. both the
+      conditions of the OpenSSL License and the original SSLeay license
+      apply to the toolkit. Actually both licenses are BSD-style Open
+      Source licenses. Note that both licenses are incompatible with
+      the GPL.
+
+      On Debian systems, you can find the complete license text in
+      /usr/share/doc/openssl/copyright'.
+
+
+Files with other licenses than the Python License
+-------------------------------------------------
+
+Files: Lib/profile.py Lib/pstats.py
+Copyright: Disney Enterprises, Inc.  All Rights Reserved.
+License: # Licensed to PSF under a Contributor Agreement
+  Licensed under the Apache License, Version 2.0 (the "License");
+  you may not use this file except in compliance with the License.
+  You may obtain a copy of the License at
+
+  http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
+  either express or implied.  See the License for the specific language
+  overning permissions and limitations under the License.
+
+  On Debian systems, the Apache 2.0 license can be found in
+  /usr/share/common-licenses/Apache-2.0.
+
+Files: Modules/zlib/*
+Copyright: (C) 1995-2010 Jean-loup Gailly and Mark Adler
+License: This software is provided 'as-is', without any express or implied
+  warranty.  In no event will the authors be held liable for any damages
+  arising from the use of this software.
+
+  Permission is granted to anyone to use this software for any purpose,
+  including commercial applications, and to alter it and redistribute it
+  freely, subject to the following restrictions:
+
+  1. The origin of this software must not be misrepresented; you must not
+     claim that you wrote the original software. If you use this software
+     in a product, an acknowledgment in the product documentation would be
+     appreciated but is not required.
+  2. Altered source versions must be plainly marked as such, and must not be
+     misrepresented as being the original software.
+  3. This notice may not be removed or altered from any source distribution.
+
+  Jean-loup Gailly        Mark Adler
+  jloup@gzip.org          madler@alumni.caltech.edu
+
+ If you use the zlib library in a product, we would appreciate *not* receiving
+ lengthy legal documents to sign.  The sources are provided for free but without
+ warranty of any kind.  The library has been entirely written by Jean-loup
+ Gailly and Mark Adler; it does not include third-party code.
+
+Files: Modules/_ctypes/libffi/*
+Copyright: Copyright (C) 1996-2009 Red Hat, Inc and others.
+License: Permission is hereby granted, free of charge, to any person obtaining
+   a copy of this software and associated documentation files (the
+   ``Software''), to deal in the Software without restriction, including
+   without limitation the rights to use, copy, modify, merge, publish,
+   distribute, sublicense, and/or sell copies of the Software, and to
+   permit persons to whom the Software is furnished to do so, subject to
+   the following conditions:
+
+   The above copyright notice and this permission notice shall be included
+   in all copies or substantial portions of the Software.
+
+   THE SOFTWARE IS PROVIDED ``AS IS'', WITHOUT WARRANTY OF ANY KIND,
+   EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+   MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+   NONINFRINGEMENT.  IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+   HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+   WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+   OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+   DEALINGS IN THE SOFTWARE.
+
+   Documentation:
+   Permission is granted to copy, distribute and/or modify this document
+   under the terms of the GNU General Public License as published by the
+   Free Software Foundation; either version 2, or (at your option) any
+   later version.  A copy of the license is included in the
+   section entitled ``GNU General Public License''.
+
+Files: Modules/expat/*
+Copyright: Copyright (c) 1998, 1999, 2000 Thai Open Source Software Center Ltd
+  and Clark Cooper
+  Copyright (c) 2001, 2002, 2003, 2004, 2005, 2006 Expat maintainers
+License: Permission is hereby granted, free of charge, to any person obtaining
+  a copy of this software and associated documentation files (the
+  "Software"), to deal in the Software without restriction, including
+  without limitation the rights to use, copy, modify, merge, publish,
+  distribute, sublicense, and/or sell copies of the Software, and to
+  permit persons to whom the Software is furnished to do so, subject to
+  the following conditions:
+ 
+  The above copyright notice and this permission notice shall be included
+  in all copies or substantial portions of the Software.
+ 
+  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+  EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+  MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+  IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
+  CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
+  TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+  SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+Files: Misc/python-mode.el
+Copyright: Copyright (C) 1992,1993,1994  Tim Peters
+License: This software is provided as-is, without express or implied
+        warranty.  Permission to use, copy, modify, distribute or sell this
+        software, without fee, for any purpose and by any individual or
+        organization, is hereby granted, provided that the above copyright
+        notice and this paragraph appear in all copies.
+
+Files: PC/_subprocess.c
+Copyright: Copyright (c) 2004 by Fredrik Lundh <fredrik@pythonware.com>
+        Copyright (c) 2004 by Secret Labs AB, http://www.pythonware.com
+        Copyright (c) 2004 by Peter Astrand <astrand@lysator.liu.se>
+License:
+ * Permission to use, copy, modify, and distribute this software and
+ * its associated documentation for any purpose and without fee is
+ * hereby granted, provided that the above copyright notice appears in
+ * all copies, and that both that copyright notice and this permission
+ * notice appear in supporting documentation, and that the name of the
+ * authors not be used in advertising or publicity pertaining to
+ * distribution of the software without specific, written prior
+ * permission.
+ *
+ * THE AUTHORS DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE,
+ * INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS.
+ * IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY SPECIAL, INDIRECT OR
+ * CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS
+ * OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT,
+ * NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION
+ * WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+
+Files: PC/winsound.c
+Copyright: Copyright (c) 1999 Toby Dickenson
+License:  * Permission to use this software in any way is granted without
+ * fee, provided that the copyright notice above appears in all
+ * copies. This software is provided "as is" without any warranty.
+ */
+
+/* Modified by Guido van Rossum */
+/* Beep added by Mark Hammond */
+/* Win9X Beep and platform identification added by Uncle Timmy */
+
+Files: Lib/ensurepip/_bundled/setuptools-*.whl
+License: see above, some license as Python.
+
+Files: Lib/ensurepip/_bundled/pip-*.whl
+Copyright: Copyright © 2008-2013 The pip developers:
+    Alex Grönholm
+    Alex Morega
+    Alexandre Conrad
+    Andrey Bulgakov
+    Antti Kaihola
+    Armin Ronacher
+    Aziz Köksal
+    Ben Rosser
+    Brian Rosner
+    Carl Meyer
+    Chris McDonough
+    Christian Oudard
+    Clay McClure
+    Cody Soyland
+    Daniel Holth
+    Dave Abrahams
+    David (d1b)
+    Dmitry Gladkov
+    Donald Stufft
+    Francesco
+    Geoffrey Lehée
+    Georgi Valkov
+    Hugo Lopes Tavares
+    Ian Bicking
+    Igor Sobreira
+    Ionel Maries Cristian
+    Jakub Vysoky
+    James Cleveland
+    Jannis Leidel
+    Jay Graves
+    John-Scott Atlakson
+    Jon Parise
+    Jonas Nockert
+    Josh Bronson
+    Kamal Bin Mustafa
+    Kelsey Hightower
+    Kenneth Belitzky
+    Kumar McMillan
+    Luke Macken
+    Masklinn
+    Marc Abramowitz
+    Marcus Smith
+    Markus Hametner
+    Matt Maker
+    Maxime R.
+    Miguel Araujo
+    Nick Stenning
+    Nowell Strite
+    Oliver Tonnhofer
+    Olivier Girardot
+    Patrick Jenkins
+    Paul Moore
+    Paul Nasrat
+    Paul Oswald
+    Paul van der Linden
+    Peter Waller
+    Phil Whelan
+    Piet Delport
+    Przemek Wrzos
+    Qiangning Hong
+    Rafael Caricio
+    Rene Dudfield
+    Roey Berman
+    Ronny Pfannschmidt
+    Rory McCann
+    Simon Cross
+    Stavros Korokithakis
+    Thomas Fenzl
+    Thomas Johansson
+    Vinay Sajip
+    Vitaly Babiy
+    W Trevor King
+    Wil Tan
+    Hsiaoming Yang
+License: Expat
+    Permission is hereby granted, free of charge, to any person obtaining
+    a copy of this software and associated documentation files (the
+    "Software"), to deal in the Software without restriction, including
+    without limitation the rights to use, copy, modify, merge, publish,
+    distribute, sublicense, and/or sell copies of the Software, and to
+    permit persons to whom the Software is furnished to do so, subject to
+    the following conditions:
+    .
+    The above copyright notice and this permission notice shall be
+    included in all copies or substantial portions of the Software.
+    .
+    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+    EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+    MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+    NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+    LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+    OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+    WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+Files: Lib/ensurepip/_bundled/pip-*.whl/pip/_vendor/ipaddress.py
+Copyright: 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010, 2011, 2012, 2013, 2014 Python Software Foundation; All Rights Reserved
+           2007 Google Inc.
+License: PSF-2
+
+Files: Lib/ensurepip/_bundled/pip-*.whl/pip/_vendor/retrying.py
+Copyright: (c) 2013-2014, <ray@blacklocus.com>
+           (c) 2014, Derek Wilson <dwilson@domaintools.com>
+           (c) 2014, Alex Kuang <akuang@bizo.com>
+License: Apache-2
+
+License: Apache-2
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+ .
+    http://www.apache.org/licenses/LICENSE-2.0
+ .
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+ .
+ On Debian-based systems the full text of the Apache version 2.0 license
+ can be found in /usr/share/common-licenses/Apache-2.0.
+
+Files: Lib/ensurepip/_bundled/pip-*.whl/pip/_vendor/six.py
+Copyright: Copyright (c) 2010-2014 Benjamin Peterson
+License: Expat
+
+Files: Lib/ensurepip/_bundled/pip-*.whl/pip/_vendor/cachecontrol
+Copyright: 2015 Eric Larson
+License: Apache-2.0
+ Copyright 2015 Eric Larson
+ .
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+ .
+     http://www.apache.org/licenses/LICENSE-2.0
+ .
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+ implied.
+ .
+ See the License for the specific language governing permissions and
+ limitations under the License.
+ .
+ On Debian systems, the license is available at
+ /usr/share/common-licenses/Apache-2.0
+
+Files: Lib/ensurepip/_bundled/pip-*.whl/pip/_vendor/colorama
+Files: Lib/ensurepip/_bundled/pip-*.whl/pip/_vendor/distlib
+Copyright: Copyright (C) 2012-2013 The Python Software Foundation
+           Copyright (C) 2012-2013 Vinay Sajip
+License: Python
+
+Files: Lib/ensurepip/_bundled/pip-*.whl/pip/_vendor/html5lib
+html5lib is Copyright (c) 2006 The Authors
+Authors:
+  James Graham - james@hoppipolla.co.uk
+  Anne van Kesteren - annevankesteren@gmail.com
+  Lachlan Hunt - lachlan.hunt@lachy.id.au
+  Matt McDonald - kanashii@kanashii.ca
+  Sam Ruby - rubys@intertwingly.net
+  Ian Hickson (Google) - ian@hixie.ch
+  Thomas Broyer - t.broyer@gmail.com
+
+License:
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of
+this software and associated documentation files (the "Software"), to deal in
+the Software without restriction, including without limitation the rights to
+use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies
+of the Software, and to permit persons to whom the Software is furnished to do
+so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+
+Files: Lib/ensurepip/_bundled/pip-*.whl/pip/_vendor/lockfile
+Copyright:
+    © 2014–2015 OpenStack Foundation <openstack-dev@lists.openstack.org>
+    © 2007–2012 Skip Montanaro <skip@pobox.com>
+    © 2008–2014 Ben Finney <ben+python@benfinney.id.au>
+License: Expat
+
+Files: Lib/ensurepip/_bundled/pip-*.whl/pip/_vendor/packaging
+Copyright: Copyright (C) 2014 Donald Stufft
+           Copyright (C) 2012-2013 Vinay Sajip
+License: Apache-2.0
+
+Files: Lib/ensurepip/_bundled/pip-*.whl/pip/_vendor/progress
+Copyright: Copyright (c) 2012 Giorgos Verigakis <verigak@gmail.com>
+License: ISC
+
+Files: Lib/ensurepip/_bundled/pip-*.whl/pip/_vendor/requests
+Copyright: 2016, Kenneth Reitz
+License: Apache
+
+Files: Lib/ensurepip/_bundled/pip-*.whl/pip/_vendor/requests/packages/urllib3/*
+Copyright: 2008-2016, Andrey Petrov
+License: Expat
+
+Files: Lib/ensurepip/_bundled/pip-*.whl/pip/_vendor/requests/packages/urllib3/packages/ordered_dict.py
+Copyright: 2009, Raymond Hettinger
+License: Expat
+
+Files: Lib/ensurepip/_bundled/pip-*.whl/pip/_vendor/requests/packages/urllib3/packages/ssl_match_hostname/__init__.py
+Copyright: 2011, Python Software Foundation
+License: PSF-2
+
+Files: Lib/ensurepip/_bundled/pip-*.whl/pip/_vendor/requests/packages/chardet/*
+Copyright: 2006-2008, Mark Pilgrim
+           2012-2013, Ian Cordasco
+License: LGPL-2.1+

syft/cataloger/deb/test-fixtures/copyright/trilicense

@@ -0,0 +1,13 @@
+Files: src/js/editline/*
+Copyright: 1993, John Doe
+           1993, Joe Average
+License: MPL-1.1 or GPL-2 or LGPL-2.1
+
+License: MPL-1.1
+ [LICENSE TEXT]
+
+License: GPL-2
+ [LICENSE TEXT]
+
+License: LGPL-2.1
+ [LICENSE TEXT]

syft/cataloger/deb/test-fixtures/image-dpkg/Dockerfile

@@ -0,0 +1,2 @@
+FROM scratch
+COPY . .

syft/cataloger/deb/test-fixtures/image-dpkg/usr/share/doc/libpam-runtime/copyright

@@ -0,0 +1,43 @@
+Format: http://www.debian.org/doc/packaging-manuals/copyright-format/1.0/
+Upstream-Name: auditd
+Source: https://people.redhat.com/sgrubb/audit/
+
+Files: *
+Copyright: 2012-2016 Steve Grubb <sgrubb@redhat.com>
+           2006-2012 Rik Faith
+License: GPL-2
+
+Files: src/libev/*
+Copyright: 2007-2009 Marc Alexamder Lehmann
+License: GPL-2
+
+Files: lib/*
+Copyright: 2005-2008 Steve Grubb <sgrubb@redhat.com>
+License: LGPL-2.1
+  The audit daemon's library libaudit.* is released under LGPL
+  so that it may be linked with 3rd party software.
+  .
+  On Debian systems, refer to /usr/share/common-licenses/LGPL-2.1
+  for the complete text of the GNU Lesser General Public License.
+
+Files: debian/*
+Copyright: 2007-2011 Philipp Matthias Hahn <pmhahn@debian.org>
+           2012-2016 Laurent Bigonville <bigon@debian.org>
+License: GPL-2
+
+License: GPL-2
+ This package is free software; you can redistribute it and/or modify
+ it under the terms of the GNU General Public License version 2,
+ as published by the Free Software Foundation.
+ .
+ This package is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ GNU General Public License for more details.
+ .
+ You should have received a copy of the GNU General Public License
+ along with this package; if not, write to the Free Software
+ Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301 USA
+ .
+ On Debian systems, the complete text of the GNU General
+ Public License can be found in `/usr/share/common-licenses/GPL-1'.

syft/cataloger/deb/test-fixtures/image-dpkg/var/lib/dpkg/info/libpam-runtime.md5sums

@@ -0,0 +1,4 @@
+55f905631797551d4d936a34c7e73474  lib/x86_64-linux-gnu/libz.so.1.2.11
+cede84bda30d2380217f97753c8ccf3a  usr/share/doc/zlib1g/changelog.Debian.gz
+f3c9dafa6da7992c47328b4464f6d122  usr/share/doc/zlib1g/changelog.gz
+a4fae96070439a5209a62ae5b8017ab2  usr/share/doc/zlib1g/copyright

syft/cataloger/deb/test-fixtures/image-dpkg/var/lib/dpkg/status

@@ -0,0 +1,21 @@
+Package: libpam-runtime
+Status: install ok installed
+Priority: required
+Section: admin
+Installed-Size: 1016
+Maintainer: Steve Langasek <vorlon@debian.org>
+Architecture: all
+Multi-Arch: foreign
+Source: pam
+Version: 1.1.8-3.6
+Replaces: libpam0g-dev, libpam0g-util
+Depends: debconf (>= 0.5) | debconf-2.0, debconf (>= 1.5.19) | cdebconf, libpam-modules (>= 1.0.1-6)
+Conflicts: libpam0g-util
+Conffiles:
+ /etc/pam.conf 87fc76f18e98ee7d3848f6b81b3391e5
+ /etc/pam.d/other 31aa7f2181889ffb00b87df4126d1701
+Description: Runtime support for the PAM library
+ Contains configuration files and  directories required for
+ authentication  to work on Debian systems.  This package is required
+ on almost all installations.
+Homepage: http://www.linux-pam.org/

syft/cataloger/deb/test-fixtures/info/zlib1g.md5sums

@@ -0,0 +1,4 @@
+55f905631797551d4d936a34c7e73474  lib/x86_64-linux-gnu/libz.so.1.2.11
+cede84bda30d2380217f97753c8ccf3a  usr/share/doc/zlib1g/changelog.Debian.gz
+f3c9dafa6da7992c47328b4464f6d122  usr/share/doc/zlib1g/changelog.gz
+a4fae96070439a5209a62ae5b8017ab2  usr/share/doc/zlib1g/copyright

syft/cataloger/java/archive_filename.go

@@ -10,36 +10,100 @@ import (
 	"github.com/anchore/syft/syft/pkg"
 )
 
-// match examples:
-//	pkg-extra-field-4.3.2-rc1 --> match(name=pkg-extra-field version=4.3.2-rc1)
-//	pkg-extra-field-4.3-rc1   --> match(name=pkg-extra-field version=4.3-rc1)
-//	pkg-extra-field-4.3       --> match(name=pkg-extra-field version=4.3)
-var versionPattern = regexp.MustCompile(`(?P<name>.+)-(?P<version>(\d+\.)?(\d+\.)?(\*|\d+)(-[a-zA-Z0-9\-\.]+)*)`)
+// nameAndVersionPattern finds the package name and version (as named capture
+// groups) in a string. The pattern's strategy is to start at the beginning of
+// the string, and for every next dash-delimited group, consider the group to be
+// a continuation of the package name, unless the group begins with a number or
+// matches any of a specified set of "version-indicating" patterns. When a given
+// group meets this criterion, consider the group and the remainder of the
+// string to be the package version.
+//
+// Regex components of note:
+//
+// (?Ui)													...	Sets the "U" and the "i" options for this Regex —— (ungreedy,
+// and case-insensitive, respectively). "Ungreedy" is important so that the '*' that trails the package name
+// component doesn't consume the rest of the string.
+//
+// [[:alpha:]][[:word:]]*									...	Matches any word, and the word can include "word" characters (
+// which includes numbers and underscores), but the first character of the word MUST be a letter.
+//
+// (?:\.[[:alpha:]][[:word:]]*)* 							... This looks redundant, but it's not. It
+// extends the previous pattern such that the net effect of both components is
+// that words can also include a period and more words (thus, when combined, not
+// only is "something" matched, but so is "com.prefix.thing"
+//
+// (?:\d.*|(?:build\d*.*)|(?:rc?\d+(?:^[[:alpha:]].*)?))	...
+// This match group covers the "version-indicating" patterns mentioned in the above description. Given the pipes (
+// '|'), this functions as a series of 'OR'-joined conditions:
+//
+//		\d.*						...	"If it starts with a numeric digit, this is a version, no matter what follows."
+//		build\d*.*					...	"If it starts with "build" and then a numeric digit immediately after, this is a version."
+//		rc?\d+(?:^[[:alpha:]].*)?	...	"If it starts with "r" or "rc" and then one or more numeric digits immediately
+//										after, but no alpha characters right after that (in the same word), this is a version."
+//
+// Match examples:
+//		some-package-4.0.1		--> name="some-package", version="4.0.1"
+//		prefix.thing-4			-->	name="prefix.thing", version="4"
+//		my-http2-server-5		-->	name="my-http2-server", version="5"
+//		jetpack-build235-rc5	-->	name="jetpack", version="build2.0-rc5"
+//		ironman-r4-2009			--> name="ironman", version="r4-2009"
+var nameAndVersionPattern = regexp.MustCompile(`(?Ui)^(?P<name>(?:[[:alpha:]][[:word:]]*(?:\.[[:alpha:]][[:word:]]*)*-?)+)(?:-(?P<version>(?:\d.*|(?:build\d*.*)|(?:rc?\d+(?:^[[:alpha:]].*)?))))?$`)
 
 type archiveFilename struct {
-	raw    string
-	fields []map[string]string
+	raw     string
+	name    string
+	version string
+}
+
+// TODO: Remove this method once we're using Go 1.15+.
+//
+// Go 1.15 introduces a `SubexpIndex` method for the Regexp type that would let
+// this code be made more elegant. Once we've reached 1.15, we should eliminate
+// this function in favor of that method.
+func subexpIndex(re *regexp.Regexp, name string) int {
+	for i, subexpName := range re.SubexpNames() {
+		if subexpName == name {
+			return i
+		}
+	}
+
+	return -1
+}
+
+func getSubexp(matches []string, subexpName string, re *regexp.Regexp, raw string) string {
+	if len(matches) < 1 {
+		log.Warnf("unexpectedly empty matches for archive '%s'", raw)
+		return ""
+	}
+
+	index := subexpIndex(re, subexpName)
+	if index < 1 {
+		log.Warnf("unexpected index of '%s' capture group for Java archive '%s'", subexpName, raw)
+		return ""
+	}
+
+	// Prevent out-of-range panic
+	if len(matches) < index+1 {
+		log.Warnf("no match found for '%s' in '%s'", subexpName, matches[0])
+		return ""
+	}
+
+	return matches[index]
 }
 
 func newJavaArchiveFilename(raw string) archiveFilename {
 	// trim the file extension and remove any path prefixes
-	name := strings.TrimSuffix(filepath.Base(raw), filepath.Ext(raw))
+	cleanedFileName := strings.TrimSuffix(filepath.Base(raw), filepath.Ext(raw))
 
-	matches := versionPattern.FindAllStringSubmatch(name, -1)
-	fields := make([]map[string]string, 0)
-	for _, match := range matches {
-		item := make(map[string]string)
-		for i, name := range versionPattern.SubexpNames() {
-			if i != 0 && name != "" {
-				item[name] = match[i]
-			}
-		}
-		fields = append(fields, item)
-	}
+	matches := nameAndVersionPattern.FindStringSubmatch(cleanedFileName)
+
+	name := getSubexp(matches, "name", nameAndVersionPattern, raw)
+	version := getSubexp(matches, "version", nameAndVersionPattern, raw)
 
 	return archiveFilename{
-		raw:    raw,
-		fields: fields,
+		raw:     raw,
+		name:    name,
+		version: version,
 	}
 }
 
@@ -57,23 +121,3 @@ func (a archiveFilename) pkgType() pkg.Type {
 		return pkg.UnknownPkg
 	}
 }
-
-func (a archiveFilename) version() string {
-	if len(a.fields) > 1 {
-		log.Errorf("discovered multiple name-version pairings from %q: %+v", a.raw, a.fields)
-		return ""
-	} else if len(a.fields) < 1 {
-		return ""
-	}
-
-	return a.fields[0]["version"]
-}
-
-func (a archiveFilename) name() string {
-	// there should be only one name, if there is more or less then something is wrong
-	if len(a.fields) != 1 {
-		return ""
-	}
-
-	return a.fields[0]["name"]
-}

syft/cataloger/java/archive_filename_test.go

@@ -1,9 +1,10 @@
 package java
 
 import (
+	"testing"
+
 	"github.com/anchore/syft/syft/pkg"
 	"github.com/sergi/go-diff/diffmatchpatch"
-	"testing"
 )
 
 func TestExtractInfoFromJavaArchiveFilename(t *testing.T) {
@@ -56,30 +57,98 @@ func TestExtractInfoFromJavaArchiveFilename(t *testing.T) {
 			name:      "pkg-extra-field-maven",
 			ty:        pkg.JenkinsPluginPkg,
 		},
+		{
+			filename:  "/some/path-with-version-5.4.3/wagon-webdav-1.0.2-beta-2.2.3a-hudson.jar",
+			version:   "1.0.2-beta-2.2.3a-hudson",
+			extension: "jar",
+			name:      "wagon-webdav",
+			ty:        pkg.JavaPkg,
+		},
+		{
+			filename:  "/some/path-with-version-5.4.3/wagon-webdav-1.0.2-beta-2.2.3-hudson.jar",
+			version:   "1.0.2-beta-2.2.3-hudson",
+			extension: "jar",
+			name:      "wagon-webdav",
+			ty:        pkg.JavaPkg,
+		},
+		{
+			filename:  "/some/path-with-version-5.4.3/windows-remote-command-1.0.jar",
+			version:   "1.0",
+			extension: "jar",
+			name:      "windows-remote-command",
+			ty:        pkg.JavaPkg,
+		},
+		{
+			filename:  "/some/path-with-version-5.4.3/wagon-http-lightweight-1.0.5-beta-2.jar",
+			version:   "1.0.5-beta-2",
+			extension: "jar",
+			name:      "wagon-http-lightweight",
+			ty:        pkg.JavaPkg,
+		},
+		{
+			filename:  "/hudson.war:WEB-INF/lib/commons-jelly-1.1-hudson-20100305.jar",
+			version:   "1.1-hudson-20100305",
+			extension: "jar",
+			name:      "commons-jelly",
+			ty:        pkg.JavaPkg,
+		},
+		{
+			filename:  "/hudson.war:WEB-INF/lib/jtidy-4aug2000r7-dev-hudson-1.jar",
+			version:   "4aug2000r7-dev-hudson-1",
+			extension: "jar",
+			name:      "jtidy",
+			ty:        pkg.JavaPkg,
+		},
+		{
+			filename:  "/hudson.war:WEB-INF/lib/trilead-ssh2-build212-hudson-5.jar",
+			version:   "build212-hudson-5",
+			extension: "jar",
+			name:      "trilead-ssh2",
+			ty:        pkg.JavaPkg,
+		},
+		{
+			filename:  "/hudson.war:WEB-INF/lib/guava-r06.jar",
+			version:   "r06",
+			extension: "jar",
+			name:      "guava",
+			ty:        pkg.JavaPkg,
+		},
+		{
+			filename:  "BOOT-INF/lib/spring-data-r2dbc-1.1.0.RELEASE.jar", // Regression: https://github.com/anchore/syft/issues/255
+			version:   "1.1.0.RELEASE",
+			extension: "jar",
+			name:      "spring-data-r2dbc",
+			ty:        pkg.JavaPkg,
+		},
 	}
 
 	for _, test := range tests {
 		t.Run(test.filename, func(t *testing.T) {
 			obj := newJavaArchiveFilename(test.filename)
 
-			version := obj.version()
+			ty := obj.pkgType()
+			if ty != test.ty {
+				t.Errorf("mismatched type: %+v != %v", ty, test.ty)
+			}
+
+			version := obj.version
 			if version != test.version {
 				dmp := diffmatchpatch.New()
-				diffs := dmp.DiffMain(version, test.version, true)
+				diffs := dmp.DiffMain(test.version, version, true)
 				t.Errorf("mismatched version:\n%s", dmp.DiffPrettyText(diffs))
 			}
 
 			extension := obj.extension()
 			if extension != test.extension {
 				dmp := diffmatchpatch.New()
-				diffs := dmp.DiffMain(extension, test.extension, true)
+				diffs := dmp.DiffMain(test.extension, extension, true)
 				t.Errorf("mismatched extension:\n%s", dmp.DiffPrettyText(diffs))
 			}
 
-			name := obj.name()
+			name := obj.name
 			if name != test.name {
 				dmp := diffmatchpatch.New()
-				diffs := dmp.DiffMain(name, test.name, true)
+				diffs := dmp.DiffMain(test.name, name, true)
 				t.Errorf("mismatched name:\n%s", dmp.DiffPrettyText(diffs))
 			}
 		})

syft/cataloger/java/archive_parser.go

@@ -5,6 +5,8 @@ import (
 	"io"
 	"strings"
 
+	"github.com/anchore/syft/internal/log"
+
 	"github.com/anchore/syft/internal"
 	"github.com/anchore/syft/internal/file"
 	"github.com/anchore/syft/syft/cataloger/common"
@@ -64,13 +66,17 @@ func newJavaArchiveParser(virtualPath string, reader io.Reader, detectNested boo
 		return nil, cleanupFn, fmt.Errorf("unable to read files from java archive: %w", err)
 	}
 
+	// fetch the last element of the virtual path
+	virtualElements := strings.Split(virtualPath, ":")
+	currentFilepath := virtualElements[len(virtualElements)-1]
+
 	return &archiveParser{
 		discoveredPkgs: internal.NewStringSet(),
 		fileManifest:   fileManifest,
 		virtualPath:    virtualPath,
 		archivePath:    archivePath,
 		contentPath:    contentPath,
-		fileInfo:       newJavaArchiveFilename(virtualPath),
+		fileInfo:       newJavaArchiveFilename(currentFilepath),
 		detectNested:   detectNested,
 	}, cleanupFn, nil
 }
@@ -118,7 +124,7 @@ func (j *archiveParser) parse() ([]pkg.Package, error) {
 // discoverMainPackage parses the root Java manifest used as the parent package to all discovered nested packages.
 func (j *archiveParser) discoverMainPackage() (*pkg.Package, error) {
 	// search and parse java manifest files
-	manifestMatches := j.fileManifest.GlobMatch(manifestPath)
+	manifestMatches := j.fileManifest.GlobMatch(manifestGlob)
 	if len(manifestMatches) > 1 {
 		return nil, fmt.Errorf("found multiple manifests in the jar: %+v", manifestMatches)
 	} else if len(manifestMatches) == 0 {
@@ -134,24 +140,27 @@ func (j *archiveParser) discoverMainPackage() (*pkg.Package, error) {
 
 	// parse the manifest file into a rich object
 	manifestContents := contents[manifestMatches[0]]
-	manifest, err := parseJavaManifest(strings.NewReader(manifestContents))
+	manifest, err := parseJavaManifest(j.archivePath, strings.NewReader(manifestContents))
 	if err != nil {
 		return nil, fmt.Errorf("failed to parse java manifest (%s): %w", j.virtualPath, err)
 	}
 
 	return &pkg.Package{
-		Name:     selectName(manifest, j.fileInfo),
-		Version:  selectVersion(manifest, j.fileInfo),
-		Language: pkg.Java,
-		Type:     pkg.JavaPkg,
+		Name:         selectName(manifest, j.fileInfo),
+		Version:      selectVersion(manifest, j.fileInfo),
+		Language:     pkg.Java,
+		Type:         pkg.JavaPkg,
+		MetadataType: pkg.JavaMetadataType,
 		Metadata: pkg.JavaMetadata{
-			Manifest: manifest,
+			VirtualPath: j.virtualPath,
+			Manifest:    manifest,
 		},
 	}, nil
 }
 
 // discoverPkgsFromPomProperties parses Maven POM properties for a given parent package, returning all listed Java packages found and
 // associating each discovered package to the given parent package.
+// nolint:funlen,gocognit
 func (j *archiveParser) discoverPkgsFromPomProperties(parentPkg *pkg.Package) ([]pkg.Package, error) {
 	var pkgs = make([]pkg.Package, 0)
 	parentKey := uniquePkgKey(parentPkg)
@@ -173,13 +182,22 @@ func (j *archiveParser) discoverPkgsFromPomProperties(parentPkg *pkg.Package) ([
 			if propsObj.Version != "" && propsObj.ArtifactID != "" {
 				// TODO: if there is no parentPkg (no java manifest) one of these poms could be the parent. We should discover the right parent and attach the correct info accordingly to each discovered package
 
+				// keep the artifact name within the virtual path if this package does not match the parent package
+				vPathSuffix := ""
+				if !strings.HasPrefix(propsObj.ArtifactID, parentPkg.Name) {
+					vPathSuffix += ":" + propsObj.ArtifactID
+				}
+				virtualPath := j.virtualPath + vPathSuffix
+
 				// discovered props = new package
 				p := pkg.Package{
-					Name:     propsObj.ArtifactID,
-					Version:  propsObj.Version,
-					Language: pkg.Java,
-					Type:     pkg.JavaPkg,
+					Name:         propsObj.ArtifactID,
+					Version:      propsObj.Version,
+					Language:     pkg.Java,
+					Type:         pkg.JavaPkg,
+					MetadataType: pkg.JavaMetadataType,
 					Metadata: pkg.JavaMetadata{
+						VirtualPath:   virtualPath,
 						PomProperties: propsObj,
 						Parent:        parentPkg,
 					},
@@ -187,16 +205,35 @@ func (j *archiveParser) discoverPkgsFromPomProperties(parentPkg *pkg.Package) ([
 
 				pkgKey := uniquePkgKey(&p)
 
-				if !j.discoveredPkgs.Contains(pkgKey) {
-					// only keep packages we haven't seen yet
-					pkgs = append(pkgs, p)
-				} else if pkgKey == parentKey {
+				// the name/version pair matches...
+				matchesParentPkg := pkgKey == parentKey
+
+				// the virtual path matches...
+				matchesParentPkg = matchesParentPkg || parentPkg.Metadata.(pkg.JavaMetadata).VirtualPath == virtualPath
+
+				// the pom artifactId has the parent name or vice versa
+				if propsObj.ArtifactID != "" {
+					matchesParentPkg = matchesParentPkg || strings.Contains(parentPkg.Name, propsObj.ArtifactID) || strings.Contains(propsObj.ArtifactID, parentPkg.Name)
+				}
+
+				if matchesParentPkg {
 					// we've run across more information about our parent package, add this info to the parent package metadata
+					// the pom properties is typically a better source of information for name and version than the manifest
+					if p.Name != parentPkg.Name {
+						parentPkg.Name = p.Name
+					}
+					if p.Version != parentPkg.Version {
+						parentPkg.Version = p.Version
+					}
+
 					parentMetadata, ok := parentPkg.Metadata.(pkg.JavaMetadata)
 					if ok {
 						parentMetadata.PomProperties = propsObj
 						parentPkg.Metadata = parentMetadata
 					}
+				} else if !j.discoveredPkgs.Contains(pkgKey) {
+					// only keep packages we haven't seen yet (and are not related to the parent package)
+					pkgs = append(pkgs, p)
 				}
 			}
 		}
@@ -214,18 +251,28 @@ func (j *archiveParser) discoverPkgsFromNestedArchives(parentPkg *pkg.Package) (
 	}
 
 	// search and parse pom.properties files & fetch the contents
-	readers, err := file.ExtractFromZipToUniqueTempFile(j.archivePath, j.contentPath, j.fileManifest.GlobMatch(archiveFormatGlobs...)...)
+	openers, err := file.ExtractFromZipToUniqueTempFile(j.archivePath, j.contentPath, j.fileManifest.GlobMatch(archiveFormatGlobs...)...)
 	if err != nil {
 		return nil, fmt.Errorf("unable to extract files from zip: %w", err)
 	}
 
 	// discover nested artifacts
-	for archivePath, archiveReader := range readers {
-		nestedPath := fmt.Sprintf("%s:%s", j.virtualPath, archivePath)
-		nestedPkgs, err := parseJavaArchive(nestedPath, archiveReader)
+	for archivePath, archiveOpener := range openers {
+		archiveReadCloser, err := archiveOpener.Open()
 		if err != nil {
+			return nil, fmt.Errorf("unable to open archived file from tempdir: %w", err)
+		}
+		nestedPath := fmt.Sprintf("%s:%s", j.virtualPath, archivePath)
+		nestedPkgs, err := parseJavaArchive(nestedPath, archiveReadCloser)
+		if err != nil {
+			if closeErr := archiveReadCloser.Close(); closeErr != nil {
+				log.Warnf("unable to close archived file from tempdir: %+v", closeErr)
+			}
 			return nil, fmt.Errorf("unable to process nested java archive (%s): %w", archivePath, err)
 		}
+		if err = archiveReadCloser.Close(); err != nil {
+			return nil, fmt.Errorf("unable to close archived file from tempdir: %w", err)
+		}
 
 		// attach the parent package to all discovered packages that are not already associated with a java archive
 		for _, p := range nestedPkgs {

syft/cataloger/java/archive_parser_test.go

@@ -78,51 +78,6 @@ func generateJavaBuildFixture(t *testing.T, fixturePath string) {
 	}
 }
 
-func TestSelectName(t *testing.T) {
-	tests := []struct {
-		desc     string
-		manifest pkg.JavaManifest
-		archive  archiveFilename
-		expected string
-	}{
-		{
-			desc:    "name from Implementation-Title",
-			archive: archiveFilename{},
-			manifest: pkg.JavaManifest{
-				Name:      "",
-				SpecTitle: "",
-				ImplTitle: "maven-wrapper",
-			},
-			expected: "maven-wrapper",
-		},
-		{
-			desc: "Implementation-Title does not override",
-			manifest: pkg.JavaManifest{
-				Name:      "Foo",
-				SpecTitle: "",
-				ImplTitle: "maven-wrapper",
-			},
-			archive: archiveFilename{
-				fields: []map[string]string{
-					{"name": "omg"},
-				},
-			},
-			expected: "omg",
-		},
-	}
-
-	for _, test := range tests {
-		t.Run(test.desc, func(t *testing.T) {
-			result := selectName(&test.manifest, test.archive)
-
-			if result != test.expected {
-				t.Errorf("mismatch in names: '%s' != '%s'", result, test.expected)
-			}
-		})
-	}
-
-}
-
 func TestParseJar(t *testing.T) {
 	tests := []struct {
 		fixture      string
@@ -137,17 +92,20 @@ func TestParseJar(t *testing.T) {
 			},
 			expected: map[string]pkg.Package{
 				"example-jenkins-plugin": {
-					Name:     "example-jenkins-plugin",
-					Version:  "1.0-SNAPSHOT",
-					Language: pkg.Java,
-					Type:     pkg.JenkinsPluginPkg,
+					Name:         "example-jenkins-plugin",
+					Version:      "1.0-SNAPSHOT",
+					Language:     pkg.Java,
+					Type:         pkg.JenkinsPluginPkg,
+					MetadataType: pkg.JavaMetadataType,
 					Metadata: pkg.JavaMetadata{
+						VirtualPath: "test-fixtures/java-builds/packages/example-jenkins-plugin.hpi",
 						Manifest: &pkg.JavaManifest{
-							ManifestVersion: "1.0",
-							SpecTitle:       "The Jenkins Plugins Parent POM Project",
-							ImplTitle:       "example-jenkins-plugin",
-							ImplVersion:     "1.0-SNAPSHOT",
-							Extra: map[string]string{
+							Main: map[string]string{
+								"Manifest-Version":       "1.0",
+								"Specification-Title":    "The Jenkins Plugins Parent POM Project",
+								"Implementation-Title":   "example-jenkins-plugin",
+								"Implementation-Version": "1.0-SNAPSHOT",
+								// extra fields...
 								"Archiver-Version":    "Plexus Archiver",
 								"Plugin-License-Url":  "https://opensource.org/licenses/MIT",
 								"Plugin-License-Name": "MIT License",
@@ -172,6 +130,7 @@ func TestParseJar(t *testing.T) {
 							GroupID:    "io.jenkins.plugins",
 							ArtifactID: "example-jenkins-plugin",
 							Version:    "1.0-SNAPSHOT",
+							Extra:      map[string]string{},
 						},
 					},
 				},
@@ -181,13 +140,17 @@ func TestParseJar(t *testing.T) {
 			fixture: "test-fixtures/java-builds/packages/example-java-app-gradle-0.1.0.jar",
 			expected: map[string]pkg.Package{
 				"example-java-app-gradle": {
-					Name:     "example-java-app-gradle",
-					Version:  "0.1.0",
-					Language: pkg.Java,
-					Type:     pkg.JavaPkg,
+					Name:         "example-java-app-gradle",
+					Version:      "0.1.0",
+					Language:     pkg.Java,
+					Type:         pkg.JavaPkg,
+					MetadataType: pkg.JavaMetadataType,
 					Metadata: pkg.JavaMetadata{
+						VirtualPath: "test-fixtures/java-builds/packages/example-java-app-gradle-0.1.0.jar",
 						Manifest: &pkg.JavaManifest{
-							ManifestVersion: "1.0",
+							Main: map[string]string{
+								"Manifest-Version": "1.0",
+							},
 						},
 					},
 				},
@@ -200,14 +163,17 @@ func TestParseJar(t *testing.T) {
 			},
 			expected: map[string]pkg.Package{
 				"example-java-app-maven": {
-					Name:     "example-java-app-maven",
-					Version:  "0.1.0",
-					Language: pkg.Java,
-					Type:     pkg.JavaPkg,
+					Name:         "example-java-app-maven",
+					Version:      "0.1.0",
+					Language:     pkg.Java,
+					Type:         pkg.JavaPkg,
+					MetadataType: pkg.JavaMetadataType,
 					Metadata: pkg.JavaMetadata{
+						VirtualPath: "test-fixtures/java-builds/packages/example-java-app-maven-0.1.0.jar",
 						Manifest: &pkg.JavaManifest{
-							ManifestVersion: "1.0",
-							Extra: map[string]string{
+							Main: map[string]string{
+								"Manifest-Version": "1.0",
+								// extra fields...
 								"Archiver-Version": "Plexus Archiver",
 								"Created-By":       "Apache Maven 3.6.3",
 								"Built-By":         "?",
@@ -220,20 +186,26 @@ func TestParseJar(t *testing.T) {
 							GroupID:    "org.anchore",
 							ArtifactID: "example-java-app-maven",
 							Version:    "0.1.0",
+							Extra:      map[string]string{},
 						},
 					},
 				},
 				"joda-time": {
-					Name:     "joda-time",
-					Version:  "2.9.2",
-					Language: pkg.Java,
-					Type:     pkg.JavaPkg,
+					Name:         "joda-time",
+					Version:      "2.9.2",
+					Language:     pkg.Java,
+					Type:         pkg.JavaPkg,
+					MetadataType: pkg.JavaMetadataType,
 					Metadata: pkg.JavaMetadata{
+						// ensure that nested packages with different names than that of the parent are appended as
+						// a suffix on the virtual path
+						VirtualPath: "test-fixtures/java-builds/packages/example-java-app-maven-0.1.0.jar:joda-time",
 						PomProperties: &pkg.PomProperties{
 							Path:       "META-INF/maven/joda-time/joda-time/pom.properties",
 							GroupID:    "joda-time",
 							ArtifactID: "joda-time",
 							Version:    "2.9.2",
+							Extra:      map[string]string{},
 						},
 					},
 				},
@@ -295,11 +267,11 @@ func TestParseJar(t *testing.T) {
 				metadata := a.Metadata.(pkg.JavaMetadata)
 				metadata.Parent = nil
 
-				// ignore select fields
+				// ignore select fields (only works for the main section)
 				for _, field := range test.ignoreExtras {
-					if metadata.Manifest != nil && metadata.Manifest.Extra != nil {
-						if _, ok := metadata.Manifest.Extra[field]; ok {
-							delete(metadata.Manifest.Extra, field)
+					if metadata.Manifest != nil && metadata.Manifest.Main != nil {
+						if _, ok := metadata.Manifest.Main[field]; ok {
+							delete(metadata.Manifest.Main, field)
 						}
 					}
 				}
@@ -527,7 +499,7 @@ func TestParseNestedJar(t *testing.T) {
 				t.Fatalf("failed to parse java archive: %+v", err)
 			}
 
-			nameVersionPairSet := internal.NewStringSet()
+			expectedNameVersionPairSet := internal.NewStringSet()
 
 			makeKey := func(p *pkg.Package) string {
 				if p == nil {
@@ -537,23 +509,31 @@ func TestParseNestedJar(t *testing.T) {
 			}
 
 			for _, e := range test.expected {
-				nameVersionPairSet.Add(makeKey(&e))
+				expectedNameVersionPairSet.Add(makeKey(&e))
 			}
 
-			if len(actual) != len(nameVersionPairSet) {
-				for _, a := range actual {
-					t.Log("   ", a)
+			actualNameVersionPairSet := internal.NewStringSet()
+			for _, a := range actual {
+				key := makeKey(&a)
+				actualNameVersionPairSet.Add(key)
+				if !expectedNameVersionPairSet.Contains(key) {
+					t.Errorf("extra package: %s", a)
 				}
-				t.Fatalf("unexpected package count: %d!=%d", len(actual), len(nameVersionPairSet))
+			}
+
+			for _, key := range expectedNameVersionPairSet.ToSlice() {
+				if !actualNameVersionPairSet.Contains(key) {
+					t.Errorf("missing package: %s", key)
+				}
+			}
+
+			if len(actual) != len(expectedNameVersionPairSet) {
+				t.Fatalf("unexpected package count: %d!=%d", len(actual), len(expectedNameVersionPairSet))
 			}
 
 			for _, a := range actual {
 				actualKey := makeKey(&a)
 
-				if !nameVersionPairSet.Contains(actualKey) {
-					t.Errorf("unexpected pkg: %q", actualKey)
-				}
-
 				metadata := a.Metadata.(pkg.JavaMetadata)
 				if actualKey == "spring-boot|0.0.1-SNAPSHOT" {
 					if metadata.Parent != nil {
@@ -563,7 +543,7 @@ func TestParseNestedJar(t *testing.T) {
 					if metadata.Parent == nil {
 						t.Errorf("unassigned error for pkg=%q", actualKey)
 					} else if makeKey(metadata.Parent) != "spring-boot|0.0.1-SNAPSHOT" {
-						// NB: this is a hard-coded condition to simplify the test harness
+						// NB: this is a hard-coded condition to simplify the test harness to account for https://github.com/micrometer-metrics/micrometer/issues/1785
 						if a.Name == "pcollections" {
 							if metadata.Parent.Name != "micrometer-core" {
 								t.Errorf("nested 'pcollections' pkg has wrong parent: %q", metadata.Parent.Name)

syft/cataloger/java/java_manifest.go

@@ -4,25 +4,41 @@ import (
 	"bufio"
 	"fmt"
 	"io"
+	"strconv"
 	"strings"
 
+	"github.com/anchore/syft/internal/log"
+
 	"github.com/anchore/syft/syft/pkg"
-	"github.com/mitchellh/mapstructure"
 )
 
-const manifestPath = "META-INF/MANIFEST.MF"
+const manifestGlob = "/META-INF/MANIFEST.MF"
 
-func parseJavaManifest(reader io.Reader) (*pkg.JavaManifest, error) {
+// nolint:funlen
+// parseJavaManifest takes MANIFEST.MF file content and returns sections of parsed key/value pairs.
+// For more information: https://docs.oracle.com/en/java/javase/11/docs/specs/jar/jar.html#jar-manifest
+func parseJavaManifest(path string, reader io.Reader) (*pkg.JavaManifest, error) {
 	var manifest pkg.JavaManifest
-	manifestMap := make(map[string]string)
+	sections := []map[string]string{
+		make(map[string]string),
+	}
+	currentSection := 0
 	scanner := bufio.NewScanner(reader)
 	var lastKey string
 	for scanner.Scan() {
 		line := scanner.Text()
 
-		// ignore empty lines
+		// empty lines denote section separators
 		if strings.TrimSpace(line) == "" {
+			currentSection++
+			// we don't want to allocate a new section map that wont necessarily be used, do that once there is
+			// a non-empty line to process
+
+			// do not process line continuations after this
+			lastKey = ""
 			continue
+		} else if currentSection >= len(sections) {
+			sections = append(sections, make(map[string]string))
 		}
 
 		if line[0] == ' ' {
@@ -30,7 +46,7 @@ func parseJavaManifest(reader io.Reader) (*pkg.JavaManifest, error) {
 			if lastKey == "" {
 				return nil, fmt.Errorf("found continuation with no previous key (%s)", line)
 			}
-			manifestMap[lastKey] += strings.TrimSpace(line)
+			sections[currentSection][lastKey] += strings.TrimSpace(line)
 		} else {
 			// this is a new key-value pair
 			idx := strings.Index(line, ":")
@@ -40,9 +56,9 @@ func parseJavaManifest(reader io.Reader) (*pkg.JavaManifest, error) {
 
 			key := strings.TrimSpace(line[0:idx])
 			value := strings.TrimSpace(line[idx+1:])
-			manifestMap[key] = value
+			sections[currentSection][key] = value
 
-			// keep track of key for future continuations
+			// keep track of key for potential future continuations
 			lastKey = key
 		}
 	}
@@ -51,14 +67,24 @@ func parseJavaManifest(reader io.Reader) (*pkg.JavaManifest, error) {
 		return nil, fmt.Errorf("unable to read java manifest: %w", err)
 	}
 
-	if err := mapstructure.Decode(manifestMap, &manifest); err != nil {
-		return nil, fmt.Errorf("unable to parse java manifest: %w", err)
-	}
-
-	// clean select fields
-	if strings.Trim(manifest.ImplVersion, " ") != "" {
-		// transform versions with dates attached to just versions (e.g. "1.3 2244 October 5 2008" --> "1.3")
-		manifest.ImplVersion = strings.Split(manifest.ImplVersion, " ")[0]
+	if len(sections) > 0 {
+		manifest.Main = sections[0]
+		if len(sections) > 1 {
+			manifest.NamedSections = make(map[string]map[string]string)
+			for i, s := range sections[1:] {
+				name, ok := s["Name"]
+				if !ok {
+					// per the manifest spec (https://docs.oracle.com/en/java/javase/11/docs/specs/jar/jar.html#jar-manifest)
+					// this should never happen. If it does we want to know about it, but not necessarily stop
+					// cataloging entirely... for this reason we only log.
+					log.Warnf("java manifest section found without a name: %s", path)
+					name = strconv.Itoa(i)
+				} else {
+					delete(s, "Name")
+				}
+				manifest.NamedSections[name] = s
+			}
+		}
 	}
 
 	return &manifest, nil
@@ -67,26 +93,23 @@ func parseJavaManifest(reader io.Reader) (*pkg.JavaManifest, error) {
 func selectName(manifest *pkg.JavaManifest, filenameObj archiveFilename) string {
 	var name string
 	switch {
-	case filenameObj.name() != "":
-		name = filenameObj.name()
-	case manifest.Name != "":
+	case filenameObj.name != "":
+		name = filenameObj.name
+	case manifest.Main["Name"] != "":
 		// Manifest original spec...
-		name = manifest.Name
-	case manifest.Extra["Bundle-Name"] != "":
+		name = manifest.Main["Name"]
+	case manifest.Main["Bundle-Name"] != "":
 		// BND tooling...
-		name = manifest.Extra["Bundle-Name"]
-	case manifest.Extra["Short-Name"] != "":
+		name = manifest.Main["Bundle-Name"]
+	case manifest.Main["Short-Name"] != "":
 		// Jenkins...
-		name = manifest.Extra["Short-Name"]
-	case manifest.Extra["Extension-Name"] != "":
+		name = manifest.Main["Short-Name"]
+	case manifest.Main["Extension-Name"] != "":
 		// Jenkins...
-		name = manifest.Extra["Extension-Name"]
-	}
-
-	// in situations where we hit this point and no name was
-	// determined, look at the Implementation-Title
-	if name == "" && manifest.ImplTitle != "" {
-		name = manifest.ImplTitle
+		name = manifest.Main["Extension-Name"]
+	case manifest.Main["Implementation-Title"] != "":
+		// last ditch effort...
+		name = manifest.Main["Implementation-Title"]
 	}
 	return name
 }
@@ -94,14 +117,14 @@ func selectName(manifest *pkg.JavaManifest, filenameObj archiveFilename) string
 func selectVersion(manifest *pkg.JavaManifest, filenameObj archiveFilename) string {
 	var version string
 	switch {
-	case manifest.ImplVersion != "":
-		version = manifest.ImplVersion
-	case filenameObj.version() != "":
-		version = filenameObj.version()
-	case manifest.SpecVersion != "":
-		version = manifest.SpecVersion
-	case manifest.Extra["Plugin-Version"] != "":
-		version = manifest.Extra["Plugin-Version"]
+	case filenameObj.version != "":
+		version = filenameObj.version
+	case manifest.Main["Implementation-Version"] != "":
+		version = manifest.Main["Implementation-Version"]
+	case manifest.Main["Specification-Version"] != "":
+		version = manifest.Main["Specification-Version"]
+	case manifest.Main["Plugin-Version"] != "":
+		version = manifest.Main["Plugin-Version"]
 	}
 	return version
 }

syft/cataloger/java/java_manifest_test.go

@@ -2,10 +2,11 @@ package java
 
 import (
 	"encoding/json"
-	"github.com/anchore/syft/syft/pkg"
-	"github.com/go-test/deep"
 	"os"
 	"testing"
+
+	"github.com/anchore/syft/syft/pkg"
+	"github.com/go-test/deep"
 )
 
 func TestParseJavaManifest(t *testing.T) {
@@ -16,49 +17,62 @@ func TestParseJavaManifest(t *testing.T) {
 		{
 			fixture: "test-fixtures/manifest/small",
 			expected: pkg.JavaManifest{
-				ManifestVersion: "1.0",
+				Main: map[string]string{
+					"Manifest-Version": "1.0",
+				},
 			},
 		},
 		{
 			fixture: "test-fixtures/manifest/standard-info",
 			expected: pkg.JavaManifest{
-				ManifestVersion: "1.0",
-				Name:            "the-best-name",
-				SpecTitle:       "the-spec-title",
-				SpecVersion:     "the-spec-version",
-				SpecVendor:      "the-spec-vendor",
-				ImplTitle:       "the-impl-title",
-				ImplVersion:     "the-impl-version",
-				ImplVendor:      "the-impl-vendor",
+				Main: map[string]string{
+					"Name":                   "the-best-name",
+					"Manifest-Version":       "1.0",
+					"Specification-Title":    "the-spec-title",
+					"Specification-Version":  "the-spec-version",
+					"Specification-Vendor":   "the-spec-vendor",
+					"Implementation-Title":   "the-impl-title",
+					"Implementation-Version": "the-impl-version",
+					"Implementation-Vendor":  "the-impl-vendor",
+				},
 			},
 		},
 		{
 			fixture: "test-fixtures/manifest/extra-info",
 			expected: pkg.JavaManifest{
-				ManifestVersion: "1.0",
-				Extra: map[string]string{
+				Main: map[string]string{
+					"Manifest-Version": "1.0",
 					"Archiver-Version": "Plexus Archiver",
-					"Build-Jdk":        "14.0.1",
-					"Built-By":         "?",
 					"Created-By":       "Apache Maven 3.6.3",
-					"Main-Class":       "hello.HelloWorld",
+				},
+				NamedSections: map[string]map[string]string{
+					"thing-1": {
+						"Built-By": "?",
+					},
+					"1": {
+						"Build-Jdk":  "14.0.1",
+						"Main-Class": "hello.HelloWorld",
+					},
 				},
 			},
 		},
 		{
 			fixture: "test-fixtures/manifest/continuation",
 			expected: pkg.JavaManifest{
-				ManifestVersion: "1.0",
-				Extra: map[string]string{
-					"Plugin-ScmUrl": "https://github.com/jenkinsci/plugin-pom/example-jenkins-plugin",
+				Main: map[string]string{
+					"Manifest-Version": "1.0",
+					"Plugin-ScmUrl":    "https://github.com/jenkinsci/plugin-pom/example-jenkins-plugin",
 				},
 			},
 		},
 		{
+			// regression test, we should always keep the full version
 			fixture: "test-fixtures/manifest/version-with-date",
 			expected: pkg.JavaManifest{
-				ManifestVersion: "1.0",
-				ImplVersion:     "1.3", // ensure the date is stripped off during processing
+				Main: map[string]string{
+					"Manifest-Version":       "1.0",
+					"Implementation-Version": "1.3 2244 October 5 2005",
+				},
 			},
 		},
 	}
@@ -70,7 +84,7 @@ func TestParseJavaManifest(t *testing.T) {
 				t.Fatalf("could not open fixture: %+v", err)
 			}
 
-			actual, err := parseJavaManifest(fixture)
+			actual, err := parseJavaManifest(test.fixture, fixture)
 			if err != nil {
 				t.Fatalf("failed to parse manifest: %+v", err)
 			}
@@ -91,3 +105,45 @@ func TestParseJavaManifest(t *testing.T) {
 		})
 	}
 }
+
+func TestSelectName(t *testing.T) {
+	tests := []struct {
+		desc     string
+		manifest pkg.JavaManifest
+		archive  archiveFilename
+		expected string
+	}{
+		{
+			desc:    "Get name from Implementation-Title",
+			archive: archiveFilename{},
+			manifest: pkg.JavaManifest{
+				Main: map[string]string{
+					"Implementation-Title": "maven-wrapper",
+				},
+			},
+			expected: "maven-wrapper",
+		},
+		{
+			desc: "Implementation-Title does not override name from filename",
+			manifest: pkg.JavaManifest{
+				Main: map[string]string{
+					"Name":                 "foo",
+					"Implementation-Title": "maven-wrapper",
+				},
+			},
+			archive:  newJavaArchiveFilename("/something/omg.jar"),
+			expected: "omg",
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.desc, func(t *testing.T) {
+			result := selectName(&test.manifest, test.archive)
+
+			if result != test.expected {
+				t.Errorf("mismatch in names: '%s' != '%s'", result, test.expected)
+			}
+		})
+	}
+
+}

syft/cataloger/java/pom_properties.go

@@ -42,6 +42,11 @@ func parsePomProperties(path string, reader io.Reader) (*pkg.PomProperties, erro
 		return nil, fmt.Errorf("unable to parse pom.properties: %w", err)
 	}
 
+	// don't allow for a nil collection, ensure it is empty
+	if props.Extra == nil {
+		props.Extra = make(map[string]string)
+	}
+
 	props.Path = path
 
 	return &props, nil

syft/cataloger/java/pom_properties_test.go

@@ -2,10 +2,11 @@ package java
 
 import (
 	"encoding/json"
-	"github.com/anchore/syft/syft/pkg"
-	"github.com/go-test/deep"
 	"os"
 	"testing"
+
+	"github.com/anchore/syft/syft/pkg"
+	"github.com/go-test/deep"
 )
 
 func TestParseJavaPomProperties(t *testing.T) {
@@ -20,6 +21,7 @@ func TestParseJavaPomProperties(t *testing.T) {
 				GroupID:    "org.anchore",
 				ArtifactID: "example-java-app-maven",
 				Version:    "0.1.0",
+				Extra:      map[string]string{},
 			},
 		},
 		{

syft/cataloger/java/save_archive_to_tmp.go

@@ -35,6 +35,7 @@ func saveArchiveToTmp(reader io.Reader) (string, string, func(), error) {
 	if err != nil {
 		return contentDir, "", cleanupFn, fmt.Errorf("unable to create archive: %w", err)
 	}
+	defer archiveFile.Close()
 
 	_, err = io.Copy(archiveFile, reader)
 	if err != nil {

syft/cataloger/java/test-fixtures/manifest/extra-info

@@ -2,6 +2,7 @@ Manifest-Version: 1.0
 Archiver-Version: Plexus Archiver
 Created-By: Apache Maven 3.6.3
 
+Name: thing-1
 Built-By: ?
 
 Build-Jdk: 14.0.1

syft/cataloger/javascript/cataloger.go

@@ -7,12 +7,21 @@ import (
 	"github.com/anchore/syft/syft/cataloger/common"
 )
 
-// NewJavascriptCataloger returns a new JavaScript cataloger object.
-func NewJavascriptCataloger() *common.GenericCataloger {
+// NewJavascriptPackageCataloger returns a new JavaScript cataloger object based on detection of npm based packages.
+func NewJavascriptPackageCataloger() *common.GenericCataloger {
+	globParsers := map[string]common.ParserFn{
+		"**/package.json": parsePackageJSON,
+	}
+
+	return common.NewGenericCataloger(nil, globParsers, "javascript-package-cataloger")
+}
+
+// NewJavascriptLockCataloger returns a new Javascript cataloger object base on package lock files.
+func NewJavascriptLockCataloger() *common.GenericCataloger {
 	globParsers := map[string]common.ParserFn{
 		"**/package-lock.json": parsePackageLock,
 		"**/yarn.lock":         parseYarnLock,
 	}
 
-	return common.NewGenericCataloger(nil, globParsers, "javascript-cataloger")
+	return common.NewGenericCataloger(nil, globParsers, "javascript-lock-cataloger")
 }

syft/cataloger/javascript/parse_package_json.go

@@ -0,0 +1,197 @@
+package javascript
+
+import (
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io"
+	"regexp"
+
+	"github.com/anchore/syft/internal"
+
+	"github.com/mitchellh/mapstructure"
+
+	"github.com/anchore/syft/syft/cataloger/common"
+	"github.com/anchore/syft/syft/pkg"
+)
+
+// integrity check
+var _ common.ParserFn = parsePackageLock
+
+// PackageJSON represents a JavaScript package.json file
+type PackageJSON struct {
+	Version      string            `json:"version"`
+	Latest       []string          `json:"latest"`
+	Author       Author            `json:"author"`
+	License      json.RawMessage   `json:"license"`
+	Licenses     []license         `json:"licenses"`
+	Name         string            `json:"name"`
+	Homepage     string            `json:"homepage"`
+	Description  string            `json:"description"`
+	Dependencies map[string]string `json:"dependencies"`
+	Repository   Repository        `json:"repository"`
+}
+
+type Author struct {
+	Name  string `json:"name" mapstruct:"name"`
+	Email string `json:"email" mapstruct:"email"`
+	URL   string `json:"url" mapstruct:"url"`
+}
+
+type Repository struct {
+	Type string `json:"type" mapstructure:"type"`
+	URL  string `json:"url" mapstructure:"url"`
+}
+
+// match example: "author": "Isaac Z. Schlueter <i@izs.me> (http://blog.izs.me)"
+// ---> name: "Isaac Z. Schlueter" email: "i@izs.me" url: "http://blog.izs.me"
+var authorPattern = regexp.MustCompile(`^\s*(?P<name>[^<(]*)(\s+<(?P<email>.*)>)?(\s\((?P<url>.*)\))?\s*$`)
+
+// Exports Author.UnmarshalJSON interface to help normalize the json structure.
+func (a *Author) UnmarshalJSON(b []byte) error {
+	var authorStr string
+	var fields map[string]string
+	var author Author
+
+	if err := json.Unmarshal(b, &authorStr); err != nil {
+		// string parsing did not work, assume a map was given
+		// for more information: https://docs.npmjs.com/files/package.json#people-fields-author-contributors
+		if err := json.Unmarshal(b, &fields); err != nil {
+			return fmt.Errorf("unable to parse package.json author: %w", err)
+		}
+	} else {
+		// parse out "name <email> (url)" into an Author struct
+		fields = internal.MatchCaptureGroups(authorPattern, authorStr)
+	}
+
+	// translate the map into a structure
+	if err := mapstructure.Decode(fields, &author); err != nil {
+		return fmt.Errorf("unable to decode package.json author: %w", err)
+	}
+
+	*a = author
+
+	return nil
+}
+
+func (a *Author) AuthorString() string {
+	result := a.Name
+	if a.Email != "" {
+		result += fmt.Sprintf(" <%s>", a.Email)
+	}
+	if a.URL != "" {
+		result += fmt.Sprintf(" (%s)", a.URL)
+	}
+	return result
+}
+
+func (r *Repository) UnmarshalJSON(b []byte) error {
+	var repositoryStr string
+	var fields map[string]string
+	var repository Repository
+
+	if err := json.Unmarshal(b, &repositoryStr); err != nil {
+		// string parsing did not work, assume a map was given
+		// for more information: https://docs.npmjs.com/files/package.json#people-fields-author-contributors
+		if err := json.Unmarshal(b, &fields); err != nil {
+			return fmt.Errorf("unable to parse package.json author: %w", err)
+		}
+		// translate the map into a structure
+		if err := mapstructure.Decode(fields, &repository); err != nil {
+			return fmt.Errorf("unable to decode package.json author: %w", err)
+		}
+
+		*r = repository
+	} else {
+		r.URL = repositoryStr
+	}
+
+	return nil
+}
+
+type license struct {
+	Type string `json:"type"`
+	URL  string `json:"url"`
+}
+
+func licenseFromJSON(b []byte) (string, error) {
+	// first try as string
+	var licenseString string
+	err := json.Unmarshal(b, &licenseString)
+	if err == nil {
+		return licenseString, nil
+	}
+
+	// then try as object (this format is deprecated)
+	var licenseObject license
+	err = json.Unmarshal(b, &licenseObject)
+	if err == nil {
+		return licenseObject.Type, nil
+	}
+
+	return "", errors.New("unable to unmarshal license field as either string or object")
+}
+
+func licensesFromJSON(p PackageJSON) ([]string, error) {
+	if p.License == nil && p.Licenses == nil {
+		// This package.json doesn't specify any licenses whatsoever
+		return []string{}, nil
+	}
+
+	singleLicense, err := licenseFromJSON(p.License)
+	if err == nil {
+		return []string{singleLicense}, nil
+	}
+
+	// The "licenses" field is deprecated. It should be inspected as a last resort.
+	if p.Licenses != nil {
+		mapLicenses := func(licenses []license) []string {
+			mappedLicenses := make([]string, len(licenses))
+			for i, l := range licenses {
+				mappedLicenses[i] = l.Type
+			}
+			return mappedLicenses
+		}
+
+		return mapLicenses(p.Licenses), nil
+	}
+
+	return nil, fmt.Errorf("unable to parse license field: %w", err)
+}
+
+// parsePackageJson parses a package.json and returns the discovered JavaScript packages.
+func parsePackageJSON(_ string, reader io.Reader) ([]pkg.Package, error) {
+	packages := make([]pkg.Package, 0)
+	dec := json.NewDecoder(reader)
+
+	for {
+		var p PackageJSON
+		if err := dec.Decode(&p); err == io.EOF {
+			break
+		} else if err != nil {
+			return nil, fmt.Errorf("failed to parse package.json file: %w", err)
+		}
+
+		licenses, err := licensesFromJSON(p)
+		if err != nil {
+			return nil, fmt.Errorf("failed to parse package.json file: %w", err)
+		}
+
+		packages = append(packages, pkg.Package{
+			Name:         p.Name,
+			Version:      p.Version,
+			Licenses:     licenses,
+			Language:     pkg.JavaScript,
+			Type:         pkg.NpmPkg,
+			MetadataType: pkg.NpmPackageJSONMetadataType,
+			Metadata: pkg.NpmPackageJSONMetadata{
+				Author:   p.Author.AuthorString(),
+				Homepage: p.Homepage,
+				URL:      p.Repository.URL,
+				Licenses: licenses,
+			},
+		})
+	}
+
+	return packages, nil
+}

syft/cataloger/javascript/parse_package_json_test.go

@@ -0,0 +1,144 @@
+package javascript
+
+import (
+	"os"
+	"testing"
+
+	"github.com/anchore/syft/syft/pkg"
+	"github.com/go-test/deep"
+)
+
+func TestParsePackageJSON(t *testing.T) {
+	tests := []struct {
+		Fixture     string
+		ExpectedPkg pkg.Package
+	}{
+		{
+			Fixture: "test-fixtures/pkg-json/package.json",
+			ExpectedPkg: pkg.Package{
+				Name:         "npm",
+				Version:      "6.14.6",
+				Type:         pkg.NpmPkg,
+				Licenses:     []string{"Artistic-2.0"},
+				Language:     pkg.JavaScript,
+				MetadataType: pkg.NpmPackageJSONMetadataType,
+				Metadata: pkg.NpmPackageJSONMetadata{
+					Author:   "Isaac Z. Schlueter <i@izs.me> (http://blog.izs.me)",
+					Homepage: "https://docs.npmjs.com/",
+					URL:      "https://github.com/npm/cli",
+					Licenses: []string{"Artistic-2.0"},
+				},
+			},
+		},
+		{
+			Fixture: "test-fixtures/pkg-json/package-license-object.json",
+			ExpectedPkg: pkg.Package{
+				Name:         "npm",
+				Version:      "6.14.6",
+				Type:         pkg.NpmPkg,
+				Licenses:     []string{"ISC"},
+				Language:     pkg.JavaScript,
+				MetadataType: pkg.NpmPackageJSONMetadataType,
+				Metadata: pkg.NpmPackageJSONMetadata{
+					Author:   "Isaac Z. Schlueter <i@izs.me> (http://blog.izs.me)",
+					Homepage: "https://docs.npmjs.com/",
+					URL:      "https://github.com/npm/cli",
+					Licenses: []string{"ISC"},
+				},
+			},
+		},
+		{
+			Fixture: "test-fixtures/pkg-json/package-license-objects.json",
+			ExpectedPkg: pkg.Package{
+				Name:         "npm",
+				Version:      "6.14.6",
+				Type:         pkg.NpmPkg,
+				Licenses:     []string{"MIT", "Apache-2.0"},
+				Language:     pkg.JavaScript,
+				MetadataType: pkg.NpmPackageJSONMetadataType,
+				Metadata: pkg.NpmPackageJSONMetadata{
+					Author:   "Isaac Z. Schlueter <i@izs.me> (http://blog.izs.me)",
+					Homepage: "https://docs.npmjs.com/",
+					URL:      "https://github.com/npm/cli",
+					Licenses: []string{"MIT", "Apache-2.0"},
+				},
+			},
+		},
+		{
+			Fixture: "test-fixtures/pkg-json/package-no-license.json",
+			ExpectedPkg: pkg.Package{
+				Name:         "npm",
+				Version:      "6.14.6",
+				Type:         pkg.NpmPkg,
+				Licenses:     []string{},
+				Language:     pkg.JavaScript,
+				MetadataType: pkg.NpmPackageJSONMetadataType,
+				Metadata: pkg.NpmPackageJSONMetadata{
+					Author:   "Isaac Z. Schlueter <i@izs.me> (http://blog.izs.me)",
+					Homepage: "https://docs.npmjs.com/",
+					URL:      "https://github.com/npm/cli",
+					Licenses: []string{},
+				},
+			},
+		},
+		{
+			Fixture: "test-fixtures/pkg-json/package-nested-author.json",
+			ExpectedPkg: pkg.Package{
+				Name:         "npm",
+				Version:      "6.14.6",
+				Type:         pkg.NpmPkg,
+				Licenses:     []string{"Artistic-2.0"},
+				Language:     pkg.JavaScript,
+				MetadataType: pkg.NpmPackageJSONMetadataType,
+				Metadata: pkg.NpmPackageJSONMetadata{
+					Author:   "Isaac Z. Schlueter <i@izs.me> (http://blog.izs.me)",
+					Homepage: "https://docs.npmjs.com/",
+					URL:      "https://github.com/npm/cli",
+					Licenses: []string{"Artistic-2.0"},
+				},
+			},
+		},
+		{
+			Fixture: "test-fixtures/pkg-json/package-repo-string.json",
+			ExpectedPkg: pkg.Package{
+				Name:         "function-bind",
+				Version:      "1.1.1",
+				Type:         pkg.NpmPkg,
+				Licenses:     []string{"MIT"},
+				Language:     pkg.JavaScript,
+				MetadataType: pkg.NpmPackageJSONMetadataType,
+				Metadata: pkg.NpmPackageJSONMetadata{
+					Author:   "Raynos <raynos2@gmail.com>",
+					Homepage: "https://github.com/Raynos/function-bind",
+					URL:      "git://github.com/Raynos/function-bind.git",
+					Licenses: []string{"MIT"},
+				},
+			},
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.Fixture, func(t *testing.T) {
+			fixture, err := os.Open(test.Fixture)
+			if err != nil {
+				t.Fatalf("failed to open fixture: %+v", err)
+			}
+
+			actual, err := parsePackageJSON("", fixture)
+			if err != nil {
+				t.Fatalf("failed to parse package-lock.json: %+v", err)
+			}
+			if len(actual) != 1 {
+				for _, a := range actual {
+					t.Log("   ", a)
+				}
+				t.Fatalf("unexpected package count: %d!=1", len(actual))
+			}
+
+			for _, d := range deep.Equal(actual[0], test.ExpectedPkg) {
+
+				t.Errorf("diff: %+v", d)
+			}
+		})
+	}
+}