Merge pull request #262 from anchore/fix-release

Fix release process

.github/scripts/mac-prepare-for-signing.sh

@@ -0,0 +1,31 @@
+#!/usr/bin/env bash
+set -eu
+
+CI_HOME="/Users/runner"
+if [[ "${HOME}" != "${CI_HOME}" ]]; then
+  printf "WARNING! It looks like this isn't the CI environment. This script modifies the macOS Keychain setup in ways you probably wouldn't want for your own machine. It also requires an Apple Developer ID Certificate that you shouldn't have outside of the CI environment.\n\nExiting early to make sure nothing bad happens.\n"
+  exit 1
+fi
+
+# Install gon (see https://github.com/mitchellh/gon for details).
+brew tap mitchellh/gon
+brew install mitchellh/gon/gon
+
+# Write signing certificate to disk from environment variable.
+CERT_FILE="$HOME/developer_id_certificate.p12"
+echo -n "$APPLE_DEVELOPER_ID_CERT" | base64 --decode > "$CERT_FILE"
+
+# In order to have all keychain interactions avoid an interactive user prompt, we need to control the password for the keychain in question, which means we need to create a new keychain into which we'll import the signing certificate and from which we'll later access this certificate during code signing.
+EPHEMERAL_KEYCHAIN="ci-ephemeral-keychain"
+EPHEMERAL_KEYCHAIN_PASSWORD="$(openssl rand -base64 100)"
+security create-keychain -p "${EPHEMERAL_KEYCHAIN_PASSWORD}" "${EPHEMERAL_KEYCHAIN}"
+
+# Import signing certificate into the keychain. (This is a pre-requisite for gon, which is invoked via goreleaser.)
+EPHEMERAL_KEYCHAIN_FULL_PATH="$HOME/Library/Keychains/${EPHEMERAL_KEYCHAIN}-db"
+security import "${CERT_FILE}" -k "${EPHEMERAL_KEYCHAIN_FULL_PATH}" -P "${APPLE_DEVELOPER_ID_CERT_PASS}" -T "$(command -v codesign)"
+
+# Setting the partition list for this certificate's private key to include "apple-tool:" and "apple:" allows the codesign command to access this keychain item without an interactive user prompt. (codesign is invoked by gon.)
+security set-key-partition-list -S "apple-tool:,apple:" -s -k "${EPHEMERAL_KEYCHAIN_PASSWORD}" "${EPHEMERAL_KEYCHAIN_FULL_PATH}"
+
+# Make this new keychain the user's default keychain, so that codesign will be able to find this certificate when we specify it during signing.
+security default-keychain -d "user" -s "${EPHEMERAL_KEYCHAIN_FULL_PATH}"

.github/scripts/mac-sign-and-notarize.sh

@@ -0,0 +1,19 @@
+#!/usr/bin/env bash
+set -eu
+
+IS_SNAPSHOT="$1"                        # e.g. "true", "false"
+
+if [[ "${IS_SNAPSHOT}" == "true" ]]; then
+  # This is a snapshot build —— skipping signing and notarization...
+  exit 0
+fi
+
+GON_CONFIG="$2"                                       # e.g. "gon.hcl"
+NEW_NAME_WITHOUT_EXTENSION="$3"                       # e.g. "./dist/syft-0.1.0"
+ORIGINAL_NAME_WITHOUT_EXTENSION="./dist/output"       # This should match dmg and zip output_path in the gon config file, without the extension.
+
+gon "${GON_CONFIG}"
+
+# Rename outputs with specified desired name
+mv -v "${ORIGINAL_NAME_WITHOUT_EXTENSION}.dmg" "${NEW_NAME_WITHOUT_EXTENSION}.dmg"
+mv -v "${ORIGINAL_NAME_WITHOUT_EXTENSION}.zip" "${NEW_NAME_WITHOUT_EXTENSION}.zip"

.github/scripts/update-version-file.sh

@@ -15,14 +15,9 @@ fi
 echo "creating and publishing version file"
 
 # create a version file for version-update checks
-echo "${VERSION}" | tee ${DISTDIR}/VERSION
+VERSION_FILE="${DISTDIR}/VERSION"
+echo "${VERSION}" | tee "${VERSION_FILE}"
 
 # upload the version file that supports the application version update check
-docker run --rm \
-    -i \
-    -e AWS_DEFAULT_REGION=us-west-2 \
-    -e AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID} \
-    -e AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY} \
-    -v $(pwd)/${DISTDIR}/:/distmount \
-    amazon/aws-cli \
-        s3 cp /distmount/VERSION s3://toolbox-data.anchore.io/${BIN}/releases/latest/VERSION
+export AWS_DEFAULT_REGION=us-west-2
+aws s3 cp "${VERSION_FILE}" s3://toolbox-data.anchore.io/${BIN}/releases/latest/VERSION

.github/workflows/acceptance-test.yaml

@@ -14,7 +14,7 @@ env:
 
 jobs:
   Build-Snapshot-Artifacts:
-    runs-on: ubuntu-latest
+    runs-on: macos-latest # We're creating these snapshot builds on macOS to be consistent with our release workflow's build process, which also takes place on macOS (due to code signing requirements).
     steps:
 
       - uses: actions/setup-go@v2
@@ -35,9 +35,9 @@ jobs:
             ${{ runner.os }}-go-${{ env.GO_VERSION }}-${{ hashFiles('**/go.sum') }}-
             ${{ runner.os }}-go-${{ env.GO_VERSION }}-
 
-      - name: Bootstrap dependencies
-        if: steps.cache.outputs.cache-hit != 'true'
-        run: make ci-bootstrap
+      - name: Bootstrap project dependencies
+        if: steps.bootstrap-cache.outputs.cache-hit != 'true'
+        run: make bootstrap
 
       - name: Import GPG key
         id: import_gpg
@@ -76,7 +76,8 @@ jobs:
   # Note: changing this job name requires making the same update in the .github/workflows/release.yaml pipeline
   Acceptance-Linux:
     needs: [ Build-Snapshot-Artifacts ]
-    runs-on: ubuntu-latest
+    # come Nov 30 2020 ubuntu-latest will be ubuntu-20.04, until then it needs to be explicitly referenced due to python 3.7 specific features being used
+    runs-on: ubuntu-20.04
     steps:
 
       - uses: actions/checkout@v2
@@ -126,7 +127,8 @@ jobs:
   # Note: changing this job name requires making the same update in the .github/workflows/release.yaml pipeline
   Inline-Compare:
     needs: [ Build-Snapshot-Artifacts ]
-    runs-on: ubuntu-latest
+    # come Nov 30 2020 ubuntu-latest will be ubuntu-20.04, until then it needs to be explicitly referenced due to python 3.7 specific features being used
+    runs-on: ubuntu-20.04
     steps:
 
       - uses: actions/checkout@v2
@@ -156,4 +158,4 @@ jobs:
           text: The syft acceptance tests have failed tragically!
         env:
           SLACK_WEBHOOK_URL: ${{ secrets.SLACK_TOOLBOX_WEBHOOK_URL }}
-        if: ${{ failure() }}
+        if: ${{ failure() }}

.github/workflows/codeql-analysis.yml

@@ -0,0 +1,71 @@
+# For most projects, this workflow file will not need changing; you simply need
+# to commit it to your repository.
+#
+# You may wish to alter this file to override the set of languages analyzed,
+# or to provide custom queries or build logic.
+name: "CodeQL Security Scan"
+
+on:
+  push:
+    branches: [main]
+  pull_request:
+    # The branches below must be a subset of the branches above
+    branches: [main]
+  schedule:
+    - cron: '0 0 * * 3'
+
+jobs:
+  analyze:
+    name: Analyze
+    runs-on: ubuntu-latest
+
+    strategy:
+      fail-fast: false
+      matrix:
+        # Override automatic language detection by changing the below list
+        # Supported options are ['csharp', 'cpp', 'go', 'java', 'javascript', 'python']
+        language: ['go', 'python']
+        # Learn more...
+        # https://docs.github.com/en/github/finding-security-vulnerabilities-and-errors-in-your-code/configuring-code-scanning#overriding-automatic-language-detection
+
+    steps:
+    - name: Checkout repository
+      uses: actions/checkout@v2
+      with:
+        # We must fetch at least the immediate parents so that if this is
+        # a pull request then we can checkout the head.
+        fetch-depth: 2
+
+    # If this run was triggered by a pull request event, then checkout
+    # the head of the pull request instead of the merge commit.
+    - run: git checkout HEAD^2
+      if: ${{ github.event_name == 'pull_request' }}
+
+    # Initializes the CodeQL tools for scanning.
+    - name: Initialize CodeQL
+      uses: github/codeql-action/init@v1
+      with:
+        languages: ${{ matrix.language }}
+        # If you wish to specify custom queries, you can do so here or in a config file.
+        # By default, queries listed here will override any specified in a config file. 
+        # Prefix the list here with "+" to use these queries and those in the config file.
+        # queries: ./path/to/local/query, your-org/your-repo/queries@main
+
+    # Autobuild attempts to build any compiled languages  (C/C++, C#, or Java).
+    # If this step fails, then you should remove it and run the build manually (see below)
+    - name: Autobuild
+      uses: github/codeql-action/autobuild@v1
+
+    # ℹ️ Command-line programs to run using the OS shell.
+    # 📚 https://git.io/JvXDl
+
+    # ✏️ If the Autobuild fails above, remove it and uncomment the following three lines
+    #    and modify them (or add more) to build your code if your project
+    #    uses a compiled language
+
+    #- run: |
+    #   make bootstrap
+    #   make release
+
+    - name: Perform CodeQL Analysis
+      uses: github/codeql-action/analyze@v1

.github/workflows/release.yaml

@@ -13,7 +13,7 @@ env:
 
 jobs:
   wait-for-checks:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-latest # This OS choice is arbitrary. None of the steps in this job are specific to either Linux or macOS.
     steps:
 
       - uses: actions/checkout@v2
@@ -82,7 +82,7 @@ jobs:
 
   release:
     needs: [ wait-for-checks ]
-    runs-on: ubuntu-latest
+    runs-on: macos-latest # Due to our code signing process, it's vital that we run our release steps on macOS.
     steps:
 
       - uses: actions/setup-go@v2
@@ -93,6 +93,7 @@ jobs:
         with:
           fetch-depth: 0
 
+      # We are expecting this cache to have been created during the "Build-Snapshot-Artifacts" job in the "Acceptance" workflow.
       - name: Restore bootstrap cache
         id: cache
         uses: actions/cache@v2
@@ -105,10 +106,6 @@ jobs:
             ${{ runner.os }}-go-${{ env.GO_VERSION }}-${{ hashFiles('**/go.sum') }}-
             ${{ runner.os }}-go-${{ env.GO_VERSION }}-
 
-      - name: Bootstrap dependencies
-        if: steps.cache.outputs.cache-hit != 'true'
-        run: make ci-bootstrap
-
       - name: Import GPG key
         id: import_gpg
         uses: crazy-max/ghaction-import-gpg@v2
@@ -132,6 +129,10 @@ jobs:
           SIGNING_FINGERPRINT: ${{ steps.import_gpg.outputs.fingerprint }}
           AWS_ACCESS_KEY_ID: ${{ secrets.TOOLBOX_AWS_ACCESS_KEY_ID }}
           AWS_SECRET_ACCESS_KEY: ${{ secrets.TOOLBOX_AWS_SECRET_ACCESS_KEY }}
+          APPLE_DEVELOPER_ID_CERT: ${{ secrets.APPLE_DEVELOPER_ID_CERT }} # Used during macOS code signing.
+          APPLE_DEVELOPER_ID_CERT_PASS: ${{ secrets.APPLE_DEVELOPER_ID_CERT_PASS }} # Used during macOS code signing.
+          AC_USERNAME: ${{ secrets.ENG_CI_APPLE_ID }} # Used during macOS notarization.
+          AC_PASSWORD: ${{ secrets.ENG_CI_APPLE_ID_PASS }} # Used during macOS notarization.
 
       - uses: 8398a7/action-slack@v3
         with:
@@ -145,4 +146,4 @@ jobs:
       - uses: actions/upload-artifact@v2
         with:
           name: artifacts
-          path: dist/**/*
+          path: dist/**/*

.gitignore

@@ -29,3 +29,6 @@ coverage.txt
 
 # Output of the go coverage tool, specifically when used with LiteIDE
 *.out
+
+# macOS Finder metadata
+.DS_STORE

.goreleaser.yaml

@@ -6,18 +6,33 @@ release:
   # If set to true, will not auto-publish the release. This is done to allow us to review the changelog before publishing.
   draft: true
 
-signs:
-  - artifacts: checksum
-    args: ["--output", "${signature}", "--detach-sign", "${artifact}"]
-
 builds:
   - binary: syft
+    id: syft
     env:
       - CGO_ENABLED=0
     goos:
       # windows not supported yet (due to jotframe)
       # - windows
       - linux
+    goarch:
+      - amd64
+    # Set the modified timestamp on the output binary to the git timestamp (to ensure a reproducible build)
+    mod_timestamp: '{{ .CommitTimestamp }}'
+    ldflags: |
+      -w
+      -s
+      -extldflags '-static'
+      -X github.com/anchore/syft/internal/version.version={{.Version}}
+      -X github.com/anchore/syft/internal/version.gitCommit={{.Commit}}
+      -X github.com/anchore/syft/internal/version.buildDate={{.Date}}
+      -X github.com/anchore/syft/internal/version.gitTreeState={{.Env.BUILD_GIT_TREE_STATE}}
+  # For more info on this macOS build, see: https://github.com/mitchellh/gon#usage-with-goreleaser
+  - binary: syft
+    id: syft-macos
+    env:
+      - CGO_ENABLED=0
+    goos:
       - darwin
     goarch:
       - amd64
@@ -32,6 +47,34 @@ builds:
       -X github.com/anchore/syft/internal/version.buildDate={{.Date}}
       -X github.com/anchore/syft/internal/version.gitTreeState={{.Env.BUILD_GIT_TREE_STATE}}
 
+archives:
+  - format: tar.gz
+    builds:
+      - syft # i.e. Linux only
+  - format: zip # This is a hack! We don't actually intend to use _this_ ZIP file, we just need goreleaser to consider the ZIP file produced by gon (which will have the same file name) to be an artifact so we can use it downstream in publishing (e.g. to a homebrew tap)
+    id: syft-zip
+    builds:
+      - syft-macos
+
+signs:
+  - artifacts: checksum
+    cmd: sh
+    args:
+      - '-c'
+      # we should not include the zip artifact, as the artifact is mutated throughout the next macOS notarization step
+      # note: sed -i is not portable
+      - 'sed "/.*\.zip/d" ${artifact} > tmpfile && mv tmpfile ${artifact} && gpg --output ${signature} --detach-sign ${artifact}'
+  - id: syft-macos-signing
+    ids:
+      - syft-macos
+    cmd: ./.github/scripts/mac-sign-and-notarize.sh
+    signature: "syft_${VERSION}_darwin_amd64.dmg" # This is somewhat unintuitive. This gets the DMG file recognized as an artifact. In fact, both a DMG and a ZIP file are being produced by this signing step.
+    args:
+      - "{{ .IsSnapshot }}"
+      - "gon.hcl"
+      - "./dist/syft_{{ .Version }}_darwin_amd64"
+    artifacts: all
+
 nfpms:
   - license: "Apache 2.0"
     maintainer: "Anchore, Inc"
@@ -47,10 +90,3 @@ brews:
       name: homebrew-syft
     homepage: *website
     description: *description
-
-archives:
-  - format: tar.gz
-    format_overrides:
-      - goos: windows
-        format: zip
-

Makefile

@@ -74,6 +74,10 @@ help:
 ci-bootstrap:
 	DEBIAN_FRONTEND=noninteractive sudo apt update && sudo -E apt install -y bc jq libxml2-utils
 
+.PHONY:
+ci-bootstrap-mac:
+	github_changelog_generator --version || sudo gem install github_changelog_generator
+
 .PHONY: bootstrap
 bootstrap: ## Download and install all go dependencies (+ prep tooling in the ./tmp dir)
 	$(call title,Bootstrapping dependencies)
@@ -122,7 +126,7 @@ validate-cyclonedx-schema:
 .PHONY: unit
 unit: fixtures ## Run unit tests (with coverage)
 	$(call title,Running unit tests)
-	go test -coverprofile $(COVER_REPORT) ./...
+	go test -coverprofile $(COVER_REPORT) $(shell go list ./... | grep -v anchore/syft/test)
 	@go tool cover -func $(COVER_REPORT) | grep total |  awk '{print substr($$3, 1, length($$3)-1)}' > $(COVER_TOTAL)
 	@echo "Coverage: $$(cat $(COVER_TOTAL))"
 	@if [ $$(echo "$$(cat $(COVER_TOTAL)) >= $(COVERAGE_THRESHOLD)" | bc -l) -ne 1 ]; then echo "$(RED)$(BOLD)Failed coverage quality gate (> $(COVERAGE_THRESHOLD)%)$(RESET)" && false; fi
@@ -130,7 +134,9 @@ unit: fixtures ## Run unit tests (with coverage)
 .PHONY: integration
 integration: ## Run integration tests
 	$(call title,Running integration tests)
-	go test -tags=integration -v ./test/integration
+
+	go test -v ./test/integration
+
 
 # note: this is used by CI to determine if the integration test fixture cache (docker image tars) should be busted
 integration-fingerprint:
@@ -224,8 +230,7 @@ acceptance-test-rpm-package-install: $(SNAPSHOTDIR)
 changelog-release:
 	@echo "Last tag: $(SECOND_TO_LAST_TAG)"
 	@echo "Current tag: $(VERSION)"
-	@docker run -i --rm  \
-		-v "$(shell pwd)":/usr/local/src/your-app ferrarimarco/github-changelog-generator \
+	@github_changelog_generator \
 		--user anchore \
 		--project $(BIN) \
 		-t ${GITHUB_TOKEN} \
@@ -239,8 +244,9 @@ changelog-release:
 
 .PHONY: changelog-unreleased
 changelog-unreleased: ## show the current changelog that will be produced on the next release (note: requires GITHUB_TOKEN set)
-	@docker run -it --rm  \
-		-v "$(shell pwd)":/usr/local/src/your-app ferrarimarco/github-changelog-generator \
+	@docker run -it --rm \
+		-v "$(shell pwd)":/usr/local/src/your-app \
+		ferrarimarco/github-changelog-generator \
 		--user anchore \
 		--project $(BIN) \
 		-t ${GITHUB_TOKEN} \
@@ -256,17 +262,24 @@ changelog-unreleased: ## show the current changelog that will be produced on the
 			/CHANGELOG.md
 
 .PHONY: release
-release: clean-dist changelog-release ## Build and publish final binaries and packages
+release: clean-dist ci-bootstrap-mac changelog-release ## Build and publish final binaries and packages. Intended to be run only on macOS.
 	$(call title,Publishing release artifacts)
+
+	# Prepare for macOS-specific signing process
+	.github/scripts/mac-prepare-for-signing.sh
+
 	# create a config with the dist dir overridden
 	echo "dist: $(DISTDIR)" > $(TEMPDIR)/goreleaser.yaml
 	cat .goreleaser.yaml >> $(TEMPDIR)/goreleaser.yaml
 
-	# release
-	bash -c "BUILD_GIT_TREE_STATE=$(GITTREESTATE) $(TEMPDIR)/goreleaser \
-		--rm-dist \
-		--config $(TEMPDIR)/goreleaser.yaml \
-		--release-notes <(cat CHANGELOG.md)"
+	# release (note the version transformation from v0.7.0 --> 0.7.0)
+	bash -c "\
+		BUILD_GIT_TREE_STATE=$(GITTREESTATE) \
+		VERSION=$(VERSION:v%=%) \
+		$(TEMPDIR)/goreleaser \
+			--rm-dist \
+			--config $(TEMPDIR)/goreleaser.yaml \
+			--release-notes <(cat CHANGELOG.md)"
 
 	# verify checksum signatures
 	.github/scripts/verify-signature.sh "$(DISTDIR)"
@@ -288,4 +301,4 @@ clean-dist:
 
 .PHONY: clean-json-schema-examples
 clean-json-schema-examples:
-	rm -f schema/json/examples/*
+	rm -f schema/json/examples/*

README.md

@@ -54,7 +54,7 @@ Where the `format`s available are:
 
 ## Installation
 
-**Recommended**
+**Recommended (macOS and Linux)**
 ```bash
 # install the latest version to /usr/local/bin
 curl -sSfL https://raw.githubusercontent.com/anchore/syft/main/install.sh | sh -s -- -b /usr/local/bin
@@ -63,17 +63,12 @@ curl -sSfL https://raw.githubusercontent.com/anchore/syft/main/install.sh | sh -
 curl -sSfL https://raw.githubusercontent.com/anchore/syft/main/install.sh | sh -s -- -b <SOME_BIN_PATH> <RELEASE_VERSION>
 ```
 
-**macOS**
+**Homebrew (macOS)**
 ```bash
 brew tap anchore/syft
 brew install syft
 ```
 
-You may experience a "macOS cannot verify app is free from malware" error upon running Syft because it is not yet signed and notarized. You can override this using `xattr`.
-```bash
-xattr -rd com.apple.quarantine syft
-```
-
 ## Configuration
 
 Configuration search paths:
@@ -112,5 +107,4 @@ check-for-app-update: true
 ## Future plans
 
 The following areas of potential development are currently being investigated:
-- Add CycloneDX to list of output formats
 - Establish a stable interchange format w/Grype

go.mod

@@ -4,14 +4,14 @@ go 1.14
 
 require (
 	github.com/adrg/xdg v0.2.1
-	github.com/anchore/go-rpmdb v0.0.0-20200811175839-cbc751c28e8e
+	github.com/anchore/go-rpmdb v0.0.0-20201106153645-0043963c2e12
 	github.com/anchore/go-testutils v0.0.0-20200925183923-d5f45b0d3c04
 	github.com/anchore/go-version v1.2.2-0.20200701162849-18adb9c92b9b
-	github.com/anchore/stereoscope v0.0.0-20200925184903-c82da54e98fe
-	github.com/bmatcuk/doublestar v1.3.1
+	github.com/anchore/stereoscope v0.0.0-20201106140100-12e75c48f409
+	github.com/bmatcuk/doublestar v1.3.3
 	github.com/docker/docker v17.12.0-ce-rc1.0.20200309214505-aa6a9891b09c+incompatible
 	github.com/dustin/go-humanize v1.0.0
-	github.com/go-test/deep v1.0.6
+	github.com/go-test/deep v1.0.7
 	github.com/google/uuid v1.1.1
 	github.com/gookit/color v1.2.7
 	github.com/hashicorp/go-multierror v1.1.0

go.sum

@@ -126,12 +126,16 @@ github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRF
 github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho=
 github.com/anchore/go-rpmdb v0.0.0-20200811175839-cbc751c28e8e h1:kty6r0R2JeaNPeWKSYDC+HW3hkqwFh4PP5TQ8pUPYFw=
 github.com/anchore/go-rpmdb v0.0.0-20200811175839-cbc751c28e8e/go.mod h1:iYuIG0Nai8dR0ri3LhZQKUyO1loxUWAGvoWhXDmjy1A=
+github.com/anchore/go-rpmdb v0.0.0-20201106153645-0043963c2e12 h1:xbeIbn5F52JVx3RUIajxCj8b0y+9lywspql4sFhcxWQ=
+github.com/anchore/go-rpmdb v0.0.0-20201106153645-0043963c2e12/go.mod h1:juoyWXIj7sJ1IDl4E/KIfyLtovbs5XQVSIdaQifFQT8=
 github.com/anchore/go-testutils v0.0.0-20200925183923-d5f45b0d3c04 h1:VzprUTpc0vW0nnNKJfJieyH/TZ9UYAnTZs5/gHTdAe8=
 github.com/anchore/go-testutils v0.0.0-20200925183923-d5f45b0d3c04/go.mod h1:6dK64g27Qi1qGQZ67gFmBFvEHScy0/C8qhQhNe5B5pQ=
 github.com/anchore/go-version v1.2.2-0.20200701162849-18adb9c92b9b h1:e1bmaoJfZVsCYMrIZBpFxwV26CbsuoEh5muXD5I1Ods=
 github.com/anchore/go-version v1.2.2-0.20200701162849-18adb9c92b9b/go.mod h1:Bkc+JYWjMCF8OyZ340IMSIi2Ebf3uwByOk6ho4wne1E=
 github.com/anchore/stereoscope v0.0.0-20200925184903-c82da54e98fe h1:m4NSyTo2fVUoUHAV/ZVqE/PFMr/y8oz9HRrhWLk9It0=
 github.com/anchore/stereoscope v0.0.0-20200925184903-c82da54e98fe/go.mod h1:2Jja/4l0zYggW52og+nn0rut4i+OYjCf9vTyrM8RT4E=
+github.com/anchore/stereoscope v0.0.0-20201106140100-12e75c48f409 h1:xKSpDRjmYrEFrdMeDh4AuSUAFc99pdro6YFBKxy2um0=
+github.com/anchore/stereoscope v0.0.0-20201106140100-12e75c48f409/go.mod h1:2Jja/4l0zYggW52og+nn0rut4i+OYjCf9vTyrM8RT4E=
 github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883/go.mod h1:rCTlJbsFo29Kk6CurOXKm700vrz8f0KW0JNfpkRJY/8=
 github.com/apex/log v1.1.4/go.mod h1:AlpoD9aScyQfJDVHmLMEcx4oU6LqzkWp4Mg9GdAcEvQ=
 github.com/apex/log v1.3.0 h1:1fyfbPvUwD10nMoh3hY6MXzvZShJQn9/ck7ATgAt5pA=
@@ -162,6 +166,8 @@ github.com/blakesmith/ar v0.0.0-20190502131153-809d4375e1fb/go.mod h1:PkYb9DJNAw
 github.com/blang/semver v3.5.0+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk=
 github.com/bmatcuk/doublestar v1.3.1 h1:rT8rxDPsavp9G+4ZULzqhhUSaI/OPsTZNG88Z3i0xvY=
 github.com/bmatcuk/doublestar v1.3.1/go.mod h1:wiQtGV+rzVYxB7WIlirSN++5HPtPlXEo9MEoZQC/PmE=
+github.com/bmatcuk/doublestar v1.3.3 h1:pVP1d49CcQQaNOl+PI6sPybIrIOD/6sux31PFdmhTH0=
+github.com/bmatcuk/doublestar v1.3.3/go.mod h1:wiQtGV+rzVYxB7WIlirSN++5HPtPlXEo9MEoZQC/PmE=
 github.com/bombsimon/wsl/v2 v2.0.0/go.mod h1:mf25kr/SqFEPhhcxW1+7pxzGlW+hIl/hYTKY95VwV8U=
 github.com/bombsimon/wsl/v2 v2.2.0/go.mod h1:Azh8c3XGEJl9LyX0/sFC+CKMc7Ssgua0g+6abzXN4Pg=
 github.com/bombsimon/wsl/v3 v3.0.0/go.mod h1:st10JtZYLE4D5sC7b8xV4zTKZwAQjCH/Hy2Pm1FNZIc=
@@ -297,6 +303,8 @@ github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LB
 github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
 github.com/go-test/deep v1.0.6 h1:UHSEyLZUwX9Qoi99vVwvewiMC8mM2bf7XEM2nqvzEn8=
 github.com/go-test/deep v1.0.6/go.mod h1:QV8Hv/iy04NyLBxAdO9njL0iVPN1S4d/A3NVv1V36o8=
+github.com/go-test/deep v1.0.7 h1:/VSMRlnY/JSyqxQUzQLKVMAskpY/NZKFA5j2P+0pP2M=
+github.com/go-test/deep v1.0.7/go.mod h1:QV8Hv/iy04NyLBxAdO9njL0iVPN1S4d/A3NVv1V36o8=
 github.com/go-toolsmith/astcast v1.0.0 h1:JojxlmI6STnFVG9yOImLeGREv8W2ocNUM+iOhR6jE7g=
 github.com/go-toolsmith/astcast v1.0.0/go.mod h1:mt2OdQTeAQcY4DQgPSArJjHCcOwlX+Wl/kwN+LbLGQ4=
 github.com/go-toolsmith/astcopy v1.0.0 h1:OMgl1b1MEpjFQ1m5ztEO06rz5CUd3oBv9RF7+DyvdG8=

gon.hcl

@@ -0,0 +1,15 @@
+source = ["./dist/syft-macos_darwin_amd64/syft"] # The 'dist' directory path should ideally reference an env var, where the source of truth is the Makefile. I wasn't able to figure out how to solve this.
+bundle_id = "com.anchore.toolbox.syft"
+
+sign {
+  application_identity = "Developer ID Application: ANCHORE, INC. (9MJHKYX5AT)"
+}
+
+dmg {
+  output_path = "./dist/output.dmg"
+  volume_name = "Syft"
+}
+
+zip {
+  output_path = "./dist/output.zip"
+}

install.sh

@@ -1,6 +1,6 @@
 #!/bin/sh
 set -e
-# Code generated by godownloader on 2020-08-10T20:55:46Z. DO NOT EDIT.
+# Code generated by godownloader on 2020-08-10T20:55:46Z.
 #
 
 usage() {
@@ -45,11 +45,16 @@ parse_args() {
 execute() {
   tmpdir=$(mktemp -d)
   log_debug "downloading files into ${tmpdir}"
-  http_download "${tmpdir}/${TARBALL}" "${TARBALL_URL}"
+  http_download "${tmpdir}/${ARCHIVE}" "${ARCHIVE_URL}"
   http_download "${tmpdir}/${CHECKSUM}" "${CHECKSUM_URL}"
-  hash_sha256_verify "${tmpdir}/${TARBALL}" "${tmpdir}/${CHECKSUM}"
+
+  # macOS has its own secure verification mechanism, and checksums.txt is not used.
+  if [ "$OS" != "darwin" ]; then
+    hash_sha256_verify "${tmpdir}/${ARCHIVE}" "${tmpdir}/${CHECKSUM}"
+  fi
+
   srcdir="${tmpdir}"
-  (cd "${tmpdir}" && untar "${TARBALL}")
+  (cd "${tmpdir}" && unpack "${ARCHIVE}")
   test ! -d "${BINDIR}" && install -d "${BINDIR}"
   for binexe in $BINARIES; do
     if [ "$OS" = "windows" ]; then
@@ -89,6 +94,7 @@ tag_to_version() {
 adjust_format() {
   # change format (tar.gz or zip) based on OS
   case ${OS} in
+    darwin) FORMAT=dmg ;;
     windows) FORMAT=zip ;;
   esac
   true
@@ -221,18 +227,26 @@ uname_arch_check() {
   log_crit "uname_arch_check '$(uname -m)' got converted to '$arch' which is not a GOARCH value.  Please file bug report at https://github.com/client9/shlib"
   return 1
 }
-untar() {
-  tarball=$1
-  case "${tarball}" in
-    *.tar.gz | *.tgz) tar --no-same-owner -xzf "${tarball}" ;;
-    *.tar) tar --no-same-owner -xf "${tarball}" ;;
-    *.zip) unzip "${tarball}" ;;
+unpack() {
+  archive=$1
+  case "${archive}" in
+    *.tar.gz | *.tgz) tar --no-same-owner -xzf "${archive}" ;;
+    *.tar) tar --no-same-owner -xf "${archive}" ;;
+    *.zip) unzip "${archive}" ;;
+    *.dmg) extract_from_dmg "${archive}" ;;
     *)
-      log_err "untar unknown archive format for ${tarball}"
+      log_err "unpack unknown archive format for ${archive}"
       return 1
       ;;
   esac
 }
+extract_from_dmg() {
+  dmg_file=$1
+  mount_point="/Volumes/tmp-dmg"
+  hdiutil attach -quiet -nobrowse -mountpoint "${mount_point}" "${dmg_file}"
+  cp -fR "${mount_point}/" ./
+  hdiutil detach -quiet -force "${mount_point}"
+}
 http_download_curl() {
   local_file=$1
   source_url=$2
@@ -366,8 +380,8 @@ adjust_arch
 log_info "found version: ${VERSION} for ${TAG}/${OS}/${ARCH}"
 
 NAME=${PROJECT_NAME}_${VERSION}_${OS}_${ARCH}
-TARBALL=${NAME}.${FORMAT}
-TARBALL_URL=${GITHUB_DOWNLOAD}/${TAG}/${TARBALL}
+ARCHIVE=${NAME}.${FORMAT}
+ARCHIVE_URL=${GITHUB_DOWNLOAD}/${TAG}/${ARCHIVE}
 CHECKSUM=${PROJECT_NAME}_${VERSION}_checksums.txt
 CHECKSUM_URL=${GITHUB_DOWNLOAD}/${TAG}/${CHECKSUM}
 

internal/config/config.go

@@ -111,7 +111,7 @@ func (cfg *Application) Build() error {
 			case v >= 2:
 				cfg.Log.LevelOpt = logrus.DebugLevel
 			default:
-				cfg.Log.LevelOpt = logrus.ErrorLevel
+				cfg.Log.LevelOpt = logrus.WarnLevel
 			}
 		}
 	}

internal/file/opener.go

@@ -0,0 +1,14 @@
+package file
+
+import (
+	"io"
+	"os"
+)
+
+type Opener struct {
+	path string
+}
+
+func (o Opener) Open() (io.ReadCloser, error) {
+	return os.Open(o.path)
+}

internal/file/test-fixtures/generate-zip-fixture.sh

@@ -1,4 +1,7 @@
 #!/usr/bin/env bash
 set -eux
 
-zip -r "$1" zip-source
+# $1 —— absolute path to destination file, should end with .zip, ideally
+# $2 —— absolute path to directory from which to add entries to the archive
+
+pushd "$2" && find . -print | zip "$1" -@ && popd

internal/file/zip_file_helpers_test.go

@@ -0,0 +1,131 @@
+package file
+
+import (
+	"fmt"
+	"io/ioutil"
+	"os"
+	"os/exec"
+	"path"
+	"path/filepath"
+	"syscall"
+	"testing"
+)
+
+var expectedZipArchiveEntries = []string{
+	"some-dir" + string(os.PathSeparator),
+	filepath.Join("some-dir", "a-file.txt"),
+	"b-file.txt",
+	"nested.zip",
+}
+
+// fatalIfError calls the supplied function. If the function returns a non-nil error, t.Fatal(err) is called.
+func fatalIfError(t *testing.T, fn func() error) {
+	t.Helper()
+
+	if fn == nil {
+		return
+	}
+
+	err := fn()
+	if err != nil {
+		t.Fatal(err)
+	}
+}
+
+// createZipArchive creates a new ZIP archive file at destinationArchivePath based on the directory found at
+// sourceDirPath.
+func createZipArchive(t *testing.T, sourceDirPath, destinationArchivePath string) error {
+	t.Helper()
+
+	cwd, err := os.Getwd()
+	if err != nil {
+		return fmt.Errorf("unable to get cwd: %+v", err)
+	}
+
+	cmd := exec.Command("./generate-zip-fixture.sh", destinationArchivePath, path.Base(sourceDirPath))
+	cmd.Dir = filepath.Join(cwd, "test-fixtures")
+
+	if err := cmd.Start(); err != nil {
+		return fmt.Errorf("unable to start generate zip fixture script: %+v", err)
+	}
+
+	if err := cmd.Wait(); err != nil {
+		if exiterr, ok := err.(*exec.ExitError); ok {
+			// The program has exited with an exit code != 0
+
+			// This works on both Unix and Windows. Although package
+			// syscall is generally platform dependent, WaitStatus is
+			// defined for both Unix and Windows and in both cases has
+			// an ExitStatus() method with the same signature.
+			if status, ok := exiterr.Sys().(syscall.WaitStatus); ok {
+				if status.ExitStatus() != 0 {
+					return fmt.Errorf("failed to generate fixture: rc=%d", status.ExitStatus())
+				}
+			}
+		} else {
+			return fmt.Errorf("unable to get generate fixture script result: %+v", err)
+		}
+	}
+
+	return nil
+}
+
+// setupZipFileTest encapsulates common test setup work for zip file tests. It returns a cleanup function,
+// which should be called (typically deferred) by the caller, the path of the created zip archive, and an error,
+// which should trigger a fatal test failure in the consuming test. The returned cleanup function will never be nil
+// (even if there's an error), and it should always be called.
+func setupZipFileTest(t *testing.T, sourceDirPath string) (func() error, string, error) {
+	t.Helper()
+
+	// Keep track of any needed cleanup work as we go
+	var cleanupFns []func() error
+	cleanup := func(fns []func() error) func() error {
+		return func() error {
+			for _, fn := range fns {
+				err := fn()
+				if err != nil {
+					return err
+				}
+			}
+
+			return nil
+		}
+	}
+
+	archivePrefix, err := ioutil.TempFile("", "syft-ziputil-archive-TEST-")
+	if err != nil {
+		return cleanup(cleanupFns), "", fmt.Errorf("unable to create tempfile: %+v", err)
+	}
+	cleanupFns = append(cleanupFns, func() error { return os.Remove(archivePrefix.Name()) })
+
+	destinationArchiveFilePath := archivePrefix.Name() + ".zip"
+	t.Logf("archive path: %s", destinationArchiveFilePath)
+	err = createZipArchive(t, sourceDirPath, destinationArchiveFilePath)
+	cleanupFns = append(cleanupFns, func() error { return os.Remove(destinationArchiveFilePath) })
+	if err != nil {
+		return cleanup(cleanupFns), "", err
+	}
+
+	cwd, err := os.Getwd()
+	if err != nil {
+		return cleanup(cleanupFns), "", fmt.Errorf("unable to get cwd: %+v", err)
+	}
+
+	t.Logf("running from: %s", cwd)
+
+	return cleanup(cleanupFns), destinationArchiveFilePath, nil
+}
+
+// TODO: Consider moving any non-git asset generation to a task (e.g. make) that's run ahead of running go tests.
+func ensureNestedZipExists(t *testing.T, sourceDirPath string) error {
+	t.Helper()
+
+	nestedArchiveFilePath := path.Join(sourceDirPath, "nested.zip")
+	err := createZipArchive(t, sourceDirPath, nestedArchiveFilePath)
+
+	if err != nil {
+		return fmt.Errorf("unable to create nested archive for test fixture: %+v", err)
+	}
+
+	return nil
+}

internal/file/zip_file_manifest.go

@@ -5,6 +5,7 @@ import (
 	"fmt"
 	"os"
 	"sort"
+	"strings"
 
 	"github.com/anchore/syft/internal"
 
@@ -26,7 +27,11 @@ func (z ZipFileManifest) GlobMatch(patterns ...string) []string {
 
 	for _, pattern := range patterns {
 		for entry := range z {
-			if GlobMatch(pattern, entry) {
+			// We want to match globs as if entries begin with a leading slash (akin to an absolute path)
+			// so that glob logic is consistent inside and outside of ZIP archives
+			normalizedEntry := normalizeZipEntryName(entry)
+
+			if GlobMatch(pattern, normalizedEntry) {
 				uniqueMatches.Add(entry)
 			}
 		}
@@ -56,3 +61,11 @@ func NewZipFileManifest(archivePath string) (ZipFileManifest, error) {
 	}
 	return manifest, nil
 }
+
+func normalizeZipEntryName(entry string) string {
+	if !strings.HasPrefix(entry, "/") {
+		return "/" + entry
+	}
+
+	return entry
+}

internal/file/zip_file_manifest_test.go

@@ -0,0 +1,109 @@
+package file
+
+import (
+	"encoding/json"
+	"os"
+	"path"
+	"testing"
+)
+
+func TestNewZipFileManifest(t *testing.T) {
+	cwd, err := os.Getwd()
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	sourceDirPath := path.Join(cwd, "test-fixtures", "zip-source")
+	err = ensureNestedZipExists(t, sourceDirPath)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	cleanup, archiveFilePath, err := setupZipFileTest(t, sourceDirPath)
+	defer fatalIfError(t, cleanup)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	actual, err := NewZipFileManifest(archiveFilePath)
+	if err != nil {
+		t.Fatalf("unable to extract from unzip archive: %+v", err)
+	}
+
+	if len(expectedZipArchiveEntries) != len(actual) {
+		t.Fatalf("mismatched manifest: %d != %d", len(actual), len(expectedZipArchiveEntries))
+	}
+
+	for _, e := range expectedZipArchiveEntries {
+		_, ok := actual[e]
+		if !ok {
+			t.Errorf("missing path: %s", e)
+		}
+	}
+
+	if t.Failed() {
+		b, err := json.MarshalIndent(actual, "", "  ")
+		if err != nil {
+			t.Fatalf("can't show results: %+v", err)
+		}
+
+		t.Errorf("full result: %s", string(b))
+	}
+}
+
+func TestZipFileManifest_GlobMatch(t *testing.T) {
+	cwd, err := os.Getwd()
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	sourceDirPath := path.Join(cwd, "test-fixtures", "zip-source")
+	err = ensureNestedZipExists(t, sourceDirPath)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	cleanup, archiveFilePath, err := setupZipFileTest(t, sourceDirPath)
+	//goland:noinspection GoNilness
+	defer fatalIfError(t, cleanup)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	z, err := NewZipFileManifest(archiveFilePath)
+	if err != nil {
+		t.Fatalf("unable to extract from unzip archive: %+v", err)
+	}
+
+	cases := []struct {
+		glob     string
+		expected string
+	}{
+		{
+			"/b*",
+			"b-file.txt",
+		},
+		{
+			"*/a-file.txt",
+			"some-dir/a-file.txt",
+		},
+		{
+			"**/*.zip",
+			"nested.zip",
+		},
+	}
+
+	for _, tc := range cases {
+		t.Run(tc.glob, func(t *testing.T) {
+			glob := tc.glob
+
+			results := z.GlobMatch(glob)
+
+			if len(results) == 1 && results[0] == tc.expected {
+				return
+			}
+
+			t.Errorf("unexpected results for glob '%s': %+v", glob, results)
+		})
+	}
+}

internal/file/zip_file_traversal.go

@@ -63,8 +63,8 @@ func TraverseFilesInZip(archivePath string, visitor func(*zip.File) error, paths
 	return nil
 }
 
-func ExtractFromZipToUniqueTempFile(archivePath, dir string, paths ...string) (map[string]io.Reader, error) {
-	results := make(map[string]io.Reader)
+func ExtractFromZipToUniqueTempFile(archivePath, dir string, paths ...string) (map[string]Opener, error) {
+	results := make(map[string]Opener)
 
 	// don't allow for full traversal, only select traversal from given paths
 	if len(paths) == 0 {
@@ -78,11 +78,21 @@ func ExtractFromZipToUniqueTempFile(archivePath, dir string, paths ...string) (m
 		if err != nil {
 			return fmt.Errorf("unable to create temp file: %w", err)
 		}
+		// we shouldn't try and keep the tempfile open as the returned result may have several files, which takes up
+		// resources (leading to "too many open files"). Instead we'll return a file opener to the caller which
+		// provides a ReadCloser. It is up to the caller to handle closing the file explicitly.
+		defer tempFile.Close()
 
 		zippedFile, err := file.Open()
 		if err != nil {
 			return fmt.Errorf("unable to read file=%q from zip=%q: %w", file.Name, archivePath, err)
 		}
+		defer func() {
+			err := zippedFile.Close()
+			if err != nil {
+				log.Errorf("unable to close source file=%q from zip=%q: %+v", file.Name, archivePath, err)
+			}
+		}()
 
 		if file.FileInfo().IsDir() {
 			return fmt.Errorf("unable to extract directories, only files: %s", file.Name)
@@ -103,12 +113,8 @@ func ExtractFromZipToUniqueTempFile(archivePath, dir string, paths ...string) (m
 			return fmt.Errorf("unable to reset file pointer (%s): %w", tempFile.Name(), err)
 		}
 
-		results[file.Name] = tempFile
+		results[file.Name] = Opener{path: tempFile.Name()}
 
-		err = zippedFile.Close()
-		if err != nil {
-			return fmt.Errorf("unable to close source file=%q from zip=%q: %w", file.Name, archivePath, err)
-		}
 		return nil
 	}
 

internal/file/zip_file_traversal_test.go

@@ -6,47 +6,14 @@ import (
 	"io"
 	"io/ioutil"
 	"os"
-	"os/exec"
+	"path"
 	"path/filepath"
 	"strings"
-	"syscall"
 	"testing"
 
 	"github.com/go-test/deep"
 )
 
-func generateFixture(t *testing.T, archivePath string) {
-	cwd, err := os.Getwd()
-	if err != nil {
-		t.Errorf("unable to get cwd: %+v", err)
-	}
-
-	cmd := exec.Command("./generate-zip-fixture.sh", archivePath)
-	cmd.Dir = filepath.Join(cwd, "test-fixtures")
-
-	if err := cmd.Start(); err != nil {
-		t.Fatalf("unable to start generate zip fixture script: %+v", err)
-	}
-
-	if err := cmd.Wait(); err != nil {
-		if exiterr, ok := err.(*exec.ExitError); ok {
-			// The program has exited with an exit code != 0
-
-			// This works on both Unix and Windows. Although package
-			// syscall is generally platform dependent, WaitStatus is
-			// defined for both Unix and Windows and in both cases has
-			// an ExitStatus() method with the same signature.
-			if status, ok := exiterr.Sys().(syscall.WaitStatus); ok {
-				if status.ExitStatus() != 0 {
-					t.Fatalf("failed to generate fixture: rc=%d", status.ExitStatus())
-				}
-			}
-		} else {
-			t.Fatalf("unable to get generate fixture script result: %+v", err)
-		}
-	}
-}
-
 func equal(r1, r2 io.Reader) (bool, error) {
 	w1 := sha256.New()
 	w2 := sha256.New()
@@ -67,54 +34,50 @@ func equal(r1, r2 io.Reader) (bool, error) {
 }
 
 func TestUnzipToDir(t *testing.T) {
-	archivePrefix, err := ioutil.TempFile("", "syft-ziputil-archive-TEST-")
+	cwd, err := os.Getwd()
 	if err != nil {
-		t.Fatalf("unable to create tempfile: %+v", err)
+		t.Fatal(err)
 	}
-	defer os.Remove(archivePrefix.Name())
-	// the zip utility will add ".zip" to the end of the given name
-	archivePath := archivePrefix.Name() + ".zip"
-	defer os.Remove(archivePath)
-	t.Logf("archive path: %s", archivePath)
 
-	generateFixture(t, archivePrefix.Name())
+	goldenRootDir := filepath.Join(cwd, "test-fixtures")
+	sourceDirPath := path.Join(goldenRootDir, "zip-source")
+	cleanup, archiveFilePath, err := setupZipFileTest(t, sourceDirPath)
+	defer fatalIfError(t, cleanup)
+	if err != nil {
+		t.Fatal(err)
+	}
 
-	contentsDir, err := ioutil.TempDir("", "syft-ziputil-contents-TEST-")
+	unzipDestinationDir, err := ioutil.TempDir("", "syft-ziputil-contents-TEST-")
+	defer os.RemoveAll(unzipDestinationDir)
 	if err != nil {
 		t.Fatalf("unable to create tempdir: %+v", err)
 	}
-	defer os.RemoveAll(contentsDir)
 
-	t.Logf("content path: %s", contentsDir)
+	t.Logf("content path: %s", unzipDestinationDir)
 
-	cwd, err := os.Getwd()
-	if err != nil {
-		t.Errorf("unable to get cwd: %+v", err)
-	}
-
-	t.Logf("running from: %s", cwd)
-
-	// note: zip utility already includes "zip-source" as a parent dir for all contained files
-	goldenRootDir := filepath.Join(cwd, "test-fixtures")
-	expectedPaths := 4
+	expectedPaths := len(expectedZipArchiveEntries)
 	observedPaths := 0
 
-	err = UnzipToDir(archivePath, contentsDir)
+	err = UnzipToDir(archiveFilePath, unzipDestinationDir)
 	if err != nil {
 		t.Fatalf("unable to unzip archive: %+v", err)
 	}
 
 	// compare the source dir tree and the unzipped tree
-	err = filepath.Walk(filepath.Join(contentsDir, "zip-source"),
+	err = filepath.Walk(unzipDestinationDir,
 		func(path string, info os.FileInfo, err error) error {
-			t.Logf("unzipped path: %s", path)
-			observedPaths++
+			// We don't unzip the root archive dir, since there's no archive entry for it
+			if path != unzipDestinationDir {
+				t.Logf("unzipped path: %s", path)
+				observedPaths++
+			}
+
 			if err != nil {
 				t.Fatalf("this should not happen")
 				return err
 			}
 
-			goldenPath := filepath.Join(goldenRootDir, strings.TrimPrefix(path, contentsDir))
+			goldenPath := filepath.Join(sourceDirPath, strings.TrimPrefix(path, unzipDestinationDir))
 
 			if info.IsDir() {
 				i, err := os.Stat(goldenPath)
@@ -156,12 +119,11 @@ func TestUnzipToDir(t *testing.T) {
 	}
 
 	if observedPaths != expectedPaths {
-		t.Errorf("missed test paths: %d!=%d", observedPaths, expectedPaths)
+		t.Errorf("missed test paths: %d != %d", observedPaths, expectedPaths)
 	}
-
 }
 
-func TestExtractFilesFromZipFile(t *testing.T) {
+func TestContentsFromZip(t *testing.T) {
 	archivePrefix, err := ioutil.TempFile("", "syft-ziputil-archive-TEST-")
 	if err != nil {
 		t.Fatalf("unable to create tempfile: %+v", err)
@@ -172,7 +134,10 @@ func TestExtractFilesFromZipFile(t *testing.T) {
 	defer os.Remove(archivePath)
 	t.Logf("archive path: %s", archivePath)
 
-	generateFixture(t, archivePrefix.Name())
+	err = createZipArchive(t, "zip-source", archivePrefix.Name())
+	if err != nil {
+		t.Fatal(err)
+	}
 
 	cwd, err := os.Getwd()
 	if err != nil {
@@ -181,8 +146,8 @@ func TestExtractFilesFromZipFile(t *testing.T) {
 
 	t.Logf("running from: %s", cwd)
 
-	aFilePath := filepath.Join("zip-source", "some-dir", "a-file.txt")
-	bFilePath := filepath.Join("zip-source", "b-file.txt")
+	aFilePath := filepath.Join("some-dir", "a-file.txt")
+	bFilePath := filepath.Join("b-file.txt")
 
 	expected := map[string]string{
 		aFilePath: "A file! nice!",
@@ -207,60 +172,4 @@ func TestExtractFilesFromZipFile(t *testing.T) {
 
 		t.Errorf("full result: %s", string(b))
 	}
-
-}
-
-func TestZipFileManifest(t *testing.T) {
-	archivePrefix, err := ioutil.TempFile("", "syft-ziputil-archive-TEST-")
-	if err != nil {
-		t.Fatalf("unable to create tempfile: %+v", err)
-	}
-	defer os.Remove(archivePrefix.Name())
-	// the zip utility will add ".zip" to the end of the given name
-	archivePath := archivePrefix.Name() + ".zip"
-	defer os.Remove(archivePath)
-	t.Logf("archive path: %s", archivePath)
-
-	generateFixture(t, archivePrefix.Name())
-
-	cwd, err := os.Getwd()
-	if err != nil {
-		t.Errorf("unable to get cwd: %+v", err)
-	}
-
-	t.Logf("running from: %s", cwd)
-
-	expected := []string{
-		filepath.Join("zip-source") + string(os.PathSeparator),
-		filepath.Join("zip-source", "some-dir") + string(os.PathSeparator),
-		filepath.Join("zip-source", "some-dir", "a-file.txt"),
-		filepath.Join("zip-source", "b-file.txt"),
-	}
-
-	actual, err := NewZipFileManifest(archivePath)
-	if err != nil {
-		t.Fatalf("unable to extract from unzip archive: %+v", err)
-	}
-
-	if len(expected) != len(actual) {
-		t.Fatalf("mismatched manifest: %d != %d", len(actual), len(expected))
-	}
-
-	for _, e := range expected {
-		_, ok := actual[e]
-		if !ok {
-			t.Errorf("missing path: %s", e)
-		}
-	}
-
-	if t.Failed() {
-
-		b, err := json.MarshalIndent(actual, "", "  ")
-		if err != nil {
-			t.Fatalf("can't show results: %+v", err)
-		}
-
-		t.Errorf("full result: %s", string(b))
-	}
-
 }

internal/log/log.go

@@ -8,6 +8,18 @@ func Errorf(format string, args ...interface{}) {
 	Log.Errorf(format, args...)
 }
 
+func Error(args ...interface{}) {
+	Log.Error(args...)
+}
+
+func Warnf(format string, args ...interface{}) {
+	Log.Warnf(format, args...)
+}
+
+func Warn(args ...interface{}) {
+	Log.Warn(args...)
+}
+
 func Infof(format string, args ...interface{}) {
 	Log.Infof(format, args...)
 }

internal/log/nop.go

@@ -3,6 +3,9 @@ package log
 type nopLogger struct{}
 
 func (l *nopLogger) Errorf(format string, args ...interface{}) {}
+func (l *nopLogger) Error(args ...interface{})                 {}
+func (l *nopLogger) Warnf(format string, args ...interface{})  {}
+func (l *nopLogger) Warn(args ...interface{})                  {}
 func (l *nopLogger) Infof(format string, args ...interface{})  {}
 func (l *nopLogger) Info(args ...interface{})                  {}
 func (l *nopLogger) Debugf(format string, args ...interface{}) {}

internal/logger/logrus.go

@@ -84,6 +84,14 @@ func (l *LogrusLogger) Infof(format string, args ...interface{}) {
 	l.Logger.Infof(format, args...)
 }
 
+func (l *LogrusLogger) Warnf(format string, args ...interface{}) {
+	l.Logger.Warnf(format, args...)
+}
+
+func (l *LogrusLogger) Errorf(format string, args ...interface{}) {
+	l.Logger.Errorf(format, args...)
+}
+
 func (l *LogrusLogger) Debug(args ...interface{}) {
 	l.Logger.Debug(args...)
 }
@@ -92,8 +100,12 @@ func (l *LogrusLogger) Info(args ...interface{}) {
 	l.Logger.Info(args...)
 }
 
-func (l *LogrusLogger) Errorf(format string, args ...interface{}) {
-	l.Logger.Errorf(format, args...)
+func (l *LogrusLogger) Warn(args ...interface{}) {
+	l.Logger.Warn(args...)
+}
+
+func (l *LogrusLogger) Error(args ...interface{}) {
+	l.Logger.Error(args...)
 }
 
 func (l *LogrusNestedLogger) Debugf(format string, args ...interface{}) {
@@ -104,6 +116,14 @@ func (l *LogrusNestedLogger) Infof(format string, args ...interface{}) {
 	l.Logger.Infof(format, args...)
 }
 
+func (l *LogrusNestedLogger) Warnf(format string, args ...interface{}) {
+	l.Logger.Warnf(format, args...)
+}
+
+func (l *LogrusNestedLogger) Errorf(format string, args ...interface{}) {
+	l.Logger.Errorf(format, args...)
+}
+
 func (l *LogrusNestedLogger) Debug(args ...interface{}) {
 	l.Logger.Debug(args...)
 }
@@ -112,6 +132,10 @@ func (l *LogrusNestedLogger) Info(args ...interface{}) {
 	l.Logger.Info(args...)
 }
 
-func (l *LogrusNestedLogger) Errorf(format string, args ...interface{}) {
-	l.Logger.Errorf(format, args...)
+func (l *LogrusNestedLogger) Warn(args ...interface{}) {
+	l.Logger.Warn(args...)
+}
+
+func (l *LogrusNestedLogger) Error(args ...interface{}) {
+	l.Logger.Error(args...)
 }

internal/parse.go

@@ -0,0 +1,15 @@
+package internal
+
+import "regexp"
+
+// MatchCaptureGroups takes a regular expression and string and returns all of the named capture group results in a map.
+func MatchCaptureGroups(regEx *regexp.Regexp, str string) map[string]string {
+	match := regEx.FindStringSubmatch(str)
+	results := make(map[string]string)
+	for i, name := range regEx.SubexpNames() {
+		if i > 0 && i <= len(match) {
+			results[name] = match[i]
+		}
+	}
+	return results
+}

schema/README.md

@@ -0,0 +1,9 @@
+## Updating the JSON schema
+Today the JSON schema is generated from integration test data. Specifically, when integration tests are run, the `/schema/json/examples` directory is populated with syft JSON output data. This examples directory is used to drive automatically generating the JSON schema.
+The caveats with this approach is:
+1) the JSON schema is only as good as the examples provided
+2) there is an integration test that ensures that the JSON schema is valid relative to what the code currently generates.
+This means to update the JSON schema you need to
+1) Open up `test/integration/json_schema_test.go` and comment out invocations of the `validateAgainstV1Schema` function.
+2) From the root of the repo run `generate-json-schema`. Now there should be a new schema generated at `/schema/json/schema.json`
+3) Uncomment the `validateAgainstV1Schema` function.

schema/json/schema.json

@@ -10,6 +10,19 @@
                         },
                         "type": "array"
                     },
+                    "licenses": {
+                        "anyOf": [
+                            {
+                                "type": "null"
+                            },
+                            {
+                                "items": {
+                                    "type": "string"
+                                },
+                                "type": "array"
+                            }
+                        ]
+                    },
                     "locations": {
                         "items": {
                             "anyOf": [
@@ -40,6 +53,12 @@
                             "architecture": {
                                 "type": "string"
                             },
+                            "author": {
+                                "type": "string"
+                            },
+                            "authorEmail": {
+                                "type": "string"
+                            },
                             "description": {
                                 "type": "string"
                             },
@@ -47,41 +66,62 @@
                                 "type": "integer"
                             },
                             "files": {
-                                "items": {
-                                    "anyOf": [
-                                        {
-                                            "type": "string"
-                                        },
-                                        {
-                                            "properties": {
-                                                "checksum": {
+                                "anyOf": [
+                                    {
+                                        "type": "null"
+                                    },
+                                    {
+                                        "items": {
+                                            "anyOf": [
+                                                {
                                                     "type": "string"
                                                 },
-                                                "ownerGid": {
-                                                    "type": "string"
-                                                },
-                                                "ownerUid": {
-                                                    "type": "string"
-                                                },
-                                                "path": {
-                                                    "type": "string"
-                                                },
-                                                "permissions": {
-                                                    "type": "string"
+                                                {
+                                                    "properties": {
+                                                        "checksum": {
+                                                            "type": "string"
+                                                        },
+                                                        "digest": {
+                                                            "properties": {
+                                                                "algorithm": {
+                                                                    "type": "string"
+                                                                },
+                                                                "value": {
+                                                                    "type": "string"
+                                                                }
+                                                            },
+                                                            "required": [
+                                                                "algorithm",
+                                                                "value"
+                                                            ],
+                                                            "type": "object"
+                                                        },
+                                                        "ownerGid": {
+                                                            "type": "string"
+                                                        },
+                                                        "ownerUid": {
+                                                            "type": "string"
+                                                        },
+                                                        "path": {
+                                                            "type": "string"
+                                                        },
+                                                        "permissions": {
+                                                            "type": "string"
+                                                        },
+                                                        "size": {
+                                                            "type": "string"
+                                                        }
+                                                    },
+                                                    "required": [
+                                                        "path"
+                                                    ],
+                                                    "type": "object"
                                                 }
-                                            },
-                                            "required": [
-                                                "checksum",
-                                                "ownerGid",
-                                                "ownerUid",
-                                                "path",
-                                                "permissions"
-                                            ],
-                                            "type": "object"
-                                        }
-                                    ]
-                                },
-                                "type": "array"
+                                            ]
+                                        },
+                                        "type": "array"
+                                    }
+                                ]
                             },
                             "gitCommitOfApkPort": {
                                 "type": "string"
@@ -105,116 +145,90 @@
                                 "type": "string"
                             },
                             "manifest": {
-                                "anyOf": [
-                                    {
-                                        "type": "null"
-                                    },
-                                    {
+                                "properties": {
+                                    "main": {
                                         "properties": {
-                                            "extraFields": {
-                                                "properties": {
-                                                    "Archiver-Version": {
-                                                        "type": "string"
-                                                    },
-                                                    "Build-Jdk": {
-                                                        "type": "string"
-                                                    },
-                                                    "Built-By": {
-                                                        "type": "string"
-                                                    },
-                                                    "Created-By": {
-                                                        "type": "string"
-                                                    },
-                                                    "Extension-Name": {
-                                                        "type": "string"
-                                                    },
-                                                    "Group-Id": {
-                                                        "type": "string"
-                                                    },
-                                                    "Hudson-Version": {
-                                                        "type": "string"
-                                                    },
-                                                    "Jenkins-Version": {
-                                                        "type": "string"
-                                                    },
-                                                    "Long-Name": {
-                                                        "type": "string"
-                                                    },
-                                                    "Main-Class": {
-                                                        "type": "string"
-                                                    },
-                                                    "Minimum-Java-Version": {
-                                                        "type": "string"
-                                                    },
-                                                    "Plugin-Dependencies": {
-                                                        "type": "string"
-                                                    },
-                                                    "Plugin-Developers": {
-                                                        "type": "string"
-                                                    },
-                                                    "Plugin-License-Name": {
-                                                        "type": "string"
-                                                    },
-                                                    "Plugin-License-Url": {
-                                                        "type": "string"
-                                                    },
-                                                    "Plugin-ScmUrl": {
-                                                        "type": "string"
-                                                    },
-                                                    "Plugin-Version": {
-                                                        "type": "string"
-                                                    },
-                                                    "Short-Name": {
-                                                        "type": "string"
-                                                    }
-                                                },
-                                                "required": [
-                                                    "Archiver-Version",
-                                                    "Build-Jdk",
-                                                    "Built-By",
-                                                    "Created-By"
-                                                ],
-                                                "type": "object"
-                                            },
-                                            "implementationTitle": {
+                                            "Archiver-Version": {
                                                 "type": "string"
                                             },
-                                            "implementationVendor": {
+                                            "Build-Jdk": {
                                                 "type": "string"
                                             },
-                                            "implementationVersion": {
+                                            "Built-By": {
                                                 "type": "string"
                                             },
-                                            "manifestVersion": {
+                                            "Created-By": {
                                                 "type": "string"
                                             },
-                                            "name": {
+                                            "Extension-Name": {
                                                 "type": "string"
                                             },
-                                            "specificationTitle": {
+                                            "Group-Id": {
                                                 "type": "string"
                                             },
-                                            "specificationVendor": {
+                                            "Hudson-Version": {
                                                 "type": "string"
                                             },
-                                            "specificationVersion": {
+                                            "Implementation-Title": {
+                                                "type": "string"
+                                            },
+                                            "Implementation-Version": {
+                                                "type": "string"
+                                            },
+                                            "Jenkins-Version": {
+                                                "type": "string"
+                                            },
+                                            "Long-Name": {
+                                                "type": "string"
+                                            },
+                                            "Main-Class": {
+                                                "type": "string"
+                                            },
+                                            "Manifest-Version": {
+                                                "type": "string"
+                                            },
+                                            "Minimum-Java-Version": {
+                                                "type": "string"
+                                            },
+                                            "Plugin-Dependencies": {
+                                                "type": "string"
+                                            },
+                                            "Plugin-Developers": {
+                                                "type": "string"
+                                            },
+                                            "Plugin-License-Name": {
+                                                "type": "string"
+                                            },
+                                            "Plugin-License-Url": {
+                                                "type": "string"
+                                            },
+                                            "Plugin-ScmUrl": {
+                                                "type": "string"
+                                            },
+                                            "Plugin-Version": {
+                                                "type": "string"
+                                            },
+                                            "Short-Name": {
+                                                "type": "string"
+                                            },
+                                            "Specification-Title": {
                                                 "type": "string"
                                             }
                                         },
                                         "required": [
-                                            "extraFields",
-                                            "implementationTitle",
-                                            "implementationVendor",
-                                            "implementationVersion",
-                                            "manifestVersion",
-                                            "name",
-                                            "specificationTitle",
-                                            "specificationVendor",
-                                            "specificationVersion"
+                                            "Archiver-Version",
+                                            "Build-Jdk",
+                                            "Built-By",
+                                            "Created-By",
+                                            "Manifest-Version"
                                         ],
                                         "type": "object"
                                     }
-                                ]
+                                },
+                                "required": [
+                                    "main"
+                                ],
+                                "type": "object"
                             },
                             "name": {
                                 "type": "string"
@@ -225,203 +239,11 @@
                             "package": {
                                 "type": "string"
                             },
-                            "parentPackage": {
-                                "anyOf": [
-                                    {
-                                        "type": "null"
-                                    },
-                                    {
-                                        "properties": {
-                                            "foundBy": {
-                                                "type": "string"
-                                            },
-                                            "language": {
-                                                "type": "integer"
-                                            },
-                                            "licenses": {
-                                                "type": "null"
-                                            },
-                                            "manifest": {
-                                                "type": "string"
-                                            },
-                                            "metadata": {
-                                                "properties": {
-                                                    "manifest": {
-                                                        "properties": {
-                                                            "extraFields": {
-                                                                "properties": {
-                                                                    "Archiver-Version": {
-                                                                        "type": "string"
-                                                                    },
-                                                                    "Build-Jdk": {
-                                                                        "type": "string"
-                                                                    },
-                                                                    "Built-By": {
-                                                                        "type": "string"
-                                                                    },
-                                                                    "Created-By": {
-                                                                        "type": "string"
-                                                                    },
-                                                                    "Extension-Name": {
-                                                                        "type": "string"
-                                                                    },
-                                                                    "Group-Id": {
-                                                                        "type": "string"
-                                                                    },
-                                                                    "Hudson-Version": {
-                                                                        "type": "string"
-                                                                    },
-                                                                    "Jenkins-Version": {
-                                                                        "type": "string"
-                                                                    },
-                                                                    "Long-Name": {
-                                                                        "type": "string"
-                                                                    },
-                                                                    "Main-Class": {
-                                                                        "type": "string"
-                                                                    },
-                                                                    "Minimum-Java-Version": {
-                                                                        "type": "string"
-                                                                    },
-                                                                    "Plugin-Dependencies": {
-                                                                        "type": "string"
-                                                                    },
-                                                                    "Plugin-Developers": {
-                                                                        "type": "string"
-                                                                    },
-                                                                    "Plugin-License-Name": {
-                                                                        "type": "string"
-                                                                    },
-                                                                    "Plugin-License-Url": {
-                                                                        "type": "string"
-                                                                    },
-                                                                    "Plugin-ScmUrl": {
-                                                                        "type": "string"
-                                                                    },
-                                                                    "Plugin-Version": {
-                                                                        "type": "string"
-                                                                    },
-                                                                    "Short-Name": {
-                                                                        "type": "string"
-                                                                    }
-                                                                },
-                                                                "required": [
-                                                                    "Archiver-Version",
-                                                                    "Build-Jdk",
-                                                                    "Built-By",
-                                                                    "Created-By"
-                                                                ],
-                                                                "type": "object"
-                                                            },
-                                                            "implementationTitle": {
-                                                                "type": "string"
-                                                            },
-                                                            "implementationVendor": {
-                                                                "type": "string"
-                                                            },
-                                                            "implementationVersion": {
-                                                                "type": "string"
-                                                            },
-                                                            "manifestVersion": {
-                                                                "type": "string"
-                                                            },
-                                                            "name": {
-                                                                "type": "string"
-                                                            },
-                                                            "specificationTitle": {
-                                                                "type": "string"
-                                                            },
-                                                            "specificationVendor": {
-                                                                "type": "string"
-                                                            },
-                                                            "specificationVersion": {
-                                                                "type": "string"
-                                                            }
-                                                        },
-                                                        "required": [
-                                                            "extraFields",
-                                                            "implementationTitle",
-                                                            "implementationVendor",
-                                                            "implementationVersion",
-                                                            "manifestVersion",
-                                                            "name",
-                                                            "specificationTitle",
-                                                            "specificationVendor",
-                                                            "specificationVersion"
-                                                        ],
-                                                        "type": "object"
-                                                    },
-                                                    "parentPackage": {
-                                                        "type": "null"
-                                                    },
-                                                    "pomProperties": {
-                                                        "properties": {
-                                                            "Path": {
-                                                                "type": "string"
-                                                            },
-                                                            "artifactId": {
-                                                                "type": "string"
-                                                            },
-                                                            "extraFields": {
-                                                                "type": "null"
-                                                            },
-                                                            "groupId": {
-                                                                "type": "string"
-                                                            },
-                                                            "name": {
-                                                                "type": "string"
-                                                            },
-                                                            "version": {
-                                                                "type": "string"
-                                                            }
-                                                        },
-                                                        "required": [
-                                                            "Path",
-                                                            "artifactId",
-                                                            "extraFields",
-                                                            "groupId",
-                                                            "name",
-                                                            "version"
-                                                        ],
-                                                        "type": "object"
-                                                    }
-                                                },
-                                                "required": [
-                                                    "manifest",
-                                                    "parentPackage",
-                                                    "pomProperties"
-                                                ],
-                                                "type": "object"
-                                            },
-                                            "sources": {
-                                                "type": "null"
-                                            },
-                                            "type": {
-                                                "type": "string"
-                                            },
-                                            "version": {
-                                                "type": "string"
-                                            }
-                                        },
-                                        "required": [
-                                            "foundBy",
-                                            "language",
-                                            "licenses",
-                                            "manifest",
-                                            "metadata",
-                                            "sources",
-                                            "type",
-                                            "version"
-                                        ],
-                                        "type": "object"
-                                    }
-                                ]
+                            "platform": {
+                                "type": "string"
                             },
                             "pomProperties": {
                                 "properties": {
-                                    "Path": {
-                                        "type": "string"
-                                    },
                                     "artifactId": {
                                         "type": "string"
                                     },
@@ -434,16 +256,19 @@
                                     "name": {
                                         "type": "string"
                                     },
+                                    "path": {
+                                        "type": "string"
+                                    },
                                     "version": {
                                         "type": "string"
                                     }
                                 },
                                 "required": [
-                                    "Path",
                                     "artifactId",
                                     "extraFields",
                                     "groupId",
                                     "name",
+                                    "path",
                                     "version"
                                 ],
                                 "type": "object"
@@ -457,6 +282,9 @@
                             "release": {
                                 "type": "string"
                             },
+                            "sitePackagesRootPath": {
+                                "type": "string"
+                            },
                             "size": {
                                 "type": "integer"
                             },
@@ -466,6 +294,12 @@
                             "sourceRpm": {
                                 "type": "string"
                             },
+                            "topLevelPackages": {
+                                "items": {
+                                    "type": "string"
+                                },
+                                "type": "array"
+                            },
                             "url": {
                                 "type": "string"
                             },
@@ -474,6 +308,9 @@
                             },
                             "version": {
                                 "type": "string"
+                            },
+                            "virtualPath": {
+                                "type": "string"
                             }
                         },
                         "type": "object"
@@ -490,6 +327,7 @@
                 },
                 "required": [
                     "foundBy",
+                    "licenses",
                     "locations",
                     "name",
                     "type",

syft/cataloger/apkdb/parse_apk_db.go

@@ -49,11 +49,12 @@ func parseApkDB(_ string, reader io.Reader) ([]pkg.Package, error) {
 		}
 		if metadata != nil {
 			packages = append(packages, pkg.Package{
-				Name:     metadata.Package,
-				Version:  metadata.Version,
-				Licenses: strings.Split(metadata.License, " "),
-				Type:     pkg.ApkPkg,
-				Metadata: *metadata,
+				Name:         metadata.Package,
+				Version:      metadata.Version,
+				Licenses:     strings.Split(metadata.License, " "),
+				Type:         pkg.ApkPkg,
+				MetadataType: pkg.ApkMetadataType,
+				Metadata:     *metadata,
 			})
 		}
 	}
@@ -88,7 +89,16 @@ func parseApkDBEntry(reader io.Reader) (*pkg.ApkMetadata, error) {
 
 		switch key {
 		case "F":
-			lastFile = "/" + value
+			currentFile := "/" + value
+
+			newFileRecord := pkg.ApkFileRecord{
+				Path: currentFile,
+			}
+			files = append(files, newFileRecord)
+			fileRecord = &files[len(files)-1]
+
+			// future aux references are relative to previous "F" records
+			lastFile = currentFile
 			continue
 		case "R":
 			newFileRecord := pkg.ApkFileRecord{
@@ -96,24 +106,24 @@ func parseApkDBEntry(reader io.Reader) (*pkg.ApkMetadata, error) {
 			}
 			files = append(files, newFileRecord)
 			fileRecord = &files[len(files)-1]
-		case "a":
+		case "a", "M":
 			ownershipFields := strings.Split(value, ":")
 			if len(ownershipFields) < 3 {
-				log.Errorf("unexpected APK ownership field: %q", value)
+				log.Warnf("unexpected APK ownership field: %q", value)
 				continue
 			}
 			if fileRecord == nil {
-				log.Errorf("ownership field with no parent record: %q", value)
+				log.Warnf("ownership field with no parent record: %q", value)
 				continue
 			}
 			fileRecord.OwnerUID = ownershipFields[0]
-			fileRecord.OwnerGUI = ownershipFields[1]
+			fileRecord.OwnerGID = ownershipFields[1]
 			fileRecord.Permissions = ownershipFields[2]
 			// note: there are more optional fields available that we are not capturing, e.g.:
 			// "0:0:755:Q1JaDEHQHBbizhEzoWK1YxuraNU/4="
 		case "Z":
 			if fileRecord == nil {
-				log.Errorf("checksum field with no parent record: %q", value)
+				log.Warnf("checksum field with no parent record: %q", value)
 				continue
 			}
 			fileRecord.Checksum = value

syft/cataloger/apkdb/parse_apk_db_test.go

@@ -19,10 +19,25 @@ func TestExtraFileAttributes(t *testing.T) {
 			name: "test extra file attributes (checksum) are ignored",
 			expected: pkg.ApkMetadata{
 				Files: []pkg.ApkFileRecord{
+					{
+						Path: "/usr",
+					},
+					{
+						Path: "/usr/lib",
+					},
+					{
+						Path: "/usr/lib/jvm",
+					},
+					{
+						Path: "/usr/lib/jvm/java-1.8-openjdk",
+					},
+					{
+						Path: "/usr/lib/jvm/java-1.8-openjdk/bin",
+					},
 					{
 						Path:        "/usr/lib/jvm/java-1.8-openjdk/bin/policytool",
 						OwnerUID:    "0",
-						OwnerGUI:    "0",
+						OwnerGID:    "0",
 						Permissions: "755",
 						Checksum:    "Q1M0C9qfC/+kdRiOodeihG2GMRtkE=",
 					},
@@ -60,13 +75,13 @@ func TestExtraFileAttributes(t *testing.T) {
 	}
 }
 
-func TestSinglePackage(t *testing.T) {
+func TestSinglePackageDetails(t *testing.T) {
 	tests := []struct {
-		name     string
+		fixture  string
 		expected pkg.ApkMetadata
 	}{
 		{
-			name: "Test Single Package",
+			fixture: "test-fixtures/single",
 			expected: pkg.ApkMetadata{
 				Package:          "musl-utils",
 				OriginPackage:    "musl",
@@ -82,51 +97,428 @@ func TestSinglePackage(t *testing.T) {
 				PullChecksum:     "Q1bTtF5526tETKfL+lnigzIDvm+2o=",
 				GitCommitOfAport: "4024cc3b29ad4c65544ad068b8f59172b5494306",
 				Files: []pkg.ApkFileRecord{
+					{
+						Path: "/sbin",
+					},
 					{
 						Path:        "/sbin/ldconfig",
 						OwnerUID:    "0",
-						OwnerGUI:    "0",
+						OwnerGID:    "0",
 						Permissions: "755",
 						Checksum:    "Q1Kja2+POZKxEkUOZqwSjC6kmaED4=",
 					},
+					{
+						Path: "/usr",
+					},
+					{
+						Path: "/usr/bin",
+					},
 					{
 						Path:        "/usr/bin/iconv",
 						OwnerUID:    "0",
-						OwnerGUI:    "0",
+						OwnerGID:    "0",
 						Permissions: "755",
 						Checksum:    "Q1CVmFbdY+Hv6/jAHl1gec2Kbx1EY=",
 					},
 					{
 						Path:        "/usr/bin/ldd",
 						OwnerUID:    "0",
-						OwnerGUI:    "0",
+						OwnerGID:    "0",
 						Permissions: "755",
 						Checksum:    "Q1yFAhGggmL7ERgbIA7KQxyTzf3ks=",
 					},
 					{
 						Path:        "/usr/bin/getconf",
 						OwnerUID:    "0",
-						OwnerGUI:    "0",
+						OwnerGID:    "0",
 						Permissions: "755",
 						Checksum:    "Q1dAdYK8M/INibRQF5B3Rw7cmNDDA=",
 					},
 					{
 						Path:        "/usr/bin/getent",
 						OwnerUID:    "0",
-						OwnerGUI:    "0",
+						OwnerGID:    "0",
 						Permissions: "755",
 						Checksum:    "Q1eR2Dz/WylabgbWMTkd2+hGmEya4=",
 					},
 				},
 			},
 		},
+		{
+			fixture: "test-fixtures/base",
+			expected: pkg.ApkMetadata{
+				Package:          "alpine-baselayout",
+				OriginPackage:    "alpine-baselayout",
+				Version:          "3.2.0-r6",
+				Description:      "Alpine base dir structure and init scripts",
+				Maintainer:       "Natanael Copa <ncopa@alpinelinux.org>",
+				License:          "GPL-2.0-only",
+				Architecture:     "x86_64",
+				URL:              "https://git.alpinelinux.org/cgit/aports/tree/main/alpine-baselayout",
+				Size:             19917,
+				InstalledSize:    409600,
+				PullDependencies: "/bin/sh so:libc.musl-x86_64.so.1",
+				PullChecksum:     "Q1myMNfd7u5v5UTgNHeq1e31qTjZU=",
+				GitCommitOfAport: "e1c51734fa96fa4bac92e9f14a474324c67916fc",
+				Files: []pkg.ApkFileRecord{
+					{
+						Path: "/dev",
+					},
+					{
+						Path: "/dev/pts",
+					},
+					{
+						Path: "/dev/shm",
+					},
+					{
+						Path: "/etc",
+					},
+					{
+						Path:     "/etc/fstab",
+						Checksum: "Q11Q7hNe8QpDS531guqCdrXBzoA/o=",
+					},
+					{
+						Path:     "/etc/group",
+						Checksum: "Q1oJ16xWudgKOrXIEquEDzlF2Lsm4=",
+					},
+					{
+						Path:     "/etc/hostname",
+						Checksum: "Q16nVwYVXP/tChvUPdukVD2ifXOmc=",
+					},
+					{
+						Path:     "/etc/hosts",
+						Checksum: "Q1BD6zJKZTRWyqGnPi4tSfd3krsMU=",
+					},
+					{
+						Path:     "/etc/inittab",
+						Checksum: "Q1TsthbhW7QzWRe1E/NKwTOuD4pHc=",
+					},
+					{
+						Path:     "/etc/modules",
+						Checksum: "Q1toogjUipHGcMgECgPJX64SwUT1M=",
+					},
+					{
+						Path:     "/etc/motd",
+						Checksum: "Q1XmduVVNURHQ27TvYp1Lr5TMtFcA=",
+					},
+					{
+						Path:        "/etc/mtab",
+						OwnerUID:    "0",
+						OwnerGID:    "0",
+						Permissions: "777",
+						Checksum:    "Q1kiljhXXH1LlQroHsEJIkPZg2eiw=",
+					},
+					{
+						Path:     "/etc/passwd",
+						Checksum: "Q1TchuuLUfur0izvfZQZxgN/LJhB8=",
+					},
+					{
+						Path:     "/etc/profile",
+						Checksum: "Q1KpFb8kl5LvwXWlY3e58FNsjrI34=",
+					},
+					{
+						Path:     "/etc/protocols",
+						Checksum: "Q13FqXUnvuOpMDrH/6rehxuYAEE34=",
+					},
+					{
+						Path:     "/etc/services",
+						Checksum: "Q1C6HJNgQvLWqt5VY+n7MZJ1rsDuY=",
+					},
+					{
+						Path:        "/etc/shadow",
+						OwnerUID:    "0",
+						OwnerGID:    "42",
+						Permissions: "640",
+						Checksum:    "Q1ltrPIAW2zHeDiajsex2Bdmq3uqA=",
+					},
+					{
+						Path:     "/etc/shells",
+						Checksum: "Q1ojm2YdpCJ6B/apGDaZ/Sdb2xJkA=",
+					},
+					{
+						Path:     "/etc/sysctl.conf",
+						Checksum: "Q14upz3tfnNxZkIEsUhWn7Xoiw96g=",
+					},
+					{
+						Path: "/etc/apk",
+					},
+					{
+						Path: "/etc/conf.d",
+					},
+					{
+						Path: "/etc/crontabs",
+					},
+					{
+						Path:        "/etc/crontabs/root",
+						OwnerUID:    "0",
+						OwnerGID:    "0",
+						Permissions: "600",
+						Checksum:    "Q1vfk1apUWI4yLJGhhNRd0kJixfvY=",
+					},
+					{
+						Path: "/etc/init.d",
+					},
+					{
+						Path: "/etc/modprobe.d",
+					},
+					{
+						Path:     "/etc/modprobe.d/aliases.conf",
+						Checksum: "Q1WUbh6TBYNVK7e4Y+uUvLs/7viqk=",
+					},
+					{
+						Path:     "/etc/modprobe.d/blacklist.conf",
+						Checksum: "Q1xxYGU6S6TLQvb7ervPrWWwAWqMg=",
+					},
+					{
+						Path:     "/etc/modprobe.d/i386.conf",
+						Checksum: "Q1pnay/njn6ol9cCssL7KiZZ8etlc=",
+					},
+					{
+						Path:     "/etc/modprobe.d/kms.conf",
+						Checksum: "Q1ynbLn3GYDpvajba/ldp1niayeog=",
+					},
+					{
+						Path: "/etc/modules-load.d",
+					},
+					{
+						Path: "/etc/network",
+					},
+					{
+						Path: "/etc/network/if-down.d",
+					},
+					{
+						Path: "/etc/network/if-post-down.d",
+					},
+					{
+						Path: "/etc/network/if-pre-up.d",
+					},
+					{
+						Path: "/etc/network/if-up.d",
+					},
+					{
+						Path: "/etc/opt",
+					},
+					{
+						Path: "/etc/periodic",
+					},
+					{
+						Path: "/etc/periodic/15min",
+					},
+					{
+						Path: "/etc/periodic/daily",
+					},
+					{
+						Path: "/etc/periodic/hourly",
+					},
+					{
+						Path: "/etc/periodic/monthly",
+					},
+					{
+						Path: "/etc/periodic/weekly",
+					},
+					{
+						Path: "/etc/profile.d",
+					},
+					{
+						Path:     "/etc/profile.d/color_prompt",
+						Checksum: "Q10wL23GuSCVfumMRgakabUI6EsSk=",
+					},
+					{
+						Path:     "/etc/profile.d/locale",
+						Checksum: "Q1R4bIEpnKxxOSrlnZy9AoawqZ5DU=",
+					},
+					{
+						Path: "/etc/sysctl.d",
+					},
+					{
+						Path: "/home",
+					},
+					{
+						Path: "/lib",
+					},
+					{
+						Path: "/lib/firmware",
+					},
+					{
+						Path: "/lib/mdev",
+					},
+					{
+						Path: "/lib/modules-load.d",
+					},
+					{
+						Path: "/lib/sysctl.d",
+					},
+					{
+						Path:     "/lib/sysctl.d/00-alpine.conf",
+						Checksum: "Q1HpElzW1xEgmKfERtTy7oommnq6c=",
+					},
+					{
+						Path: "/media",
+					},
+					{
+						Path: "/media/cdrom",
+					},
+					{
+						Path: "/media/floppy",
+					},
+					{
+						Path: "/media/usb",
+					},
+					{
+						Path: "/mnt",
+					},
+					{
+						Path: "/opt",
+					},
+					{
+						Path: "/proc",
+					},
+					{
+						Path:        "/root",
+						OwnerUID:    "0",
+						OwnerGID:    "0",
+						Permissions: "700",
+					},
+					{
+						Path: "/run",
+					},
+					{
+						Path: "/sbin",
+					},
+					{
+						Path:        "/sbin/mkmntdirs",
+						OwnerUID:    "0",
+						OwnerGID:    "0",
+						Permissions: "755",
+						Checksum:    "Q1YeuSmC7iDbEWrusPzA/zUQF6YSg=",
+					},
+					{
+						Path: "/srv",
+					},
+					{
+						Path: "/sys",
+					},
+					{
+						Path:        "/tmp",
+						OwnerUID:    "0",
+						OwnerGID:    "0",
+						Permissions: "1777",
+					},
+					{
+						Path: "/usr",
+					},
+					{
+						Path: "/usr/lib",
+					},
+					{
+						Path: "/usr/lib/modules-load.d",
+					},
+					{
+						Path: "/usr/local",
+					},
+					{
+						Path: "/usr/local/bin",
+					},
+					{
+						Path: "/usr/local/lib",
+					},
+					{
+						Path: "/usr/local/share",
+					},
+					{
+						Path: "/usr/sbin",
+					},
+					{
+						Path: "/usr/share",
+					},
+					{
+						Path: "/usr/share/man",
+					},
+					{
+						Path: "/usr/share/misc",
+					},
+					{
+						Path: "/var",
+					},
+					{
+						Path:        "/var/run",
+						OwnerUID:    "0",
+						OwnerGID:    "0",
+						Permissions: "777",
+						Checksum:    "Q11/SNZz/8cK2dSKK+cJpVrZIuF4Q=",
+					},
+					{
+						Path: "/var/cache",
+					},
+					{
+						Path: "/var/cache/misc",
+					},
+					{
+						Path:        "/var/empty",
+						OwnerUID:    "0",
+						OwnerGID:    "0",
+						Permissions: "555",
+					},
+					{
+						Path: "/var/lib",
+					},
+					{
+						Path: "/var/lib/misc",
+					},
+					{
+						Path: "/var/local",
+					},
+					{
+						Path: "/var/lock",
+					},
+					{
+						Path: "/var/lock/subsys",
+					},
+					{
+						Path: "/var/log",
+					},
+					{
+						Path: "/var/mail",
+					},
+					{
+						Path: "/var/opt",
+					},
+					{
+						Path: "/var/spool",
+					},
+					{
+						Path:        "/var/spool/mail",
+						OwnerUID:    "0",
+						OwnerGID:    "0",
+						Permissions: "777",
+						Checksum:    "Q1dzbdazYZA2nTzSIG3YyNw7d4Juc=",
+					},
+					{
+						Path: "/var/spool/cron",
+					},
+					{
+						Path:        "/var/spool/cron/crontabs",
+						OwnerUID:    "0",
+						OwnerGID:    "0",
+						Permissions: "777",
+						Checksum:    "Q1OFZt+ZMp7j0Gny0rqSKuWJyqYmA=",
+					},
+					{
+						Path:        "/var/tmp",
+						OwnerUID:    "0",
+						OwnerGID:    "0",
+						Permissions: "1777",
+					},
+				},
+			},
+		},
 	}
 
 	for _, test := range tests {
-		t.Run(test.name, func(t *testing.T) {
-			file, err := os.Open("test-fixtures/single")
+		t.Run(test.fixture, func(t *testing.T) {
+			file, err := os.Open(test.fixture)
 			if err != nil {
-				t.Fatal("Unable to read test_fixtures/single: ", err)
+				t.Fatal("Unable to read fixture: ", err)
 			}
 			defer func() {
 				err := file.Close()
@@ -160,10 +552,11 @@ func TestMultiplePackages(t *testing.T) {
 			fixture: "test-fixtures/multiple",
 			expected: []pkg.Package{
 				{
-					Name:     "libc-utils",
-					Version:  "0.7.2-r0",
-					Licenses: []string{"BSD"},
-					Type:     pkg.ApkPkg,
+					Name:         "libc-utils",
+					Version:      "0.7.2-r0",
+					Licenses:     []string{"BSD"},
+					Type:         pkg.ApkPkg,
+					MetadataType: pkg.ApkMetadataType,
 					Metadata: pkg.ApkMetadata{
 						Package:          "libc-utils",
 						OriginPackage:    "libc-dev",
@@ -182,10 +575,11 @@ func TestMultiplePackages(t *testing.T) {
 					},
 				},
 				{
-					Name:     "musl-utils",
-					Version:  "1.1.24-r2",
-					Licenses: []string{"MIT", "BSD", "GPL2+"},
-					Type:     pkg.ApkPkg,
+					Name:         "musl-utils",
+					Version:      "1.1.24-r2",
+					Licenses:     []string{"MIT", "BSD", "GPL2+"},
+					Type:         pkg.ApkPkg,
+					MetadataType: pkg.ApkMetadataType,
 					Metadata: pkg.ApkMetadata{
 						Package:          "musl-utils",
 						OriginPackage:    "musl",
@@ -201,38 +595,47 @@ func TestMultiplePackages(t *testing.T) {
 						PullChecksum:     "Q1bTtF5526tETKfL+lnigzIDvm+2o=",
 						GitCommitOfAport: "4024cc3b29ad4c65544ad068b8f59172b5494306",
 						Files: []pkg.ApkFileRecord{
+							{
+								Path: "/sbin",
+							},
 							{
 								Path:        "/sbin/ldconfig",
 								OwnerUID:    "0",
-								OwnerGUI:    "0",
+								OwnerGID:    "0",
 								Permissions: "755",
 								Checksum:    "Q1Kja2+POZKxEkUOZqwSjC6kmaED4=",
 							},
+							{
+								Path: "/usr",
+							},
+							{
+								Path: "/usr/bin",
+							},
 							{
 								Path:        "/usr/bin/iconv",
 								OwnerUID:    "0",
-								OwnerGUI:    "0",
+								OwnerGID:    "0",
 								Permissions: "755",
 								Checksum:    "Q1CVmFbdY+Hv6/jAHl1gec2Kbx1EY=",
 							},
 							{
 								Path:        "/usr/bin/ldd",
 								OwnerUID:    "0",
-								OwnerGUI:    "0",
+								OwnerGID:    "0",
 								Permissions: "755",
 								Checksum:    "Q1yFAhGggmL7ERgbIA7KQxyTzf3ks=",
 							},
 							{
 								Path:        "/usr/bin/getconf",
 								OwnerUID:    "0",
-								OwnerGUI:    "0",
+								OwnerGID:    "0",
 								Permissions: "755",
 								Checksum:    "Q1dAdYK8M/INibRQF5B3Rw7cmNDDA=",
 							},
 							{
 								Path:        "/usr/bin/getent",
 								OwnerUID:    "0",
-								OwnerGUI:    "0",
+								OwnerGID:    "0",
 								Permissions: "755",
 								Checksum:    "Q1eR2Dz/WylabgbWMTkd2+hGmEya4=",
 							},

syft/cataloger/apkdb/test-fixtures/base

@@ -0,0 +1,149 @@
+C:Q1myMNfd7u5v5UTgNHeq1e31qTjZU=
+P:alpine-baselayout
+V:3.2.0-r6
+A:x86_64
+S:19917
+I:409600
+T:Alpine base dir structure and init scripts
+U:https://git.alpinelinux.org/cgit/aports/tree/main/alpine-baselayout
+L:GPL-2.0-only
+o:alpine-baselayout
+m:Natanael Copa <ncopa@alpinelinux.org>
+t:1590679557
+c:e1c51734fa96fa4bac92e9f14a474324c67916fc
+D:/bin/sh so:libc.musl-x86_64.so.1
+p:cmd:mkmntdirs
+F:dev
+F:dev/pts
+F:dev/shm
+F:etc
+R:fstab
+Z:Q11Q7hNe8QpDS531guqCdrXBzoA/o=
+R:group
+Z:Q1oJ16xWudgKOrXIEquEDzlF2Lsm4=
+R:hostname
+Z:Q16nVwYVXP/tChvUPdukVD2ifXOmc=
+R:hosts
+Z:Q1BD6zJKZTRWyqGnPi4tSfd3krsMU=
+R:inittab
+Z:Q1TsthbhW7QzWRe1E/NKwTOuD4pHc=
+R:modules
+Z:Q1toogjUipHGcMgECgPJX64SwUT1M=
+R:motd
+Z:Q1XmduVVNURHQ27TvYp1Lr5TMtFcA=
+R:mtab
+a:0:0:777
+Z:Q1kiljhXXH1LlQroHsEJIkPZg2eiw=
+R:passwd
+Z:Q1TchuuLUfur0izvfZQZxgN/LJhB8=
+R:profile
+Z:Q1KpFb8kl5LvwXWlY3e58FNsjrI34=
+R:protocols
+Z:Q13FqXUnvuOpMDrH/6rehxuYAEE34=
+R:services
+Z:Q1C6HJNgQvLWqt5VY+n7MZJ1rsDuY=
+R:shadow
+a:0:42:640
+Z:Q1ltrPIAW2zHeDiajsex2Bdmq3uqA=
+R:shells
+Z:Q1ojm2YdpCJ6B/apGDaZ/Sdb2xJkA=
+R:sysctl.conf
+Z:Q14upz3tfnNxZkIEsUhWn7Xoiw96g=
+F:etc/apk
+F:etc/conf.d
+F:etc/crontabs
+R:root
+a:0:0:600
+Z:Q1vfk1apUWI4yLJGhhNRd0kJixfvY=
+F:etc/init.d
+F:etc/modprobe.d
+R:aliases.conf
+Z:Q1WUbh6TBYNVK7e4Y+uUvLs/7viqk=
+R:blacklist.conf
+Z:Q1xxYGU6S6TLQvb7ervPrWWwAWqMg=
+R:i386.conf
+Z:Q1pnay/njn6ol9cCssL7KiZZ8etlc=
+R:kms.conf
+Z:Q1ynbLn3GYDpvajba/ldp1niayeog=
+F:etc/modules-load.d
+F:etc/network
+F:etc/network/if-down.d
+F:etc/network/if-post-down.d
+F:etc/network/if-pre-up.d
+F:etc/network/if-up.d
+F:etc/opt
+F:etc/periodic
+F:etc/periodic/15min
+F:etc/periodic/daily
+F:etc/periodic/hourly
+F:etc/periodic/monthly
+F:etc/periodic/weekly
+F:etc/profile.d
+R:color_prompt
+Z:Q10wL23GuSCVfumMRgakabUI6EsSk=
+R:locale
+Z:Q1R4bIEpnKxxOSrlnZy9AoawqZ5DU=
+F:etc/sysctl.d
+F:home
+F:lib
+F:lib/firmware
+F:lib/mdev
+F:lib/modules-load.d
+F:lib/sysctl.d
+R:00-alpine.conf
+Z:Q1HpElzW1xEgmKfERtTy7oommnq6c=
+F:media
+F:media/cdrom
+F:media/floppy
+F:media/usb
+F:mnt
+F:opt
+F:proc
+F:root
+M:0:0:700
+F:run
+F:sbin
+R:mkmntdirs
+a:0:0:755
+Z:Q1YeuSmC7iDbEWrusPzA/zUQF6YSg=
+F:srv
+F:sys
+F:tmp
+M:0:0:1777
+F:usr
+F:usr/lib
+F:usr/lib/modules-load.d
+F:usr/local
+F:usr/local/bin
+F:usr/local/lib
+F:usr/local/share
+F:usr/sbin
+F:usr/share
+F:usr/share/man
+F:usr/share/misc
+F:var
+R:run
+a:0:0:777
+Z:Q11/SNZz/8cK2dSKK+cJpVrZIuF4Q=
+F:var/cache
+F:var/cache/misc
+F:var/empty
+M:0:0:555
+F:var/lib
+F:var/lib/misc
+F:var/local
+F:var/lock
+F:var/lock/subsys
+F:var/log
+F:var/mail
+F:var/opt
+F:var/spool
+R:mail
+a:0:0:777
+Z:Q1dzbdazYZA2nTzSIG3YyNw7d4Juc=
+F:var/spool/cron
+R:crontabs
+a:0:0:777
+Z:Q1OFZt+ZMp7j0Gny0rqSKuWJyqYmA=
+F:var/tmp
+M:0:0:1777

syft/cataloger/catalog.go

@@ -1,7 +1,6 @@
 package cataloger
 
 import (
-	"github.com/anchore/stereoscope/pkg/file"
 	"github.com/anchore/syft/internal/bus"
 	"github.com/anchore/syft/internal/log"
 	"github.com/anchore/syft/syft/event"
@@ -39,36 +38,21 @@ func newMonitor() (*progress.Manual, *progress.Manual) {
 // request.
 func Catalog(resolver scope.Resolver, catalogers ...Cataloger) (*pkg.Catalog, error) {
 	catalog := pkg.NewCatalog()
-	fileSelection := make([]file.Reference, 0)
-
 	filesProcessed, packagesDiscovered := newMonitor()
 
-	// ask catalogers for files to extract from the image tar
-	for _, a := range catalogers {
-		fileSelection = append(fileSelection, a.SelectFiles(resolver)...)
-		log.Debugf("cataloger '%s' selected '%d' files", a.Name(), len(fileSelection))
-		filesProcessed.N += int64(len(fileSelection))
-	}
-
-	// fetch contents for requested selection by catalogers
-	// TODO: we should consider refactoring to return a set of io.Readers instead of the full contents themselves (allow for optional buffering).
-	contents, err := resolver.MultipleFileContentsByRef(fileSelection...)
-	if err != nil {
-		return nil, err
-	}
-
 	// perform analysis, accumulating errors for each failed analysis
 	var errs error
-	for _, a := range catalogers {
-		// TODO: check for multiple rounds of analyses by Iterate error
-		packages, err := a.Catalog(contents)
+	for _, theCataloger := range catalogers {
+		packages, err := theCataloger.Catalog(resolver)
 		if err != nil {
 			errs = multierror.Append(errs, err)
 			continue
 		}
 
-		log.Debugf("cataloger '%s' discovered '%d' packages", a.Name(), len(packages))
-		packagesDiscovered.N += int64(len(packages))
+		catalogedPackages := len(packages)
+
+		log.Debugf("cataloger '%s' discovered '%d' packages", theCataloger.Name(), catalogedPackages)
+		packagesDiscovered.N += int64(catalogedPackages)
 
 		for _, p := range packages {
 			catalog.Add(p)

syft/cataloger/cataloger.go

@@ -6,7 +6,6 @@ catalogers defined in child packages as well as the interface definition to impl
 package cataloger
 
 import (
-	"github.com/anchore/stereoscope/pkg/file"
 	"github.com/anchore/syft/syft/cataloger/apkdb"
 	"github.com/anchore/syft/syft/cataloger/deb"
 	"github.com/anchore/syft/syft/cataloger/golang"
@@ -25,20 +24,16 @@ import (
 type Cataloger interface {
 	// Name returns a string that uniquely describes a cataloger
 	Name() string
-	// SelectFiles discovers and returns specific files that the cataloger would like to inspect the contents of.
-	SelectFiles(scope.FileResolver) []file.Reference
-	// Catalog is given the file contents and should return any discovered Packages after analyzing the contents.
-	Catalog(map[file.Reference]string) ([]pkg.Package, error)
-	// TODO: add "IterationNeeded" error to indicate to the driver to continue with another Select/Catalog pass
-	// TODO: we should consider refactoring to return a set of io.Readers instead of the full contents themselves (allow for optional buffering).
+	// Catalog is given an object to resolve file references and content, this function returns any discovered Packages after analyzing the catalog source.
+	Catalog(resolver scope.Resolver) ([]pkg.Package, error)
 }
 
 // ImageCatalogers returns a slice of locally implemented catalogers that are fit for detecting installations of packages.
 func ImageCatalogers() []Cataloger {
 	return []Cataloger{
 		ruby.NewGemSpecCataloger(),
-		python.NewPythonCataloger(),         // TODO: split and replace me
-		javascript.NewJavascriptCataloger(), // TODO: split and replace me
+		python.NewPythonPackageCataloger(),
+		javascript.NewJavascriptPackageCataloger(),
 		deb.NewDpkgdbCataloger(),
 		rpmdb.NewRpmdbCataloger(),
 		java.NewJavaCataloger(),
@@ -51,8 +46,9 @@ func ImageCatalogers() []Cataloger {
 func DirectoryCatalogers() []Cataloger {
 	return []Cataloger{
 		ruby.NewGemFileLockCataloger(),
-		python.NewPythonCataloger(),         // TODO: split and replace me
-		javascript.NewJavascriptCataloger(), // TODO: split and replace me
+		python.NewPythonIndexCataloger(),
+		python.NewPythonPackageCataloger(),
+		javascript.NewJavascriptLockCataloger(),
 		deb.NewDpkgdbCataloger(),
 		rpmdb.NewRpmdbCataloger(),
 		java.NewJavaCataloger(),

syft/cataloger/common/generic_cataloger.go

@@ -34,73 +34,83 @@ func NewGenericCataloger(pathParsers map[string]ParserFn, globParsers map[string
 }
 
 // Name returns a string that uniquely describes the upstream cataloger that this Generic Cataloger represents.
-func (a *GenericCataloger) Name() string {
-	return a.upstreamCataloger
+func (c *GenericCataloger) Name() string {
+	return c.upstreamCataloger
 }
 
 // register pairs a set of file references with a parser function for future cataloging (when the file contents are resolved)
-func (a *GenericCataloger) register(files []file.Reference, parser ParserFn) {
-	a.selectedFiles = append(a.selectedFiles, files...)
+func (c *GenericCataloger) register(files []file.Reference, parser ParserFn) {
+	c.selectedFiles = append(c.selectedFiles, files...)
 	for _, f := range files {
-		a.parsers[f] = parser
+		c.parsers[f] = parser
 	}
 }
 
 // clear deletes all registered file-reference-to-parser-function pairings from former SelectFiles() and register() calls
-func (a *GenericCataloger) clear() {
-	a.selectedFiles = make([]file.Reference, 0)
-	a.parsers = make(map[file.Reference]ParserFn)
+func (c *GenericCataloger) clear() {
+	c.selectedFiles = make([]file.Reference, 0)
+	c.parsers = make(map[file.Reference]ParserFn)
+}
+
+// Catalog is given an object to resolve file references and content, this function returns any discovered Packages after analyzing the catalog source.
+func (c *GenericCataloger) Catalog(resolver scope.Resolver) ([]pkg.Package, error) {
+	fileSelection := c.selectFiles(resolver)
+	contents, err := resolver.MultipleFileContentsByRef(fileSelection...)
+	if err != nil {
+		return nil, err
+	}
+	return c.catalog(contents)
 }
 
 // SelectFiles takes a set of file trees and resolves and file references of interest for future cataloging
-func (a *GenericCataloger) SelectFiles(resolver scope.FileResolver) []file.Reference {
+func (c *GenericCataloger) selectFiles(resolver scope.FileResolver) []file.Reference {
 	// select by exact path
-	for path, parser := range a.pathParsers {
+	for path, parser := range c.pathParsers {
 		files, err := resolver.FilesByPath(file.Path(path))
 		if err != nil {
-			log.Errorf("cataloger failed to select files by path: %+v", err)
+			log.Warnf("cataloger failed to select files by path: %+v", err)
 		}
 		if files != nil {
-			a.register(files, parser)
+			c.register(files, parser)
 		}
 	}
 
 	// select by glob pattern
-	for globPattern, parser := range a.globParsers {
+	for globPattern, parser := range c.globParsers {
 		fileMatches, err := resolver.FilesByGlob(globPattern)
 		if err != nil {
-			log.Errorf("failed to find files by glob: %s", globPattern)
+			log.Warnf("failed to find files by glob: %s", globPattern)
 		}
 		if fileMatches != nil {
-			a.register(fileMatches, parser)
+			c.register(fileMatches, parser)
 		}
 	}
 
-	return a.selectedFiles
+	return c.selectedFiles
 }
 
-// Catalog takes a set of file contents and uses any configured parser functions to resolve and return discovered packages
-func (a *GenericCataloger) Catalog(contents map[file.Reference]string) ([]pkg.Package, error) {
-	defer a.clear()
+// catalog takes a set of file contents and uses any configured parser functions to resolve and return discovered packages
+func (c *GenericCataloger) catalog(contents map[file.Reference]string) ([]pkg.Package, error) {
+	defer c.clear()
 
 	packages := make([]pkg.Package, 0)
 
-	for reference, parser := range a.parsers {
+	for reference, parser := range c.parsers {
 		content, ok := contents[reference]
 		if !ok {
-			log.Errorf("cataloger '%s' missing file content: %+v", a.upstreamCataloger, reference)
+			log.Warnf("cataloger '%s' missing file content: %+v", c.upstreamCataloger, reference)
 			continue
 		}
 
 		entries, err := parser(string(reference.Path), strings.NewReader(content))
 		if err != nil {
 			// TODO: should we fail? or only log?
-			log.Errorf("cataloger '%s' failed to parse entries (reference=%+v): %+v", a.upstreamCataloger, reference, err)
+			log.Warnf("cataloger '%s' failed to parse entries (reference=%+v): %+v", c.upstreamCataloger, reference, err)
 			continue
 		}
 
 		for _, entry := range entries {
-			entry.FoundBy = a.upstreamCataloger
+			entry.FoundBy = c.upstreamCataloger
 			entry.Source = []file.Reference{reference}
 
 			packages = append(packages, entry)

syft/cataloger/common/generic_cataloger_test.go

@@ -7,21 +7,28 @@ import (
 	"testing"
 
 	"github.com/anchore/stereoscope/pkg/file"
-	"github.com/anchore/syft/internal"
 	"github.com/anchore/syft/syft/pkg"
 )
 
-type testResolver struct {
+type testResolverMock struct {
 	contents map[file.Reference]string
 }
 
-func newTestResolver() *testResolver {
-	return &testResolver{
+func newTestResolver() *testResolverMock {
+	return &testResolverMock{
 		contents: make(map[file.Reference]string),
 	}
 }
 
-func (r *testResolver) FilesByPath(paths ...file.Path) ([]file.Reference, error) {
+func (r *testResolverMock) FileContentsByRef(_ file.Reference) (string, error) {
+	return "", fmt.Errorf("not implemented")
+}
+
+func (r *testResolverMock) MultipleFileContentsByRef(_ ...file.Reference) (map[file.Reference]string, error) {
+	return r.contents, nil
+}
+
+func (r *testResolverMock) FilesByPath(paths ...file.Path) ([]file.Reference, error) {
 	results := make([]file.Reference, len(paths))
 
 	for idx, p := range paths {
@@ -32,13 +39,17 @@ func (r *testResolver) FilesByPath(paths ...file.Path) ([]file.Reference, error)
 	return results, nil
 }
 
-func (r *testResolver) FilesByGlob(patterns ...string) ([]file.Reference, error) {
+func (r *testResolverMock) FilesByGlob(_ ...string) ([]file.Reference, error) {
 	path := "/a-path.txt"
 	ref := file.NewFileReference(file.Path(path))
 	r.contents[ref] = fmt.Sprintf("%s file contents!", path)
 	return []file.Reference{ref}, nil
 }
 
+func (r *testResolverMock) RelativeFileByPath(_ file.Reference, _ string) (*file.Reference, error) {
+	return nil, fmt.Errorf("not implemented")
+}
+
 func parser(_ string, reader io.Reader) ([]pkg.Package, error) {
 	contents, err := ioutil.ReadAll(reader)
 	if err != nil {
@@ -64,31 +75,16 @@ func TestGenericCataloger(t *testing.T) {
 	resolver := newTestResolver()
 	cataloger := NewGenericCataloger(pathParsers, globParsers, upstream)
 
-	selected := cataloger.SelectFiles(resolver)
-
-	if len(selected) != 3 {
-		t.Fatalf("unexpected selection length: %d", len(selected))
-	}
-
-	expectedSelection := internal.NewStringSetFromSlice([]string{"/last/path.txt", "/another-path.txt", "/a-path.txt"})
-	selectionByPath := make(map[string]file.Reference)
-	for _, s := range selected {
-		if !expectedSelection.Contains(string(s.Path)) {
-			t.Errorf("unexpected selection path: %+v", s.Path)
-		}
-		selectionByPath[string(s.Path)] = s
-	}
-
-	expectedPkgs := make(map[file.Reference]pkg.Package)
-	for path, ref := range selectionByPath {
-		expectedPkgs[ref] = pkg.Package{
+	expectedSelection := []string{"/last/path.txt", "/another-path.txt", "/a-path.txt"}
+	expectedPkgs := make(map[string]pkg.Package)
+	for _, path := range expectedSelection {
+		expectedPkgs[path] = pkg.Package{
 			FoundBy: upstream,
-			Source:  []file.Reference{ref},
 			Name:    fmt.Sprintf("%s file contents!", path),
 		}
 	}
 
-	actualPkgs, err := cataloger.Catalog(resolver.contents)
+	actualPkgs, err := cataloger.Catalog(resolver)
 	if err != nil {
 		t.Fatalf("cataloger catalog action failed: %+v", err)
 	}
@@ -99,7 +95,7 @@ func TestGenericCataloger(t *testing.T) {
 
 	for _, p := range actualPkgs {
 		ref := p.Source[0]
-		exP, ok := expectedPkgs[ref]
+		exP, ok := expectedPkgs[string(ref.Path)]
 		if !ok {
 			t.Errorf("missing expected pkg: ref=%+v", ref)
 			continue

syft/cataloger/deb/cataloger.go

@@ -4,14 +4,193 @@ Package dpkg provides a concrete Cataloger implementation for Debian package DB
 package deb
 
 import (
-	"github.com/anchore/syft/syft/cataloger/common"
+	"fmt"
+	"io"
+	"path"
+	"strings"
+
+	"github.com/anchore/stereoscope/pkg/file"
+	"github.com/anchore/syft/syft/pkg"
+	"github.com/anchore/syft/syft/scope"
 )
 
+const (
+	dpkgStatusGlob = "**/var/lib/dpkg/status"
+	md5sumsExt     = ".md5sums"
+	docsPath       = "/usr/share/doc"
+)
+
+type Cataloger struct{}
+
 // NewDpkgdbCataloger returns a new Deb package cataloger object.
-func NewDpkgdbCataloger() *common.GenericCataloger {
-	globParsers := map[string]common.ParserFn{
-		"**/var/lib/dpkg/status": parseDpkgStatus,
+func NewDpkgdbCataloger() *Cataloger {
+	return &Cataloger{}
+}
+
+// Name returns a string that uniquely describes a cataloger
+func (c *Cataloger) Name() string {
+	return "dpkgdb-cataloger"
+}
+
+// Catalog is given an object to resolve file references and content, this function returns any discovered Packages after analyzing dpkg support files.
+func (c *Cataloger) Catalog(resolver scope.Resolver) ([]pkg.Package, error) {
+	dbFileMatches, err := resolver.FilesByGlob(dpkgStatusGlob)
+	if err != nil {
+		return nil, fmt.Errorf("failed to find dpkg status files's by glob: %w", err)
 	}
 
-	return common.NewGenericCataloger(nil, globParsers, "dpkgdb-cataloger")
+	var pkgs []pkg.Package
+	for _, dbRef := range dbFileMatches {
+		dbContents, err := resolver.FileContentsByRef(dbRef)
+		if err != nil {
+			return nil, err
+		}
+
+		pkgs, err = parseDpkgStatus(strings.NewReader(dbContents))
+		if err != nil {
+			return nil, fmt.Errorf("unable to catalog dpkg package=%+v: %w", dbRef.Path, err)
+		}
+
+		md5ContentsByName, md5RefsByName, err := fetchMd5Contents(resolver, dbRef, pkgs)
+		if err != nil {
+			return nil, fmt.Errorf("unable to find dpkg md5 contents: %w", err)
+		}
+
+		copyrightContentsByName, copyrightRefsByName, err := fetchCopyrightContents(resolver, dbRef, pkgs)
+		if err != nil {
+			return nil, fmt.Errorf("unable to find dpkg copyright contents: %w", err)
+		}
+
+		for i := range pkgs {
+			p := &pkgs[i]
+			p.FoundBy = c.Name()
+			p.Source = []file.Reference{dbRef}
+
+			if md5Reader, ok := md5ContentsByName[md5Key(*p)]; ok {
+				// attach the file list
+				metadata := p.Metadata.(pkg.DpkgMetadata)
+				metadata.Files = parseDpkgMD5Info(md5Reader)
+				p.Metadata = metadata
+
+				// keep a record of the file where this was discovered
+				if ref, ok := md5RefsByName[md5Key(*p)]; ok {
+					p.Source = append(p.Source, ref)
+				}
+			}
+
+			copyrightReader, ok := copyrightContentsByName[p.Name]
+			if ok {
+				// attach the licenses
+				p.Licenses = parseLicensesFromCopyright(copyrightReader)
+
+				// keep a record of the file where this was discovered
+				if ref, ok := copyrightRefsByName[p.Name]; ok {
+					p.Source = append(p.Source, ref)
+				}
+			}
+		}
+	}
+	return pkgs, nil
+}
+
+func fetchMd5Contents(resolver scope.Resolver, dbRef file.Reference, pkgs []pkg.Package) (map[string]io.Reader, map[string]file.Reference, error) {
+	// fetch all MD5 file contents. This approach is more efficient than fetching each MD5 file one at a time
+
+	var md5FileMatches []file.Reference
+	var nameByRef = make(map[file.Reference]string)
+	parentPath, err := dbRef.Path.ParentPath()
+	if err != nil {
+		return nil, nil, fmt.Errorf("unable to find parent of path=%+v: %w", dbRef.Path, err)
+	}
+	for _, p := range pkgs {
+		// look for /var/lib/dpkg/info/NAME:ARCH.md5sums
+		name := md5Key(p)
+		md5sumPath := path.Join(string(parentPath), "info", name+md5sumsExt)
+		md5SumRef, err := resolver.RelativeFileByPath(dbRef, md5sumPath)
+		if err != nil {
+			return nil, nil, fmt.Errorf("unable to find relative md5sum from path=%+v: %w", dbRef.Path, err)
+		}
+
+		if md5SumRef == nil {
+			// the most specific key did not work, fallback to just the name
+			// look for /var/lib/dpkg/info/NAME.md5sums
+			name := p.Name
+			md5sumPath := path.Join(string(parentPath), "info", name+md5sumsExt)
+			md5SumRef, err = resolver.RelativeFileByPath(dbRef, md5sumPath)
+			if err != nil {
+				return nil, nil, fmt.Errorf("unable to find relative md5sum from path=%+v: %w", dbRef.Path, err)
+			}
+		}
+		// we should have at least one reference
+		if md5SumRef != nil {
+			md5FileMatches = append(md5FileMatches, *md5SumRef)
+			nameByRef[*md5SumRef] = name
+		}
+	}
+
+	// fetch the md5 contents
+	md5ContentsByRef, err := resolver.MultipleFileContentsByRef(md5FileMatches...)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	// organize content results and refs by a combination of name and architecture
+	var contentsByName = make(map[string]io.Reader)
+	var refsByName = make(map[string]file.Reference)
+	for ref, contents := range md5ContentsByRef {
+		name := nameByRef[ref]
+		contentsByName[name] = strings.NewReader(contents)
+		refsByName[name] = ref
+	}
+
+	return contentsByName, refsByName, nil
+}
+
+func fetchCopyrightContents(resolver scope.Resolver, dbRef file.Reference, pkgs []pkg.Package) (map[string]io.Reader, map[string]file.Reference, error) {
+	// fetch all copyright file contents. This approach is more efficient than fetching each copyright file one at a time
+
+	var copyrightFileMatches []file.Reference
+	var nameByRef = make(map[file.Reference]string)
+	for _, p := range pkgs {
+		// look for /usr/share/docs/NAME/copyright files
+		name := p.Name
+		copyrightPath := path.Join(docsPath, name, "copyright")
+		copyrightRef, err := resolver.RelativeFileByPath(dbRef, copyrightPath)
+		if err != nil {
+			return nil, nil, fmt.Errorf("unable to find relative copyright from path=%+v: %w", dbRef.Path, err)
+		}
+
+		// we may not have a copyright file for each package, ignore missing files
+		if copyrightRef != nil {
+			copyrightFileMatches = append(copyrightFileMatches, *copyrightRef)
+			nameByRef[*copyrightRef] = name
+		}
+	}
+
+	// fetch the copyright contents
+	copyrightContentsByRef, err := resolver.MultipleFileContentsByRef(copyrightFileMatches...)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	// organize content results and refs by package name
+	var contentsByName = make(map[string]io.Reader)
+	var refsByName = make(map[string]file.Reference)
+	for ref, contents := range copyrightContentsByRef {
+		name := nameByRef[ref]
+		contentsByName[name] = strings.NewReader(contents)
+		refsByName[name] = ref
+	}
+
+	return contentsByName, refsByName, nil
+}
+
+func md5Key(p pkg.Package) string {
+	metadata := p.Metadata.(pkg.DpkgMetadata)
+
+	contentKey := p.Name
+	if metadata.Architecture != "" && metadata.Architecture != "all" {
+		contentKey = contentKey + ":" + metadata.Architecture
+	}
+	return contentKey
 }

syft/cataloger/deb/cataloger_test.go

@@ -0,0 +1,99 @@
+package deb
+
+import (
+	"testing"
+
+	"github.com/anchore/stereoscope/pkg/imagetest"
+	"github.com/anchore/syft/syft/pkg"
+	"github.com/anchore/syft/syft/scope"
+	"github.com/go-test/deep"
+)
+
+func TestDpkgCataloger(t *testing.T) {
+
+	tests := []struct {
+		name     string
+		sources  map[string][]string
+		expected []pkg.Package
+	}{
+		{
+			name: "go-case",
+			sources: map[string][]string{
+				"libpam-runtime": {"/var/lib/dpkg/status", "/var/lib/dpkg/info/libpam-runtime.md5sums", "/usr/share/doc/libpam-runtime/copyright"},
+			},
+			expected: []pkg.Package{
+				{
+					Name:         "libpam-runtime",
+					Version:      "1.1.8-3.6",
+					FoundBy:      "dpkgdb-cataloger",
+					Licenses:     []string{"GPL-2", "LGPL-2.1"},
+					Type:         pkg.DebPkg,
+					MetadataType: pkg.DpkgMetadataType,
+					Metadata: pkg.DpkgMetadata{
+						Package:       "libpam-runtime",
+						Source:        "pam",
+						Version:       "1.1.8-3.6",
+						Architecture:  "all",
+						Maintainer:    "Steve Langasek <vorlon@debian.org>",
+						InstalledSize: 1016,
+						Files: []pkg.DpkgFileRecord{
+							{Path: "/lib/x86_64-linux-gnu/libz.so.1.2.11", MD5: "55f905631797551d4d936a34c7e73474"},
+							{Path: "/usr/share/doc/zlib1g/changelog.Debian.gz", MD5: "cede84bda30d2380217f97753c8ccf3a"},
+							{Path: "/usr/share/doc/zlib1g/changelog.gz", MD5: "f3c9dafa6da7992c47328b4464f6d122"},
+							{Path: "/usr/share/doc/zlib1g/copyright", MD5: "a4fae96070439a5209a62ae5b8017ab2"},
+						},
+					},
+				},
+			},
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+
+			img, cleanup := imagetest.GetFixtureImage(t, "docker-archive", "image-dpkg")
+			defer cleanup()
+
+			s, err := scope.NewScopeFromImage(img, scope.AllLayersScope)
+			if err != nil {
+				t.Fatal(err)
+			}
+
+			c := NewDpkgdbCataloger()
+
+			actual, err := c.Catalog(s.Resolver)
+			if err != nil {
+				t.Fatalf("failed to catalog: %+v", err)
+			}
+
+			if len(actual) != len(test.expected) {
+				for _, a := range actual {
+					t.Logf("   %+v", a)
+				}
+				t.Fatalf("unexpected package count: %d!=%d", len(actual), len(test.expected))
+			}
+
+			// test sources...
+			for idx := range actual {
+				a := &actual[idx]
+				// we will test the sources separately
+				var sourcesList = make([]string, len(a.Source))
+				for i, s := range a.Source {
+					sourcesList[i] = string(s.Path)
+				}
+				a.Source = nil
+
+				for _, d := range deep.Equal(sourcesList, test.sources[a.Name]) {
+					t.Errorf("diff: %+v", d)
+				}
+			}
+
+			// test remaining fields...
+			for _, d := range deep.Equal(actual, test.expected) {
+				t.Errorf("diff: %+v", d)
+			}
+
+		})
+	}
+
+}

syft/cataloger/deb/parse_copyright.go

@@ -0,0 +1,38 @@
+package deb
+
+import (
+	"bufio"
+	"io"
+	"sort"
+	"strings"
+
+	"github.com/anchore/syft/internal"
+)
+
+// For more information see: https://www.debian.org/doc/packaging-manuals/copyright-format/1.0/#license-syntax
+
+func parseLicensesFromCopyright(reader io.Reader) []string {
+	findings := internal.NewStringSet()
+	scanner := bufio.NewScanner(reader)
+
+	for scanner.Scan() {
+		line := scanner.Text()
+		if strings.HasPrefix(line, "License:") {
+			candidate := strings.Replace(line, "License:", "", 1)
+			candidate = strings.TrimSpace(candidate)
+			if strings.Contains(candidate, " or ") || strings.Contains(candidate, " and ") {
+				// this is a multi-license summary, ignore this as other recurrent license lines should cover this
+				continue
+			}
+			if candidate != "" && strings.ToLower(candidate) != "none" {
+				findings.Add(candidate)
+			}
+		}
+	}
+
+	results := findings.ToSlice()
+
+	sort.Strings(results)
+
+	return results
+}

syft/cataloger/deb/parse_copyright_test.go

@@ -0,0 +1,58 @@
+package deb
+
+import (
+	"os"
+	"testing"
+
+	"github.com/go-test/deep"
+)
+
+func TestParseLicensesFromCopyright(t *testing.T) {
+	tests := []struct {
+		fixture  string
+		expected []string
+	}{
+		{
+			fixture:  "test-fixtures/copyright/trilicense",
+			expected: []string{"GPL-2", "LGPL-2.1", "MPL-1.1"},
+		},
+		{
+			fixture:  "test-fixtures/copyright/liblzma5",
+			expected: []string{"Autoconf", "GPL-2", "GPL-2+", "LGPL-2.1+", "PD", "PD-debian", "config-h", "noderivs", "permissive-fsf", "permissive-nowarranty", "probably-PD"},
+		},
+		{
+			fixture:  "test-fixtures/copyright/libaudit-common",
+			expected: []string{"GPL-2", "LGPL-2.1"},
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.fixture, func(t *testing.T) {
+			file, err := os.Open(test.fixture)
+			if err != nil {
+				t.Fatal("Unable to read: ", err)
+			}
+			defer func() {
+				err := file.Close()
+				if err != nil {
+					t.Fatal("closing file failed:", err)
+				}
+			}()
+
+			actual := parseLicensesFromCopyright(file)
+
+			if len(actual) != len(test.expected) {
+				for _, a := range actual {
+					t.Logf("   %+v", a)
+				}
+				t.Fatalf("unexpected package count: %d!=%d", len(actual), len(test.expected))
+			}
+
+			diffs := deep.Equal(actual, test.expected)
+			for _, d := range diffs {
+				t.Errorf("diff: %+v", d)
+			}
+
+		})
+	}
+}

syft/cataloger/deb/parse_dpkg_info_files.go

@@ -0,0 +1,30 @@
+package deb
+
+import (
+	"bufio"
+	"io"
+	"strings"
+
+	"github.com/anchore/syft/syft/pkg"
+)
+
+func parseDpkgMD5Info(reader io.Reader) []pkg.DpkgFileRecord {
+	var findings []pkg.DpkgFileRecord
+	scanner := bufio.NewScanner(reader)
+
+	for scanner.Scan() {
+		line := scanner.Text()
+		fields := strings.SplitN(line, " ", 2)
+		if len(fields) == 2 {
+			path := strings.TrimSpace(fields[1])
+			if !strings.HasPrefix(path, "/") {
+				path = "/" + path
+			}
+			findings = append(findings, pkg.DpkgFileRecord{
+				Path: path,
+				MD5:  strings.TrimSpace(fields[0]),
+			})
+		}
+	}
+	return findings
+}

syft/cataloger/deb/parse_dpkg_info_files_test.go

@@ -0,0 +1,57 @@
+package deb
+
+import (
+	"os"
+	"testing"
+
+	"github.com/go-test/deep"
+
+	"github.com/anchore/syft/syft/pkg"
+)
+
+func TestMD5SumInfoParsing(t *testing.T) {
+	tests := []struct {
+		fixture  string
+		expected []pkg.DpkgFileRecord
+	}{
+		{
+			fixture: "test-fixtures/info/zlib1g.md5sums",
+			expected: []pkg.DpkgFileRecord{
+				{Path: "/lib/x86_64-linux-gnu/libz.so.1.2.11", MD5: "55f905631797551d4d936a34c7e73474"},
+				{Path: "/usr/share/doc/zlib1g/changelog.Debian.gz", MD5: "cede84bda30d2380217f97753c8ccf3a"},
+				{Path: "/usr/share/doc/zlib1g/changelog.gz", MD5: "f3c9dafa6da7992c47328b4464f6d122"},
+				{Path: "/usr/share/doc/zlib1g/copyright", MD5: "a4fae96070439a5209a62ae5b8017ab2"},
+			},
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.fixture, func(t *testing.T) {
+			file, err := os.Open(test.fixture)
+			if err != nil {
+				t.Fatal("Unable to read: ", err)
+			}
+			defer func() {
+				err := file.Close()
+				if err != nil {
+					t.Fatal("closing file failed:", err)
+				}
+			}()
+
+			actual := parseDpkgMD5Info(file)
+
+			if len(actual) != len(test.expected) {
+				for _, a := range actual {
+					t.Logf("   %+v", a)
+				}
+				t.Fatalf("unexpected package count: %d!=%d", len(actual), len(test.expected))
+			}
+
+			diffs := deep.Equal(actual, test.expected)
+			for _, d := range diffs {
+				t.Errorf("diff: %+v", d)
+			}
+
+		})
+	}
+}

syft/cataloger/deb/parse_dpkg_status.go

@@ -2,39 +2,43 @@ package deb
 
 import (
 	"bufio"
+	"errors"
 	"fmt"
 	"io"
+	"strconv"
 	"strings"
 
-	"github.com/anchore/syft/syft/cataloger/common"
 	"github.com/anchore/syft/syft/pkg"
 	"github.com/mitchellh/mapstructure"
 )
 
-// integrity check
-var _ common.ParserFn = parseDpkgStatus
-
 var errEndOfPackages = fmt.Errorf("no more packages to read")
 
 // parseDpkgStatus is a parser function for Debian DB status contents, returning all Debian packages listed.
-func parseDpkgStatus(_ string, reader io.Reader) ([]pkg.Package, error) {
+func parseDpkgStatus(reader io.Reader) ([]pkg.Package, error) {
 	buffedReader := bufio.NewReader(reader)
 	var packages = make([]pkg.Package, 0)
 
-	for {
+	continueProcessing := true
+	for continueProcessing {
 		entry, err := parseDpkgStatusEntry(buffedReader)
 		if err != nil {
-			if err == errEndOfPackages {
-				break
+			if errors.Is(err, errEndOfPackages) {
+				continueProcessing = false
+			} else {
+				return nil, err
 			}
-			return nil, err
 		}
-		packages = append(packages, pkg.Package{
-			Name:     entry.Package,
-			Version:  entry.Version,
-			Type:     pkg.DebPkg,
-			Metadata: entry,
-		})
+
+		if entry.Package != "" {
+			packages = append(packages, pkg.Package{
+				Name:         entry.Package,
+				Version:      entry.Version,
+				Type:         pkg.DebPkg,
+				MetadataType: pkg.DpkgMetadataType,
+				Metadata:     entry,
+			})
+		}
 	}
 
 	return packages, nil
@@ -42,14 +46,16 @@ func parseDpkgStatus(_ string, reader io.Reader) ([]pkg.Package, error) {
 
 // parseDpkgStatusEntry returns an individual Dpkg entry, or returns errEndOfPackages if there are no more packages to parse from the reader.
 func parseDpkgStatusEntry(reader *bufio.Reader) (entry pkg.DpkgMetadata, err error) {
-	dpkgFields := make(map[string]string)
+	dpkgFields := make(map[string]interface{})
+	var retErr error
 	var key string
 
 	for {
 		line, err := reader.ReadString('\n')
 		if err != nil {
 			if err == io.EOF {
-				return pkg.DpkgMetadata{}, errEndOfPackages
+				retErr = errEndOfPackages
+				break
 			}
 			return pkg.DpkgMetadata{}, err
 		}
@@ -81,18 +87,16 @@ func parseDpkgStatusEntry(reader *bufio.Reader) (entry pkg.DpkgMetadata, err err
 			dpkgFields[key] = val
 		default:
 			// parse a new key
-			if i := strings.Index(line, ":"); i > 0 {
-				key = strings.TrimSpace(line[0:i])
-				val := strings.TrimSpace(line[i+1:])
-
-				if _, ok := dpkgFields[key]; ok {
-					return pkg.DpkgMetadata{}, fmt.Errorf("duplicate key discovered: %s", key)
-				}
-
-				dpkgFields[key] = val
-			} else {
-				return pkg.DpkgMetadata{}, fmt.Errorf("cannot parse field from line: '%s'", line)
+			var val interface{}
+			key, val, err = handleNewKeyValue(line)
+			if err != nil {
+				return pkg.DpkgMetadata{}, err
 			}
+
+			if _, ok := dpkgFields[key]; ok {
+				return pkg.DpkgMetadata{}, fmt.Errorf("duplicate key discovered: %s", key)
+			}
+			dpkgFields[key] = val
 		}
 	}
 
@@ -101,5 +105,29 @@ func parseDpkgStatusEntry(reader *bufio.Reader) (entry pkg.DpkgMetadata, err err
 		return pkg.DpkgMetadata{}, err
 	}
 
-	return entry, nil
+	return entry, retErr
+}
+
+// handleNewKeyValue parse a new key-value pair from the given unprocessed line
+func handleNewKeyValue(line string) (string, interface{}, error) {
+	if i := strings.Index(line, ":"); i > 0 {
+		var key = strings.TrimSpace(line[0:i])
+		// mapstruct cant handle "-"
+		key = strings.ReplaceAll(key, "-", "")
+		val := strings.TrimSpace(line[i+1:])
+
+		// further processing of values based on the key that was discovered
+		switch key {
+		case "InstalledSize":
+			numVal, err := strconv.Atoi(val)
+			if err != nil {
+				return "", nil, fmt.Errorf("bad installed-size value=%q: %w", val, err)
+			}
+			return key, numVal, nil
+		default:
+			return key, val, nil
+		}
+	}
+
+	return "", nil, fmt.Errorf("cannot parse field from line: '%s'", line)
 }

syft/cataloger/deb/parse_dpkg_status_test.go

@@ -24,17 +24,19 @@ func TestSinglePackage(t *testing.T) {
 		{
 			name: "Test Single Package",
 			expected: pkg.DpkgMetadata{
-				Package:      "apt",
-				Source:       "apt-dev",
-				Version:      "1.8.2",
-				Architecture: "amd64",
+				Package:       "apt",
+				Source:        "apt-dev",
+				Version:       "1.8.2",
+				Architecture:  "amd64",
+				InstalledSize: 4064,
+				Maintainer:    "APT Development Team <deity@lists.debian.org>",
 			},
 		},
 	}
 
 	for _, test := range tests {
 		t.Run(test.name, func(t *testing.T) {
-			file, err := os.Open("test-fixtures/single")
+			file, err := os.Open("test-fixtures/status/single")
 			if err != nil {
 				t.Fatal("Unable to read test_fixtures/single: ", err)
 			}
@@ -66,15 +68,19 @@ func TestMultiplePackages(t *testing.T) {
 			name: "Test Multiple Package",
 			expected: []pkg.DpkgMetadata{
 				{
-					Package:      "tzdata",
-					Version:      "2020a-0+deb10u1",
-					Source:       "tzdata-dev",
-					Architecture: "all",
+					Package:       "tzdata",
+					Version:       "2020a-0+deb10u1",
+					Source:        "tzdata-dev",
+					Architecture:  "all",
+					InstalledSize: 3036,
+					Maintainer:    "GNU Libc Maintainers <debian-glibc@lists.debian.org>",
 				},
 				{
-					Package:      "util-linux",
-					Version:      "2.33.1-0.1",
-					Architecture: "amd64",
+					Package:       "util-linux",
+					Version:       "2.33.1-0.1",
+					Architecture:  "amd64",
+					InstalledSize: 4327,
+					Maintainer:    "LaMont Jones <lamont@debian.org>",
 				},
 			},
 		},
@@ -82,7 +88,7 @@ func TestMultiplePackages(t *testing.T) {
 
 	for _, test := range tests {
 		t.Run(test.name, func(t *testing.T) {
-			file, err := os.Open("test-fixtures/multiple")
+			file, err := os.Open("test-fixtures/status/multiple")
 			if err != nil {
 				t.Fatal("Unable to read: ", err)
 			}
@@ -93,7 +99,7 @@ func TestMultiplePackages(t *testing.T) {
 				}
 			}()
 
-			pkgs, err := parseDpkgStatus(file.Name(), file)
+			pkgs, err := parseDpkgStatus(file)
 			if err != nil {
 				t.Fatal("Unable to read file contents: ", err)
 			}

syft/cataloger/deb/test-fixtures/copyright/libaudit-common

@@ -0,0 +1,43 @@
+Format: http://www.debian.org/doc/packaging-manuals/copyright-format/1.0/
+Upstream-Name: auditd
+Source: https://people.redhat.com/sgrubb/audit/
+
+Files: *
+Copyright: 2012-2016 Steve Grubb <sgrubb@redhat.com>
+           2006-2012 Rik Faith
+License: GPL-2
+
+Files: src/libev/*
+Copyright: 2007-2009 Marc Alexamder Lehmann
+License: GPL-2
+
+Files: lib/*
+Copyright: 2005-2008 Steve Grubb <sgrubb@redhat.com>
+License: LGPL-2.1
+  The audit daemon's library libaudit.* is released under LGPL
+  so that it may be linked with 3rd party software.
+  .
+  On Debian systems, refer to /usr/share/common-licenses/LGPL-2.1
+  for the complete text of the GNU Lesser General Public License.
+
+Files: debian/*
+Copyright: 2007-2011 Philipp Matthias Hahn <pmhahn@debian.org>
+           2012-2016 Laurent Bigonville <bigon@debian.org>
+License: GPL-2
+
+License: GPL-2
+ This package is free software; you can redistribute it and/or modify
+ it under the terms of the GNU General Public License version 2,
+ as published by the Free Software Foundation.
+ .
+ This package is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ GNU General Public License for more details.
+ .
+ You should have received a copy of the GNU General Public License
+ along with this package; if not, write to the Free Software
+ Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301 USA
+ .
+ On Debian systems, the complete text of the GNU General
+ Public License can be found in `/usr/share/common-licenses/GPL-1'.

syft/cataloger/deb/test-fixtures/copyright/liblzma5

@@ -0,0 +1,383 @@
+Format: http://www.debian.org/doc/packaging-manuals/copyright-format/1.0/
+Upstream-Name: XZ Utils
+Upstream-Contact:
+ Lasse Collin <lasse.collin@tukaani.org>
+ http://tukaani.org/xz/lists.html
+Source:
+ http://tukaani.org/xz
+ http://git.tukaani.org/xz.git
+Comment:
+ XZ Utils is developed and maintained upstream by Lasse Collin.  Major
+ portions are based on code by other authors; see AUTHORS for details.
+ Most of the source has been put into the public domain, but some files
+ have not (details below).
+ .
+ This file describes the source package.  The binary packages contain
+ some files derived from other works: for example, images in the API
+ documentation come from Doxygen.
+License:
+ Different licenses apply to different files in this package. Here
+ is a rough summary of which licenses apply to which parts of this
+ package (but check the individual files to be sure!):
+ .
+   - liblzma is in the public domain.
+ .
+   - xz, xzdec, and lzmadec command line tools are in the public
+     domain unless GNU getopt_long had to be compiled and linked
+     in from the lib directory. The getopt_long code is under
+     GNU LGPLv2.1+.
+ .
+   - The scripts to grep, diff, and view compressed files have been
+     adapted from gzip. These scripts and their documentation are
+     under GNU GPLv2+.
+ .
+   - All the documentation in the doc directory and most of the
+     XZ Utils specific documentation files in other directories
+     are in the public domain.
+ .
+   - Translated messages are in the public domain.
+ .
+   - The build system contains public domain files, and files that
+     are under GNU GPLv2+ or GNU GPLv3+. None of these files end up
+     in the binaries being built.
+ .
+   - Test files and test code in the tests directory, and debugging
+     utilities in the debug directory are in the public domain.
+ .
+   - The extra directory may contain public domain files, and files
+     that are under various free software licenses.
+ .
+ You can do whatever you want with the files that have been put into
+ the public domain. If you find public domain legally problematic,
+ take the previous sentence as a license grant. If you still find
+ the lack of copyright legally problematic, you have too many
+ lawyers.
+ .
+ As usual, this software is provided "as is", without any warranty.
+ .
+ If you copy significant amounts of public domain code from XZ Utils
+ into your project, acknowledging this somewhere in your software is
+ polite (especially if it is proprietary, non-free software), but
+ naturally it is not legally required. Here is an example of a good
+ notice to put into "about box" or into documentation:
+ .
+     This software includes code from XZ Utils <http://tukaani.org/xz/>.
+ .
+ The following license texts are included in the following files:
+   - COPYING.LGPLv2.1: GNU Lesser General Public License version 2.1
+   - COPYING.GPLv2: GNU General Public License version 2
+   - COPYING.GPLv3: GNU General Public License version 3
+ .
+ Note that the toolchain (compiler, linker etc.) may add some code
+ pieces that are copyrighted. Thus, it is possible that e.g. liblzma
+ binary wouldn't actually be in the public domain in its entirety
+ even though it contains no copyrighted code from the XZ Utils source
+ package.
+ .
+ If you have questions, don't hesitate to ask the author(s) for more
+ information.
+
+Files: *
+Copyright: 2006-2012, Lasse Collin
+           1999-2008, Igor Pavlov
+           2006, Ville Koskinen
+           1998, Steve Reid
+           2000, Wei Dai
+           2003, Kevin Springle
+           2009, Jonathan Nieder
+           2010, Anders F Björklund
+License: PD
+ This file has been put in the public domain.
+ You can do whatever you want with this file.
+Comment:
+  From: Lasse Collin <lasse.collin@tukaani.org>
+  To: Jonathan Nieder <jrnieder@gmail.com>
+  Subject: Re: XZ utils for Debian
+  Date: Sun, 19 Jul 2009 13:28:23 +0300
+  Message-Id: <200907191328.23816.lasse.collin@tukaani.org>
+ .
+ [...]
+ .
+  > AUTHORS, ChangeLog, COPYING, README, THANKS, TODO,
+  > dos/README, windows/README
+ .
+  COPYING says that most docs are in the public domain. Maybe that's not
+  clear enough, but on the other hand it looks a bit stupid to put
+  copyright information in tiny and relatively small docs like README.
+ .
+  I don't dare to say that _all_ XZ Utils specific docs are in the public
+  domain unless otherwise mentioned in the file. I'm including PDF files
+  generated by groff + ps2pdf, and some day I might include Doxygen-
+  generated HTML docs too. Those don't include any copyright notices, but
+  it seems likely that groff + ps2pdf or at least Doxygen put some
+  copyrighted content into the generated files.
+
+Files: INSTALL NEWS PACKAGERS
+ windows/README-Windows.txt
+ windows/INSTALL-Windows.txt
+Copyright: 2009-2010, Lasse Collin
+License: probably-PD
+ See the note on AUTHORS, README, and so on above.
+
+Files: src/scripts/* lib/* extra/scanlzma/scanlzma.c
+Copyright: © 1993, Jean-loup Gailly
+           © 1989-1994, 1996-1999, 2001-2007, Free Software Foundation, Inc.
+           © 2006 Timo Lindfors
+           2005, Charles Levert
+           2005, 2009, Lasse Collin
+           2009, Andrew Dudman
+Other-Authors: Paul Eggert, Ulrich Drepper
+License: GPL-2+
+
+Files: src/scripts/Makefile.am src/scripts/xzless.1
+Copyright: 2009, Andrew Dudman
+           2009, Lasse Collin
+License: PD
+ This file has been put in the public domain.
+ You can do whatever you want with this file.
+
+Files: doc/examples/xz_pipe_comp.c doc/examples/xz_pipe_decomp.c
+Copyright: 2010, Daniel Mealha Cabrita
+License: PD
+ Not copyrighted -- provided to the public domain.
+
+Files: lib/getopt.c lib/getopt1.c lib/getopt.in.h
+Copyright: © 1987-2007 Free Software Foundation, Inc.
+Other-Authors: Ulrich Drepper
+License: LGPL-2.1+
+
+Files: m4/getopt.m4 m4/posix-shell.m4
+Copyright: © 2002-2006, 2008 Free Software Foundation, Inc.
+           © 2007-2008 Free Software Foundation, Inc.
+Other-Authors: Bruno Haible, Paul Eggert
+License: permissive-fsf
+
+Files: m4/acx_pthread.m4
+Copyright: © 2008, Steven G. Johnson <stevenj@alum.mit.edu>
+License: Autoconf
+
+Files: Doxyfile.in
+Copyright: © 1997-2007 by Dimitri van Heesch
+Origin: Doxygen 1.4.7
+License: GPL-2
+
+Files: src/liblzma/check/crc32_table_?e.h
+ src/liblzma/check/crc64_table_?e.h
+ src/liblzma/lzma/fastpos_table.c
+ src/liblzma/rangecoder/price_table.c
+Copyright: none, automatically generated data
+Generated-With:
+ src/liblzma/check/crc32_tablegen.c
+ src/liblzma/check/crc64_tablegen.c
+ src/liblzma/lzma/fastpos_tablegen.c
+ src/liblzma/rangecoder/price_tablegen.c
+License: none
+ No copyright to license.
+
+Files: .gitignore m4/.gitignore po/.gitignore po/LINGUAS po/POTFILES.in
+Copyright: none; these are just short lists.
+License: none
+ No copyright to license.
+
+Files: tests/compress_prepared_bcj_*
+Copyright: 2008-2009, Lasse Collin
+Source-Code: tests/bcj_test.c
+License: PD
+ This file has been put into the public domain.
+ You can do whatever you want with this file.
+Comment:
+ changelog.gz (commit 975d8fd) explains:
+ .
+ Recreated the BCJ test files for x86 and SPARC. The old files
+ were linked with crt*.o, which are copyrighted, and thus the
+ old test files were not in the public domain as a whole. They
+ are freely distributable though, but it is better to be careful
+ and avoid including any copyrighted pieces in the test files.
+ The new files are just compiled and assembled object files,
+ and thus don't contain any copyrighted code.
+
+Files: po/cs.po po/de.po po/fr.po
+Copyright: 2010, Marek Černocký
+           2010, Andre Noll
+           2011, Adrien Nader
+License: PD
+ This file is put in the public domain.
+
+Files: po/it.po po/pl.po
+Copyright: 2009, 2010, Gruppo traduzione italiano di Ubuntu-it
+           2010, Lorenzo De Liso
+           2009, 2010, 2011, Milo Casagrande
+           2011, Jakub Bogusz
+License: PD
+ This file is in the public domain
+
+Files: INSTALL.generic
+Copyright: © 1994, 1995, 1996, 1999, 2000, 2001, 2002, 2004, 2005,
+             2006, 2007, 2008, 2009, 2010 Free Software Foundation, Inc.
+License: permissive-nowarranty
+
+Files: dos/config.h
+Copyright: © 1992, 1993, 1994, 1999, 2000, 2001, 2002, 2005
+             Free Software Foundation, Inc.
+           2007-2010, Lasse Collin
+Other-Authors: Roland McGrath, Akim Demaille, Paul Eggert,
+               David Mackenzie, Bruno Haible, and many others.
+Origin: configure.ac from XZ Utils,
+        visibility.m4 serial 1 (gettext-0.15),
+        Autoconf 2.52g
+License: config-h
+ configure.ac:
+ .
+  # Author: Lasse Collin
+  #
+  # This file has been put into the public domain.
+  # You can do whatever you want with this file.
+ .
+ visibility.m4:
+ .
+  dnl Copyright (C) 2005 Free Software Foundation, Inc.
+  dnl This file is free software; the Free Software Foundation
+  dnl gives unlimited permission to copy and/or distribute it,
+  dnl with or without modifications, as long as this notice is preserved.
+ .
+ dnl From Bruno Haible.
+ .
+ comments from Autoconf 2.52g:
+ .
+  # Copyright 1992, 1993, 1994, 1999, 2000, 2001, 2002
+  # Free Software Foundation, Inc.
+ .
+ [...]
+ .
+  # As a special exception, the Free Software Foundation gives unlimited
+  # permission to copy, distribute and modify the configure scripts that
+  # are the output of Autoconf.  You need not follow the terms of the GNU
+  # General Public License when using or distributing such scripts, even
+  # though portions of the text of Autoconf appear in them.  The GNU
+  # General Public License (GPL) does govern all other use of the material
+  # that constitutes the Autoconf program.
+ .
+ On Debian systems, the complete text of the GNU General Public
+ License version 2 can be found in ‘/usr/share/common-licenses/GPL-2’.
+ dos/config.h was generated with autoheader, which tells Autoconf to
+ output a script to generate a config.h file and then runs it.
+
+Files: po/Makevars
+Origin: gettext-runtime/po/Makevars (gettext-0.12)
+Copyright: © 2003 Free Software Foundation, Inc.
+Authors: Bruno Haible
+License: LGPL-2.1+
+ The gettext-runtime package is under the LGPL, see files intl/COPYING.LIB-2.0
+ and intl/COPYING.LIB-2.1.
+ .
+ On Debian systems, the complete text of intl/COPYING.LIB-2.0 from
+ gettext-runtime 0.12 can be found in ‘/usr/share/common-licenses/LGPL-2’
+ and the text of intl/COPYING.LIB-2.1 can be found in
+ ‘/usr/share/common-licenses/LGPL-2.1’.
+ .
+ po/Makevars consists mostly of helpful comments and does not contain a
+ copyright and license notice.
+
+Files: COPYING.GPLv2 COPYING.GPLv3 COPYING.LGPLv2.1
+Copyright: © 1989, 1991, 1999, 2007 Free Software Foundation, Inc.
+License: noderivs
+ Everyone is permitted to copy and distribute verbatim copies
+ of this license document, but changing it is not allowed.
+
+Files: debian/*
+Copyright: 2009-2012, Jonathan Nieder
+License: PD-debian
+ The Debian packaging files are in the public domain.
+ You may freely use, modify, distribute, and relicense them.
+
+License: LGPL-2.1+
+ This program is free software; you can redistribute it and/or modify
+ it under the terms of the GNU Lesser General Public License as published by
+ the Free Software Foundation; either version 2.1, or (at your option)
+ any later version.
+ .
+ This program is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ GNU Lesser General Public License for more details.
+ .
+ You should have received a copy of the GNU Lesser General Public License along
+ with this program; if not, write to the Free Software Foundation,
+ Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+ .
+ On Debian systems, the complete text of the GNU Lesser General Public
+ License version 2.1 can be found in ‘/usr/share/common-licenses/LGPL-2.1’.
+
+License: GPL-2
+ Permission to use, copy, modify, and distribute this software and its
+ documentation under the terms of the GNU General Public License is
+ hereby granted. No representations are made about the suitability of
+ this software for any purpose. It is provided "as is" without express
+ or implied warranty. See the GNU General Public License for more
+ details.
+ .
+ Documents produced by doxygen are derivative works derived from the
+ input used in their production; they are not affected by this license.
+ .
+ On Debian systems, the complete text of the version of the GNU General
+ Public License distributed with Doxygen can be found in
+ ‘/usr/share/common-licenses/GPL-2’.
+
+License: GPL-2+
+ This program is free software; you can redistribute it and/or modify
+ it under the terms of the GNU General Public License as published by
+ the Free Software Foundation; either version 2, or (at your option)
+ any later version.
+ .
+ This program is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ GNU General Public License for more details.
+ .
+ You should have received a copy of the GNU General Public License along
+ with this program; if not, write to the Free Software Foundation,
+ Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+ .
+ On Debian systems, the complete text of the GNU General Public License
+ version 2 can be found in ‘/usr/share/common-licenses/GPL-2’.
+
+License: Autoconf
+ This program is free software: you can redistribute it and/or modify it
+ under the terms of the GNU General Public License as published by the
+ Free Software Foundation, either version 3 of the License, or (at your
+ option) any later version.
+ .
+ This program is distributed in the hope that it will be useful, but
+ WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General
+ Public License for more details.
+ .
+ You should have received a copy of the GNU General Public License along
+ with this program. If not, see <http://www.gnu.org/licenses/>.
+ .
+ As a special exception, the respective Autoconf Macro's copyright owner
+ gives unlimited permission to copy, distribute and modify the configure
+ scripts that are the output of Autoconf when processing the Macro. You
+ need not follow the terms of the GNU General Public License when using
+ or distributing such scripts, even though portions of the text of the
+ Macro appear in them. The GNU General Public License (GPL) does govern
+ all other use of the material that constitutes the Autoconf Macro.
+ .
+ This special exception to the GPL applies to versions of the Autoconf
+ Macro released by the Autoconf Archive. When you make and distribute a
+ modified version of the Autoconf Macro, you may extend this special
+ exception to the GPL to apply to your modified version as well.
+ .
+ On Debian systems, the complete text of the GNU General Public
+ License version 3 can be found in ‘/usr/share/common-licenses/GPL-3’.
+
+License: permissive-fsf
+ This file is free software; the Free Software Foundation
+ gives unlimited permission to copy and/or distribute it,
+ with or without modifications, as long as this notice is preserved.
+
+License: permissive-nowarranty
+ Copying and distribution of this file, with or without modification,
+ are permitted in any medium without royalty provided the copyright
+ notice and this notice are preserved.  This file is offered as-is,
+ without warranty of any kind.

syft/cataloger/deb/test-fixtures/copyright/trilicense

@@ -0,0 +1,13 @@
+Files: src/js/editline/*
+Copyright: 1993, John Doe
+           1993, Joe Average
+License: MPL-1.1 or GPL-2 or LGPL-2.1
+
+License: MPL-1.1
+ [LICENSE TEXT]
+
+License: GPL-2
+ [LICENSE TEXT]
+
+License: LGPL-2.1
+ [LICENSE TEXT]

syft/cataloger/deb/test-fixtures/image-dpkg/Dockerfile

@@ -0,0 +1,2 @@
+FROM scratch
+COPY . .

syft/cataloger/deb/test-fixtures/image-dpkg/usr/share/doc/libpam-runtime/copyright

@@ -0,0 +1,43 @@
+Format: http://www.debian.org/doc/packaging-manuals/copyright-format/1.0/
+Upstream-Name: auditd
+Source: https://people.redhat.com/sgrubb/audit/
+
+Files: *
+Copyright: 2012-2016 Steve Grubb <sgrubb@redhat.com>
+           2006-2012 Rik Faith
+License: GPL-2
+
+Files: src/libev/*
+Copyright: 2007-2009 Marc Alexamder Lehmann
+License: GPL-2
+
+Files: lib/*
+Copyright: 2005-2008 Steve Grubb <sgrubb@redhat.com>
+License: LGPL-2.1
+  The audit daemon's library libaudit.* is released under LGPL
+  so that it may be linked with 3rd party software.
+  .
+  On Debian systems, refer to /usr/share/common-licenses/LGPL-2.1
+  for the complete text of the GNU Lesser General Public License.
+
+Files: debian/*
+Copyright: 2007-2011 Philipp Matthias Hahn <pmhahn@debian.org>
+           2012-2016 Laurent Bigonville <bigon@debian.org>
+License: GPL-2
+
+License: GPL-2
+ This package is free software; you can redistribute it and/or modify
+ it under the terms of the GNU General Public License version 2,
+ as published by the Free Software Foundation.
+ .
+ This package is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ GNU General Public License for more details.
+ .
+ You should have received a copy of the GNU General Public License
+ along with this package; if not, write to the Free Software
+ Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301 USA
+ .
+ On Debian systems, the complete text of the GNU General
+ Public License can be found in `/usr/share/common-licenses/GPL-1'.

syft/cataloger/deb/test-fixtures/image-dpkg/var/lib/dpkg/info/libpam-runtime.md5sums

@@ -0,0 +1,4 @@
+55f905631797551d4d936a34c7e73474  lib/x86_64-linux-gnu/libz.so.1.2.11
+cede84bda30d2380217f97753c8ccf3a  usr/share/doc/zlib1g/changelog.Debian.gz
+f3c9dafa6da7992c47328b4464f6d122  usr/share/doc/zlib1g/changelog.gz
+a4fae96070439a5209a62ae5b8017ab2  usr/share/doc/zlib1g/copyright

syft/cataloger/deb/test-fixtures/image-dpkg/var/lib/dpkg/status

@@ -0,0 +1,21 @@
+Package: libpam-runtime
+Status: install ok installed
+Priority: required
+Section: admin
+Installed-Size: 1016
+Maintainer: Steve Langasek <vorlon@debian.org>
+Architecture: all
+Multi-Arch: foreign
+Source: pam
+Version: 1.1.8-3.6
+Replaces: libpam0g-dev, libpam0g-util
+Depends: debconf (>= 0.5) | debconf-2.0, debconf (>= 1.5.19) | cdebconf, libpam-modules (>= 1.0.1-6)
+Conflicts: libpam0g-util
+Conffiles:
+ /etc/pam.conf 87fc76f18e98ee7d3848f6b81b3391e5
+ /etc/pam.d/other 31aa7f2181889ffb00b87df4126d1701
+Description: Runtime support for the PAM library
+ Contains configuration files and  directories required for
+ authentication  to work on Debian systems.  This package is required
+ on almost all installations.
+Homepage: http://www.linux-pam.org/

syft/cataloger/deb/test-fixtures/info/zlib1g.md5sums

@@ -0,0 +1,4 @@
+55f905631797551d4d936a34c7e73474  lib/x86_64-linux-gnu/libz.so.1.2.11
+cede84bda30d2380217f97753c8ccf3a  usr/share/doc/zlib1g/changelog.Debian.gz
+f3c9dafa6da7992c47328b4464f6d122  usr/share/doc/zlib1g/changelog.gz
+a4fae96070439a5209a62ae5b8017ab2  usr/share/doc/zlib1g/copyright

syft/cataloger/java/archive_filename.go

@@ -10,11 +10,20 @@ import (
 	"github.com/anchore/syft/syft/pkg"
 )
 
+// match on versions and anything after the version. This is used to isolate the name from the version.
+// match examples:
+// wagon-webdav-1.0.2-rc1-hudson.jar      --->     -1.0.2-rc1-hudson.jar
+// windows-remote-command-1.0.jar         --->     -1.0.jar
+// wstx-asl-1-2.jar                       --->     -1-2.jar
+// guava-rc0.jar                          --->     -rc0.jar
+var versionAreaPattern = regexp.MustCompile(`-(?P<version>(\d+\.)?(\d+\.)?(r?c?\d+)(-[a-zA-Z0-9\-.]+)*)(?P<remaining>.*)$`)
+
+// match on explicit versions. This is used for extracting version information.
 // match examples:
 //	pkg-extra-field-4.3.2-rc1 --> match(name=pkg-extra-field version=4.3.2-rc1)
 //	pkg-extra-field-4.3-rc1   --> match(name=pkg-extra-field version=4.3-rc1)
 //	pkg-extra-field-4.3       --> match(name=pkg-extra-field version=4.3)
-var versionPattern = regexp.MustCompile(`(?P<name>.+)-(?P<version>(\d+\.)?(\d+\.)?(\*|\d+)(-[a-zA-Z0-9\-\.]+)*)`)
+var versionPattern = regexp.MustCompile(`-(?P<version>(\d+\.)?(\d+\.)?(r?c?\d+)(-[a-zA-Z0-9\-.]+)*)`)
 
 type archiveFilename struct {
 	raw    string
@@ -60,7 +69,7 @@ func (a archiveFilename) pkgType() pkg.Type {
 
 func (a archiveFilename) version() string {
 	if len(a.fields) > 1 {
-		log.Errorf("discovered multiple name-version pairings from %q: %+v", a.raw, a.fields)
+		log.Warnf("discovered multiple name-version pairings from %q: %+v", a.raw, a.fields)
 		return ""
 	} else if len(a.fields) < 1 {
 		return ""
@@ -70,10 +79,8 @@ func (a archiveFilename) version() string {
 }
 
 func (a archiveFilename) name() string {
-	// there should be only one name, if there is more or less then something is wrong
-	if len(a.fields) != 1 {
-		return ""
-	}
-
-	return a.fields[0]["name"]
+	// derive the name from the archive name (no path or extension) and remove any versions found
+	basename := filepath.Base(a.raw)
+	cleaned := strings.TrimSuffix(basename, filepath.Ext(basename))
+	return versionAreaPattern.ReplaceAllString(cleaned, "")
 }

syft/cataloger/java/archive_filename_test.go

@@ -1,9 +1,10 @@
 package java
 
 import (
+	"testing"
+
 	"github.com/anchore/syft/syft/pkg"
 	"github.com/sergi/go-diff/diffmatchpatch"
-	"testing"
 )
 
 func TestExtractInfoFromJavaArchiveFilename(t *testing.T) {
@@ -56,12 +57,78 @@ func TestExtractInfoFromJavaArchiveFilename(t *testing.T) {
 			name:      "pkg-extra-field-maven",
 			ty:        pkg.JenkinsPluginPkg,
 		},
+		{
+			filename:  "/some/path-with-version-5.4.3/wagon-webdav-1.0.2-beta-2.2.3a-hudson.jar",
+			version:   "1.0.2-beta-2.2.3a-hudson",
+			extension: "jar",
+			name:      "wagon-webdav",
+			ty:        pkg.JavaPkg,
+		},
+		{
+			filename:  "/some/path-with-version-5.4.3/wagon-webdav-1.0.2-beta-2.2.3-hudson.jar",
+			version:   "1.0.2-beta-2.2.3-hudson",
+			extension: "jar",
+			name:      "wagon-webdav",
+			ty:        pkg.JavaPkg,
+		},
+		{
+			filename:  "/some/path-with-version-5.4.3/windows-remote-command-1.0.jar",
+			version:   "1.0",
+			extension: "jar",
+			name:      "windows-remote-command",
+			ty:        pkg.JavaPkg,
+		},
+		{
+			filename:  "/some/path-with-version-5.4.3/wagon-http-lightweight-1.0.5-beta-2.jar",
+			version:   "1.0.5-beta-2",
+			extension: "jar",
+			name:      "wagon-http-lightweight",
+			ty:        pkg.JavaPkg,
+		},
+		{
+			filename:  "/hudson.war:WEB-INF/lib/commons-jelly-1.1-hudson-20100305.jar",
+			version:   "1.1-hudson-20100305",
+			extension: "jar",
+			name:      "commons-jelly",
+			ty:        pkg.JavaPkg,
+		},
+		{
+			filename: "/hudson.war:WEB-INF/lib/jtidy-4aug2000r7-dev-hudson-1.jar",
+			// I don't see how we can reliably account for this case
+			//version:   "4aug2000r7-dev-hudson-1",
+			version:   "",
+			extension: "jar",
+			name:      "jtidy",
+			ty:        pkg.JavaPkg,
+		},
+		{
+			filename: "/hudson.war:WEB-INF/lib/trilead-ssh2-build212-hudson-5.jar",
+			// I don't see how we can reliably account for this case
+			//version:   "build212-hudson-5",
+			version:   "5",
+			extension: "jar",
+			// name: "trilead-ssh2",
+			name: "trilead-ssh2-build212-hudson",
+			ty:   pkg.JavaPkg,
+		},
+		{
+			filename:  "/hudson.war:WEB-INF/lib/guava-r06.jar",
+			version:   "r06",
+			extension: "jar",
+			name:      "guava",
+			ty:        pkg.JavaPkg,
+		},
 	}
 
 	for _, test := range tests {
 		t.Run(test.filename, func(t *testing.T) {
 			obj := newJavaArchiveFilename(test.filename)
 
+			ty := obj.pkgType()
+			if ty != test.ty {
+				t.Errorf("mismatched type: %+v != %v", ty, test.ty)
+			}
+
 			version := obj.version()
 			if version != test.version {
 				dmp := diffmatchpatch.New()

syft/cataloger/java/archive_parser.go

@@ -5,6 +5,8 @@ import (
 	"io"
 	"strings"
 
+	"github.com/anchore/syft/internal/log"
+
 	"github.com/anchore/syft/internal"
 	"github.com/anchore/syft/internal/file"
 	"github.com/anchore/syft/syft/cataloger/common"
@@ -64,13 +66,17 @@ func newJavaArchiveParser(virtualPath string, reader io.Reader, detectNested boo
 		return nil, cleanupFn, fmt.Errorf("unable to read files from java archive: %w", err)
 	}
 
+	// fetch the last element of the virtual path
+	virtualElements := strings.Split(virtualPath, ":")
+	currentFilepath := virtualElements[len(virtualElements)-1]
+
 	return &archiveParser{
 		discoveredPkgs: internal.NewStringSet(),
 		fileManifest:   fileManifest,
 		virtualPath:    virtualPath,
 		archivePath:    archivePath,
 		contentPath:    contentPath,
-		fileInfo:       newJavaArchiveFilename(virtualPath),
+		fileInfo:       newJavaArchiveFilename(currentFilepath),
 		detectNested:   detectNested,
 	}, cleanupFn, nil
 }
@@ -118,7 +124,7 @@ func (j *archiveParser) parse() ([]pkg.Package, error) {
 // discoverMainPackage parses the root Java manifest used as the parent package to all discovered nested packages.
 func (j *archiveParser) discoverMainPackage() (*pkg.Package, error) {
 	// search and parse java manifest files
-	manifestMatches := j.fileManifest.GlobMatch(manifestPath)
+	manifestMatches := j.fileManifest.GlobMatch(manifestGlob)
 	if len(manifestMatches) > 1 {
 		return nil, fmt.Errorf("found multiple manifests in the jar: %+v", manifestMatches)
 	} else if len(manifestMatches) == 0 {
@@ -134,24 +140,27 @@ func (j *archiveParser) discoverMainPackage() (*pkg.Package, error) {
 
 	// parse the manifest file into a rich object
 	manifestContents := contents[manifestMatches[0]]
-	manifest, err := parseJavaManifest(strings.NewReader(manifestContents))
+	manifest, err := parseJavaManifest(j.archivePath, strings.NewReader(manifestContents))
 	if err != nil {
 		return nil, fmt.Errorf("failed to parse java manifest (%s): %w", j.virtualPath, err)
 	}
 
 	return &pkg.Package{
-		Name:     selectName(manifest, j.fileInfo),
-		Version:  selectVersion(manifest, j.fileInfo),
-		Language: pkg.Java,
-		Type:     pkg.JavaPkg,
+		Name:         selectName(manifest, j.fileInfo),
+		Version:      selectVersion(manifest, j.fileInfo),
+		Language:     pkg.Java,
+		Type:         pkg.JavaPkg,
+		MetadataType: pkg.JavaMetadataType,
 		Metadata: pkg.JavaMetadata{
-			Manifest: manifest,
+			VirtualPath: j.virtualPath,
+			Manifest:    manifest,
 		},
 	}, nil
 }
 
 // discoverPkgsFromPomProperties parses Maven POM properties for a given parent package, returning all listed Java packages found and
 // associating each discovered package to the given parent package.
+// nolint:funlen,gocognit
 func (j *archiveParser) discoverPkgsFromPomProperties(parentPkg *pkg.Package) ([]pkg.Package, error) {
 	var pkgs = make([]pkg.Package, 0)
 	parentKey := uniquePkgKey(parentPkg)
@@ -173,13 +182,22 @@ func (j *archiveParser) discoverPkgsFromPomProperties(parentPkg *pkg.Package) ([
 			if propsObj.Version != "" && propsObj.ArtifactID != "" {
 				// TODO: if there is no parentPkg (no java manifest) one of these poms could be the parent. We should discover the right parent and attach the correct info accordingly to each discovered package
 
+				// keep the artifact name within the virtual path if this package does not match the parent package
+				vPathSuffix := ""
+				if !strings.HasPrefix(propsObj.ArtifactID, parentPkg.Name) {
+					vPathSuffix += ":" + propsObj.ArtifactID
+				}
+				virtualPath := j.virtualPath + vPathSuffix
+
 				// discovered props = new package
 				p := pkg.Package{
-					Name:     propsObj.ArtifactID,
-					Version:  propsObj.Version,
-					Language: pkg.Java,
-					Type:     pkg.JavaPkg,
+					Name:         propsObj.ArtifactID,
+					Version:      propsObj.Version,
+					Language:     pkg.Java,
+					Type:         pkg.JavaPkg,
+					MetadataType: pkg.JavaMetadataType,
 					Metadata: pkg.JavaMetadata{
+						VirtualPath:   virtualPath,
 						PomProperties: propsObj,
 						Parent:        parentPkg,
 					},
@@ -187,16 +205,35 @@ func (j *archiveParser) discoverPkgsFromPomProperties(parentPkg *pkg.Package) ([
 
 				pkgKey := uniquePkgKey(&p)
 
-				if !j.discoveredPkgs.Contains(pkgKey) {
-					// only keep packages we haven't seen yet
-					pkgs = append(pkgs, p)
-				} else if pkgKey == parentKey {
+				// the name/version pair matches...
+				matchesParentPkg := pkgKey == parentKey
+
+				// the virtual path matches...
+				matchesParentPkg = matchesParentPkg || parentPkg.Metadata.(pkg.JavaMetadata).VirtualPath == virtualPath
+
+				// the pom artifactId has the parent name or vice versa
+				if propsObj.ArtifactID != "" {
+					matchesParentPkg = matchesParentPkg || strings.Contains(parentPkg.Name, propsObj.ArtifactID) || strings.Contains(propsObj.ArtifactID, parentPkg.Name)
+				}
+
+				if matchesParentPkg {
 					// we've run across more information about our parent package, add this info to the parent package metadata
+					// the pom properties is typically a better source of information for name and version than the manifest
+					if p.Name != parentPkg.Name {
+						parentPkg.Name = p.Name
+					}
+					if p.Version != parentPkg.Version {
+						parentPkg.Version = p.Version
+					}
+
 					parentMetadata, ok := parentPkg.Metadata.(pkg.JavaMetadata)
 					if ok {
 						parentMetadata.PomProperties = propsObj
 						parentPkg.Metadata = parentMetadata
 					}
+				} else if !j.discoveredPkgs.Contains(pkgKey) {
+					// only keep packages we haven't seen yet (and are not related to the parent package)
+					pkgs = append(pkgs, p)
 				}
 			}
 		}
@@ -214,18 +251,28 @@ func (j *archiveParser) discoverPkgsFromNestedArchives(parentPkg *pkg.Package) (
 	}
 
 	// search and parse pom.properties files & fetch the contents
-	readers, err := file.ExtractFromZipToUniqueTempFile(j.archivePath, j.contentPath, j.fileManifest.GlobMatch(archiveFormatGlobs...)...)
+	openers, err := file.ExtractFromZipToUniqueTempFile(j.archivePath, j.contentPath, j.fileManifest.GlobMatch(archiveFormatGlobs...)...)
 	if err != nil {
 		return nil, fmt.Errorf("unable to extract files from zip: %w", err)
 	}
 
 	// discover nested artifacts
-	for archivePath, archiveReader := range readers {
-		nestedPath := fmt.Sprintf("%s:%s", j.virtualPath, archivePath)
-		nestedPkgs, err := parseJavaArchive(nestedPath, archiveReader)
+	for archivePath, archiveOpener := range openers {
+		archiveReadCloser, err := archiveOpener.Open()
 		if err != nil {
+			return nil, fmt.Errorf("unable to open archived file from tempdir: %w", err)
+		}
+		nestedPath := fmt.Sprintf("%s:%s", j.virtualPath, archivePath)
+		nestedPkgs, err := parseJavaArchive(nestedPath, archiveReadCloser)
+		if err != nil {
+			if closeErr := archiveReadCloser.Close(); closeErr != nil {
+				log.Warnf("unable to close archived file from tempdir: %+v", closeErr)
+			}
 			return nil, fmt.Errorf("unable to process nested java archive (%s): %w", archivePath, err)
 		}
+		if err = archiveReadCloser.Close(); err != nil {
+			return nil, fmt.Errorf("unable to close archived file from tempdir: %w", err)
+		}
 
 		// attach the parent package to all discovered packages that are not already associated with a java archive
 		for _, p := range nestedPkgs {

syft/cataloger/java/archive_parser_test.go

@@ -78,51 +78,6 @@ func generateJavaBuildFixture(t *testing.T, fixturePath string) {
 	}
 }
 
-func TestSelectName(t *testing.T) {
-	tests := []struct {
-		desc     string
-		manifest pkg.JavaManifest
-		archive  archiveFilename
-		expected string
-	}{
-		{
-			desc:    "name from Implementation-Title",
-			archive: archiveFilename{},
-			manifest: pkg.JavaManifest{
-				Name:      "",
-				SpecTitle: "",
-				ImplTitle: "maven-wrapper",
-			},
-			expected: "maven-wrapper",
-		},
-		{
-			desc: "Implementation-Title does not override",
-			manifest: pkg.JavaManifest{
-				Name:      "Foo",
-				SpecTitle: "",
-				ImplTitle: "maven-wrapper",
-			},
-			archive: archiveFilename{
-				fields: []map[string]string{
-					{"name": "omg"},
-				},
-			},
-			expected: "omg",
-		},
-	}
-
-	for _, test := range tests {
-		t.Run(test.desc, func(t *testing.T) {
-			result := selectName(&test.manifest, test.archive)
-
-			if result != test.expected {
-				t.Errorf("mismatch in names: '%s' != '%s'", result, test.expected)
-			}
-		})
-	}
-
-}
-
 func TestParseJar(t *testing.T) {
 	tests := []struct {
 		fixture      string
@@ -137,17 +92,20 @@ func TestParseJar(t *testing.T) {
 			},
 			expected: map[string]pkg.Package{
 				"example-jenkins-plugin": {
-					Name:     "example-jenkins-plugin",
-					Version:  "1.0-SNAPSHOT",
-					Language: pkg.Java,
-					Type:     pkg.JenkinsPluginPkg,
+					Name:         "example-jenkins-plugin",
+					Version:      "1.0-SNAPSHOT",
+					Language:     pkg.Java,
+					Type:         pkg.JenkinsPluginPkg,
+					MetadataType: pkg.JavaMetadataType,
 					Metadata: pkg.JavaMetadata{
+						VirtualPath: "test-fixtures/java-builds/packages/example-jenkins-plugin.hpi",
 						Manifest: &pkg.JavaManifest{
-							ManifestVersion: "1.0",
-							SpecTitle:       "The Jenkins Plugins Parent POM Project",
-							ImplTitle:       "example-jenkins-plugin",
-							ImplVersion:     "1.0-SNAPSHOT",
-							Extra: map[string]string{
+							Main: map[string]string{
+								"Manifest-Version":       "1.0",
+								"Specification-Title":    "The Jenkins Plugins Parent POM Project",
+								"Implementation-Title":   "example-jenkins-plugin",
+								"Implementation-Version": "1.0-SNAPSHOT",
+								// extra fields...
 								"Archiver-Version":    "Plexus Archiver",
 								"Plugin-License-Url":  "https://opensource.org/licenses/MIT",
 								"Plugin-License-Name": "MIT License",
@@ -181,13 +139,17 @@ func TestParseJar(t *testing.T) {
 			fixture: "test-fixtures/java-builds/packages/example-java-app-gradle-0.1.0.jar",
 			expected: map[string]pkg.Package{
 				"example-java-app-gradle": {
-					Name:     "example-java-app-gradle",
-					Version:  "0.1.0",
-					Language: pkg.Java,
-					Type:     pkg.JavaPkg,
+					Name:         "example-java-app-gradle",
+					Version:      "0.1.0",
+					Language:     pkg.Java,
+					Type:         pkg.JavaPkg,
+					MetadataType: pkg.JavaMetadataType,
 					Metadata: pkg.JavaMetadata{
+						VirtualPath: "test-fixtures/java-builds/packages/example-java-app-gradle-0.1.0.jar",
 						Manifest: &pkg.JavaManifest{
-							ManifestVersion: "1.0",
+							Main: map[string]string{
+								"Manifest-Version": "1.0",
+							},
 						},
 					},
 				},
@@ -200,14 +162,17 @@ func TestParseJar(t *testing.T) {
 			},
 			expected: map[string]pkg.Package{
 				"example-java-app-maven": {
-					Name:     "example-java-app-maven",
-					Version:  "0.1.0",
-					Language: pkg.Java,
-					Type:     pkg.JavaPkg,
+					Name:         "example-java-app-maven",
+					Version:      "0.1.0",
+					Language:     pkg.Java,
+					Type:         pkg.JavaPkg,
+					MetadataType: pkg.JavaMetadataType,
 					Metadata: pkg.JavaMetadata{
+						VirtualPath: "test-fixtures/java-builds/packages/example-java-app-maven-0.1.0.jar",
 						Manifest: &pkg.JavaManifest{
-							ManifestVersion: "1.0",
-							Extra: map[string]string{
+							Main: map[string]string{
+								"Manifest-Version": "1.0",
+								// extra fields...
 								"Archiver-Version": "Plexus Archiver",
 								"Created-By":       "Apache Maven 3.6.3",
 								"Built-By":         "?",
@@ -224,11 +189,15 @@ func TestParseJar(t *testing.T) {
 					},
 				},
 				"joda-time": {
-					Name:     "joda-time",
-					Version:  "2.9.2",
-					Language: pkg.Java,
-					Type:     pkg.JavaPkg,
+					Name:         "joda-time",
+					Version:      "2.9.2",
+					Language:     pkg.Java,
+					Type:         pkg.JavaPkg,
+					MetadataType: pkg.JavaMetadataType,
 					Metadata: pkg.JavaMetadata{
+						// ensure that nested packages with different names than that of the parent are appended as
+						// a suffix on the virtual path
+						VirtualPath: "test-fixtures/java-builds/packages/example-java-app-maven-0.1.0.jar:joda-time",
 						PomProperties: &pkg.PomProperties{
 							Path:       "META-INF/maven/joda-time/joda-time/pom.properties",
 							GroupID:    "joda-time",
@@ -295,11 +264,11 @@ func TestParseJar(t *testing.T) {
 				metadata := a.Metadata.(pkg.JavaMetadata)
 				metadata.Parent = nil
 
-				// ignore select fields
+				// ignore select fields (only works for the main section)
 				for _, field := range test.ignoreExtras {
-					if metadata.Manifest != nil && metadata.Manifest.Extra != nil {
-						if _, ok := metadata.Manifest.Extra[field]; ok {
-							delete(metadata.Manifest.Extra, field)
+					if metadata.Manifest != nil && metadata.Manifest.Main != nil {
+						if _, ok := metadata.Manifest.Main[field]; ok {
+							delete(metadata.Manifest.Main, field)
 						}
 					}
 				}
@@ -334,7 +303,7 @@ func TestParseNestedJar(t *testing.T) {
 				},
 				{
 					Name:    "spring-boot-starter",
-					Version: "2.2.2.RELEASE",
+					Version: "2.2.2",
 				},
 				{
 					Name:    "jul-to-slf4j",
@@ -346,7 +315,7 @@ func TestParseNestedJar(t *testing.T) {
 				},
 				{
 					Name:    "spring-boot-starter-validation",
-					Version: "2.2.2.RELEASE",
+					Version: "2.2.2",
 				},
 				{
 					Name:    "hibernate-validator",
@@ -358,7 +327,7 @@ func TestParseNestedJar(t *testing.T) {
 				},
 				{
 					Name:    "spring-expression",
-					Version: "5.2.2.RELEASE",
+					Version: "5.2.2",
 				},
 				{
 					Name:    "jakarta.validation-api",
@@ -366,11 +335,11 @@ func TestParseNestedJar(t *testing.T) {
 				},
 				{
 					Name:    "spring-web",
-					Version: "5.2.2.RELEASE",
+					Version: "5.2.2",
 				},
 				{
 					Name:    "spring-boot-starter-actuator",
-					Version: "2.2.2.RELEASE",
+					Version: "2.2.2",
 				},
 				{
 					Name:    "log4j-api",
@@ -390,23 +359,23 @@ func TestParseNestedJar(t *testing.T) {
 				},
 				{
 					Name:    "spring-aop",
-					Version: "5.2.2.RELEASE",
+					Version: "5.2.2",
 				},
 				{
 					Name:    "spring-boot-actuator-autoconfigure",
-					Version: "2.2.2.RELEASE",
+					Version: "2.2.2",
 				},
 				{
 					Name:    "spring-jcl",
-					Version: "5.2.2.RELEASE",
+					Version: "5.2.2",
 				},
 				{
 					Name:    "spring-boot",
-					Version: "2.2.2.RELEASE",
+					Version: "2.2.2",
 				},
 				{
 					Name:    "spring-boot-starter-logging",
-					Version: "2.2.2.RELEASE",
+					Version: "2.2.2",
 				},
 				{
 					Name:    "jakarta.annotation-api",
@@ -414,7 +383,7 @@ func TestParseNestedJar(t *testing.T) {
 				},
 				{
 					Name:    "spring-webmvc",
-					Version: "5.2.2.RELEASE",
+					Version: "5.2.2",
 				},
 				{
 					Name:    "HdrHistogram",
@@ -422,7 +391,7 @@ func TestParseNestedJar(t *testing.T) {
 				},
 				{
 					Name:    "spring-boot-starter-web",
-					Version: "2.2.2.RELEASE",
+					Version: "2.2.2",
 				},
 				{
 					Name:    "logback-classic",
@@ -434,7 +403,7 @@ func TestParseNestedJar(t *testing.T) {
 				},
 				{
 					Name:    "spring-boot-starter-json",
-					Version: "2.2.2.RELEASE",
+					Version: "2.2.2",
 				},
 				{
 					Name:    "jackson-databind",
@@ -450,7 +419,7 @@ func TestParseNestedJar(t *testing.T) {
 				},
 				{
 					Name:    "spring-boot-autoconfigure",
-					Version: "2.2.2.RELEASE",
+					Version: "2.2.2",
 				},
 				{
 					Name:    "jackson-datatype-jdk8",
@@ -466,11 +435,11 @@ func TestParseNestedJar(t *testing.T) {
 				},
 				{
 					Name:    "spring-beans",
-					Version: "5.2.2.RELEASE",
+					Version: "5.2.2",
 				},
 				{
 					Name:    "spring-boot-actuator",
-					Version: "2.2.2.RELEASE",
+					Version: "2.2.2",
 				},
 				{
 					Name:    "slf4j-api",
@@ -478,7 +447,7 @@ func TestParseNestedJar(t *testing.T) {
 				},
 				{
 					Name:    "spring-core",
-					Version: "5.2.2.RELEASE",
+					Version: "5.2.2",
 				},
 				{
 					Name:    "logback-core",
@@ -498,7 +467,7 @@ func TestParseNestedJar(t *testing.T) {
 				},
 				{
 					Name:    "spring-boot-starter-tomcat",
-					Version: "2.2.2.RELEASE",
+					Version: "2.2.2",
 				},
 				{
 					Name:    "classmate",
@@ -506,7 +475,7 @@ func TestParseNestedJar(t *testing.T) {
 				},
 				{
 					Name:    "spring-context",
-					Version: "5.2.2.RELEASE",
+					Version: "5.2.2",
 				},
 			},
 		},
@@ -527,7 +496,7 @@ func TestParseNestedJar(t *testing.T) {
 				t.Fatalf("failed to parse java archive: %+v", err)
 			}
 
-			nameVersionPairSet := internal.NewStringSet()
+			expectedNameVersionPairSet := internal.NewStringSet()
 
 			makeKey := func(p *pkg.Package) string {
 				if p == nil {
@@ -537,20 +506,32 @@ func TestParseNestedJar(t *testing.T) {
 			}
 
 			for _, e := range test.expected {
-				nameVersionPairSet.Add(makeKey(&e))
+				expectedNameVersionPairSet.Add(makeKey(&e))
 			}
 
-			if len(actual) != len(nameVersionPairSet) {
+			if len(actual) != len(expectedNameVersionPairSet) {
+				actualNameVersionPairSet := internal.NewStringSet()
 				for _, a := range actual {
-					t.Log("   ", a)
+					key := makeKey(&a)
+					actualNameVersionPairSet.Add(key)
+					if !expectedNameVersionPairSet.Contains(key) {
+						t.Logf("extra package: %s", a)
+					}
 				}
-				t.Fatalf("unexpected package count: %d!=%d", len(actual), len(nameVersionPairSet))
+
+				for _, key := range expectedNameVersionPairSet.ToSlice() {
+					if !actualNameVersionPairSet.Contains(key) {
+						t.Logf("missing package: %s", key)
+					}
+				}
+
+				t.Fatalf("unexpected package count: %d!=%d", len(actual), len(expectedNameVersionPairSet))
 			}
 
 			for _, a := range actual {
 				actualKey := makeKey(&a)
 
-				if !nameVersionPairSet.Contains(actualKey) {
+				if !expectedNameVersionPairSet.Contains(actualKey) {
 					t.Errorf("unexpected pkg: %q", actualKey)
 				}
 
@@ -563,7 +544,7 @@ func TestParseNestedJar(t *testing.T) {
 					if metadata.Parent == nil {
 						t.Errorf("unassigned error for pkg=%q", actualKey)
 					} else if makeKey(metadata.Parent) != "spring-boot|0.0.1-SNAPSHOT" {
-						// NB: this is a hard-coded condition to simplify the test harness
+						// NB: this is a hard-coded condition to simplify the test harness to account for https://github.com/micrometer-metrics/micrometer/issues/1785
 						if a.Name == "pcollections" {
 							if metadata.Parent.Name != "micrometer-core" {
 								t.Errorf("nested 'pcollections' pkg has wrong parent: %q", metadata.Parent.Name)

syft/cataloger/java/java_manifest.go

@@ -4,25 +4,41 @@ import (
 	"bufio"
 	"fmt"
 	"io"
+	"strconv"
 	"strings"
 
+	"github.com/anchore/syft/internal/log"
+
 	"github.com/anchore/syft/syft/pkg"
-	"github.com/mitchellh/mapstructure"
 )
 
-const manifestPath = "META-INF/MANIFEST.MF"
+const manifestGlob = "/META-INF/MANIFEST.MF"
 
-func parseJavaManifest(reader io.Reader) (*pkg.JavaManifest, error) {
+// nolint:funlen
+// parseJavaManifest takes MANIFEST.MF file content and returns sections of parsed key/value pairs.
+// For more information: https://docs.oracle.com/en/java/javase/11/docs/specs/jar/jar.html#jar-manifest
+func parseJavaManifest(path string, reader io.Reader) (*pkg.JavaManifest, error) {
 	var manifest pkg.JavaManifest
-	manifestMap := make(map[string]string)
+	sections := []map[string]string{
+		make(map[string]string),
+	}
+	currentSection := 0
 	scanner := bufio.NewScanner(reader)
 	var lastKey string
 	for scanner.Scan() {
 		line := scanner.Text()
 
-		// ignore empty lines
+		// empty lines denote section separators
 		if strings.TrimSpace(line) == "" {
+			currentSection++
+			// we don't want to allocate a new section map that wont necessarily be used, do that once there is
+			// a non-empty line to process
+
+			// do not process line continuations after this
+			lastKey = ""
 			continue
+		} else if currentSection >= len(sections) {
+			sections = append(sections, make(map[string]string))
 		}
 
 		if line[0] == ' ' {
@@ -30,7 +46,7 @@ func parseJavaManifest(reader io.Reader) (*pkg.JavaManifest, error) {
 			if lastKey == "" {
 				return nil, fmt.Errorf("found continuation with no previous key (%s)", line)
 			}
-			manifestMap[lastKey] += strings.TrimSpace(line)
+			sections[currentSection][lastKey] += strings.TrimSpace(line)
 		} else {
 			// this is a new key-value pair
 			idx := strings.Index(line, ":")
@@ -40,9 +56,9 @@ func parseJavaManifest(reader io.Reader) (*pkg.JavaManifest, error) {
 
 			key := strings.TrimSpace(line[0:idx])
 			value := strings.TrimSpace(line[idx+1:])
-			manifestMap[key] = value
+			sections[currentSection][key] = value
 
-			// keep track of key for future continuations
+			// keep track of key for potential future continuations
 			lastKey = key
 		}
 	}
@@ -51,14 +67,24 @@ func parseJavaManifest(reader io.Reader) (*pkg.JavaManifest, error) {
 		return nil, fmt.Errorf("unable to read java manifest: %w", err)
 	}
 
-	if err := mapstructure.Decode(manifestMap, &manifest); err != nil {
-		return nil, fmt.Errorf("unable to parse java manifest: %w", err)
-	}
-
-	// clean select fields
-	if strings.Trim(manifest.ImplVersion, " ") != "" {
-		// transform versions with dates attached to just versions (e.g. "1.3 2244 October 5 2008" --> "1.3")
-		manifest.ImplVersion = strings.Split(manifest.ImplVersion, " ")[0]
+	if len(sections) > 0 {
+		manifest.Main = sections[0]
+		if len(sections) > 1 {
+			manifest.NamedSections = make(map[string]map[string]string)
+			for i, s := range sections[1:] {
+				name, ok := s["Name"]
+				if !ok {
+					// per the manifest spec (https://docs.oracle.com/en/java/javase/11/docs/specs/jar/jar.html#jar-manifest)
+					// this should never happen. If it does we want to know about it, but not necessarily stop
+					// cataloging entirely... for this reason we only log.
+					log.Warnf("java manifest section found without a name: %s", path)
+					name = strconv.Itoa(i)
+				} else {
+					delete(s, "Name")
+				}
+				manifest.NamedSections[name] = s
+			}
+		}
 	}
 
 	return &manifest, nil
@@ -69,24 +95,21 @@ func selectName(manifest *pkg.JavaManifest, filenameObj archiveFilename) string
 	switch {
 	case filenameObj.name() != "":
 		name = filenameObj.name()
-	case manifest.Name != "":
+	case manifest.Main["Name"] != "":
 		// Manifest original spec...
-		name = manifest.Name
-	case manifest.Extra["Bundle-Name"] != "":
+		name = manifest.Main["Name"]
+	case manifest.Main["Bundle-Name"] != "":
 		// BND tooling...
-		name = manifest.Extra["Bundle-Name"]
-	case manifest.Extra["Short-Name"] != "":
+		name = manifest.Main["Bundle-Name"]
+	case manifest.Main["Short-Name"] != "":
 		// Jenkins...
-		name = manifest.Extra["Short-Name"]
-	case manifest.Extra["Extension-Name"] != "":
+		name = manifest.Main["Short-Name"]
+	case manifest.Main["Extension-Name"] != "":
 		// Jenkins...
-		name = manifest.Extra["Extension-Name"]
-	}
-
-	// in situations where we hit this point and no name was
-	// determined, look at the Implementation-Title
-	if name == "" && manifest.ImplTitle != "" {
-		name = manifest.ImplTitle
+		name = manifest.Main["Extension-Name"]
+	case manifest.Main["Implementation-Title"] != "":
+		// last ditch effort...
+		name = manifest.Main["Implementation-Title"]
 	}
 	return name
 }
@@ -94,14 +117,14 @@ func selectName(manifest *pkg.JavaManifest, filenameObj archiveFilename) string
 func selectVersion(manifest *pkg.JavaManifest, filenameObj archiveFilename) string {
 	var version string
 	switch {
-	case manifest.ImplVersion != "":
-		version = manifest.ImplVersion
 	case filenameObj.version() != "":
 		version = filenameObj.version()
-	case manifest.SpecVersion != "":
-		version = manifest.SpecVersion
-	case manifest.Extra["Plugin-Version"] != "":
-		version = manifest.Extra["Plugin-Version"]
+	case manifest.Main["Implementation-Version"] != "":
+		version = manifest.Main["Implementation-Version"]
+	case manifest.Main["Specification-Version"] != "":
+		version = manifest.Main["Specification-Version"]
+	case manifest.Main["Plugin-Version"] != "":
+		version = manifest.Main["Plugin-Version"]
 	}
 	return version
 }

syft/cataloger/java/java_manifest_test.go

@@ -2,10 +2,11 @@ package java
 
 import (
 	"encoding/json"
-	"github.com/anchore/syft/syft/pkg"
-	"github.com/go-test/deep"
 	"os"
 	"testing"
+
+	"github.com/anchore/syft/syft/pkg"
+	"github.com/go-test/deep"
 )
 
 func TestParseJavaManifest(t *testing.T) {
@@ -16,49 +17,62 @@ func TestParseJavaManifest(t *testing.T) {
 		{
 			fixture: "test-fixtures/manifest/small",
 			expected: pkg.JavaManifest{
-				ManifestVersion: "1.0",
+				Main: map[string]string{
+					"Manifest-Version": "1.0",
+				},
 			},
 		},
 		{
 			fixture: "test-fixtures/manifest/standard-info",
 			expected: pkg.JavaManifest{
-				ManifestVersion: "1.0",
-				Name:            "the-best-name",
-				SpecTitle:       "the-spec-title",
-				SpecVersion:     "the-spec-version",
-				SpecVendor:      "the-spec-vendor",
-				ImplTitle:       "the-impl-title",
-				ImplVersion:     "the-impl-version",
-				ImplVendor:      "the-impl-vendor",
+				Main: map[string]string{
+					"Name":                   "the-best-name",
+					"Manifest-Version":       "1.0",
+					"Specification-Title":    "the-spec-title",
+					"Specification-Version":  "the-spec-version",
+					"Specification-Vendor":   "the-spec-vendor",
+					"Implementation-Title":   "the-impl-title",
+					"Implementation-Version": "the-impl-version",
+					"Implementation-Vendor":  "the-impl-vendor",
+				},
 			},
 		},
 		{
 			fixture: "test-fixtures/manifest/extra-info",
 			expected: pkg.JavaManifest{
-				ManifestVersion: "1.0",
-				Extra: map[string]string{
+				Main: map[string]string{
+					"Manifest-Version": "1.0",
 					"Archiver-Version": "Plexus Archiver",
-					"Build-Jdk":        "14.0.1",
-					"Built-By":         "?",
 					"Created-By":       "Apache Maven 3.6.3",
-					"Main-Class":       "hello.HelloWorld",
+				},
+				NamedSections: map[string]map[string]string{
+					"thing-1": {
+						"Built-By": "?",
+					},
+					"1": {
+						"Build-Jdk":  "14.0.1",
+						"Main-Class": "hello.HelloWorld",
+					},
 				},
 			},
 		},
 		{
 			fixture: "test-fixtures/manifest/continuation",
 			expected: pkg.JavaManifest{
-				ManifestVersion: "1.0",
-				Extra: map[string]string{
-					"Plugin-ScmUrl": "https://github.com/jenkinsci/plugin-pom/example-jenkins-plugin",
+				Main: map[string]string{
+					"Manifest-Version": "1.0",
+					"Plugin-ScmUrl":    "https://github.com/jenkinsci/plugin-pom/example-jenkins-plugin",
 				},
 			},
 		},
 		{
+			// regression test, we should always keep the full version
 			fixture: "test-fixtures/manifest/version-with-date",
 			expected: pkg.JavaManifest{
-				ManifestVersion: "1.0",
-				ImplVersion:     "1.3", // ensure the date is stripped off during processing
+				Main: map[string]string{
+					"Manifest-Version":       "1.0",
+					"Implementation-Version": "1.3 2244 October 5 2005",
+				},
 			},
 		},
 	}
@@ -70,7 +84,7 @@ func TestParseJavaManifest(t *testing.T) {
 				t.Fatalf("could not open fixture: %+v", err)
 			}
 
-			actual, err := parseJavaManifest(fixture)
+			actual, err := parseJavaManifest(test.fixture, fixture)
 			if err != nil {
 				t.Fatalf("failed to parse manifest: %+v", err)
 			}
@@ -91,3 +105,45 @@ func TestParseJavaManifest(t *testing.T) {
 		})
 	}
 }
+
+func TestSelectName(t *testing.T) {
+	tests := []struct {
+		desc     string
+		manifest pkg.JavaManifest
+		archive  archiveFilename
+		expected string
+	}{
+		{
+			desc:    "Get name from Implementation-Title",
+			archive: archiveFilename{},
+			manifest: pkg.JavaManifest{
+				Main: map[string]string{
+					"Implementation-Title": "maven-wrapper",
+				},
+			},
+			expected: "maven-wrapper",
+		},
+		{
+			desc: "Implementation-Title does not override name from filename",
+			manifest: pkg.JavaManifest{
+				Main: map[string]string{
+					"Name":                 "foo",
+					"Implementation-Title": "maven-wrapper",
+				},
+			},
+			archive:  newJavaArchiveFilename("/something/omg.jar"),
+			expected: "omg",
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.desc, func(t *testing.T) {
+			result := selectName(&test.manifest, test.archive)
+
+			if result != test.expected {
+				t.Errorf("mismatch in names: '%s' != '%s'", result, test.expected)
+			}
+		})
+	}
+
+}

syft/cataloger/java/save_archive_to_tmp.go

@@ -35,6 +35,7 @@ func saveArchiveToTmp(reader io.Reader) (string, string, func(), error) {
 	if err != nil {
 		return contentDir, "", cleanupFn, fmt.Errorf("unable to create archive: %w", err)
 	}
+	defer archiveFile.Close()
 
 	_, err = io.Copy(archiveFile, reader)
 	if err != nil {

syft/cataloger/java/test-fixtures/manifest/extra-info

@@ -2,6 +2,7 @@ Manifest-Version: 1.0
 Archiver-Version: Plexus Archiver
 Created-By: Apache Maven 3.6.3
 
+Name: thing-1
 Built-By: ?
 
 Build-Jdk: 14.0.1

syft/cataloger/javascript/cataloger.go

@@ -7,12 +7,21 @@ import (
 	"github.com/anchore/syft/syft/cataloger/common"
 )
 
-// NewJavascriptCataloger returns a new JavaScript cataloger object.
-func NewJavascriptCataloger() *common.GenericCataloger {
+// NewJavascriptPackageCataloger returns a new JavaScript cataloger object based on detection of npm based packages.
+func NewJavascriptPackageCataloger() *common.GenericCataloger {
+	globParsers := map[string]common.ParserFn{
+		"**/package.json": parsePackageJSON,
+	}
+
+	return common.NewGenericCataloger(nil, globParsers, "javascript-package-cataloger")
+}
+
+// NewJavascriptLockCataloger returns a new Javascript cataloger object base on package lock files.
+func NewJavascriptLockCataloger() *common.GenericCataloger {
 	globParsers := map[string]common.ParserFn{
 		"**/package-lock.json": parsePackageLock,
 		"**/yarn.lock":         parseYarnLock,
 	}
 
-	return common.NewGenericCataloger(nil, globParsers, "javascript-cataloger")
+	return common.NewGenericCataloger(nil, globParsers, "javascript-lock-cataloger")
 }

syft/cataloger/javascript/parse_package_json.go

@@ -0,0 +1,140 @@
+package javascript
+
+import (
+	"encoding/json"
+	"fmt"
+	"io"
+	"regexp"
+
+	"github.com/anchore/syft/internal"
+
+	"github.com/mitchellh/mapstructure"
+
+	"github.com/anchore/syft/syft/cataloger/common"
+	"github.com/anchore/syft/syft/pkg"
+)
+
+// integrity check
+var _ common.ParserFn = parsePackageLock
+
+// PackageJSON represents a JavaScript package.json file
+type PackageJSON struct {
+	Version      string            `json:"version"`
+	Latest       []string          `json:"latest"`
+	Author       Author            `json:"author"`
+	License      string            `json:"license"`
+	Name         string            `json:"name"`
+	Homepage     string            `json:"homepage"`
+	Description  string            `json:"description"`
+	Dependencies map[string]string `json:"dependencies"`
+	Repository   Repository        `json:"repository"`
+}
+
+type Author struct {
+	Name  string `json:"name" mapstruct:"name"`
+	Email string `json:"email" mapstruct:"email"`
+	URL   string `json:"url" mapstruct:"url"`
+}
+
+type Repository struct {
+	Type string `json:"type" mapstructure:"type"`
+	URL  string `json:"url" mapstructure:"url"`
+}
+
+// match example: "author": "Isaac Z. Schlueter <i@izs.me> (http://blog.izs.me)"
+// ---> name: "Isaac Z. Schlueter" email: "i@izs.me" url: "http://blog.izs.me"
+var authorPattern = regexp.MustCompile(`^\s*(?P<name>[^<(]*)(\s+<(?P<email>.*)>)?(\s\((?P<url>.*)\))?\s*$`)
+
+// Exports Author.UnmarshalJSON interface to help normalize the json structure.
+func (a *Author) UnmarshalJSON(b []byte) error {
+	var authorStr string
+	var fields map[string]string
+	var author Author
+
+	if err := json.Unmarshal(b, &authorStr); err != nil {
+		// string parsing did not work, assume a map was given
+		// for more information: https://docs.npmjs.com/files/package.json#people-fields-author-contributors
+		if err := json.Unmarshal(b, &fields); err != nil {
+			return fmt.Errorf("unable to parse package.json author: %w", err)
+		}
+	} else {
+		// parse out "name <email> (url)" into an Author struct
+		fields = internal.MatchCaptureGroups(authorPattern, authorStr)
+	}
+
+	// translate the map into a structure
+	if err := mapstructure.Decode(fields, &author); err != nil {
+		return fmt.Errorf("unable to decode package.json author: %w", err)
+	}
+
+	*a = author
+
+	return nil
+}
+
+func (a *Author) AuthorString() string {
+	result := a.Name
+	if a.Email != "" {
+		result += fmt.Sprintf(" <%s>", a.Email)
+	}
+	if a.URL != "" {
+		result += fmt.Sprintf(" (%s)", a.URL)
+	}
+	return result
+}
+
+func (r *Repository) UnmarshalJSON(b []byte) error {
+	var repositoryStr string
+	var fields map[string]string
+	var repository Repository
+
+	if err := json.Unmarshal(b, &repositoryStr); err != nil {
+		// string parsing did not work, assume a map was given
+		// for more information: https://docs.npmjs.com/files/package.json#people-fields-author-contributors
+		if err := json.Unmarshal(b, &fields); err != nil {
+			return fmt.Errorf("unable to parse package.json author: %w", err)
+		}
+		// translate the map into a structure
+		if err := mapstructure.Decode(fields, &repository); err != nil {
+			return fmt.Errorf("unable to decode package.json author: %w", err)
+		}
+
+		*r = repository
+	} else {
+		r.URL = repositoryStr
+	}
+
+	return nil
+}
+
+// parsePackageJson parses a package.json and returns the discovered JavaScript packages.
+func parsePackageJSON(_ string, reader io.Reader) ([]pkg.Package, error) {
+	packages := make([]pkg.Package, 0)
+	dec := json.NewDecoder(reader)
+
+	for {
+		var p PackageJSON
+		if err := dec.Decode(&p); err == io.EOF {
+			break
+		} else if err != nil {
+			return nil, fmt.Errorf("failed to parse package.json file: %w", err)
+		}
+
+		packages = append(packages, pkg.Package{
+			Name:         p.Name,
+			Version:      p.Version,
+			Licenses:     []string{p.License},
+			Language:     pkg.JavaScript,
+			Type:         pkg.NpmPkg,
+			MetadataType: pkg.NpmPackageJSONMetadataType,
+			Metadata: pkg.NpmPackageJSONMetadata{
+				Author:   p.Author.AuthorString(),
+				Homepage: p.Homepage,
+				URL:      p.Repository.URL,
+				Licenses: []string{p.License},
+			},
+		})
+	}
+
+	return packages, nil
+}

syft/cataloger/javascript/parse_package_json_test.go

@@ -0,0 +1,93 @@
+package javascript
+
+import (
+	"os"
+	"testing"
+
+	"github.com/anchore/syft/syft/pkg"
+	"github.com/go-test/deep"
+)
+
+func TestParsePackageJSON(t *testing.T) {
+	tests := []struct {
+		Fixture     string
+		ExpectedPkg pkg.Package
+	}{
+		{
+			Fixture: "test-fixtures/pkg-json/package.json",
+			ExpectedPkg: pkg.Package{
+				Name:         "npm",
+				Version:      "6.14.6",
+				Type:         pkg.NpmPkg,
+				Licenses:     []string{"Artistic-2.0"},
+				Language:     pkg.JavaScript,
+				MetadataType: pkg.NpmPackageJSONMetadataType,
+				Metadata: pkg.NpmPackageJSONMetadata{
+					Author:   "Isaac Z. Schlueter <i@izs.me> (http://blog.izs.me)",
+					Homepage: "https://docs.npmjs.com/",
+					URL:      "https://github.com/npm/cli",
+					Licenses: []string{"Artistic-2.0"},
+				},
+			},
+		},
+		{
+			Fixture: "test-fixtures/pkg-json/package-nested-author.json",
+			ExpectedPkg: pkg.Package{
+				Name:         "npm",
+				Version:      "6.14.6",
+				Type:         pkg.NpmPkg,
+				Licenses:     []string{"Artistic-2.0"},
+				Language:     pkg.JavaScript,
+				MetadataType: pkg.NpmPackageJSONMetadataType,
+				Metadata: pkg.NpmPackageJSONMetadata{
+					Author:   "Isaac Z. Schlueter <i@izs.me> (http://blog.izs.me)",
+					Homepage: "https://docs.npmjs.com/",
+					URL:      "https://github.com/npm/cli",
+					Licenses: []string{"Artistic-2.0"},
+				},
+			},
+		},
+		{
+			Fixture: "test-fixtures/pkg-json/package-repo-string.json",
+			ExpectedPkg: pkg.Package{
+				Name:         "function-bind",
+				Version:      "1.1.1",
+				Type:         pkg.NpmPkg,
+				Licenses:     []string{"MIT"},
+				Language:     pkg.JavaScript,
+				MetadataType: pkg.NpmPackageJSONMetadataType,
+				Metadata: pkg.NpmPackageJSONMetadata{
+					Author:   "Raynos <raynos2@gmail.com>",
+					Homepage: "https://github.com/Raynos/function-bind",
+					URL:      "git://github.com/Raynos/function-bind.git",
+					Licenses: []string{"MIT"},
+				},
+			},
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.Fixture, func(t *testing.T) {
+			fixture, err := os.Open(test.Fixture)
+			if err != nil {
+				t.Fatalf("failed to open fixture: %+v", err)
+			}
+
+			actual, err := parsePackageJSON(fixture.Name(), fixture)
+			if err != nil {
+				t.Fatalf("failed to parse package-lock.json: %+v", err)
+			}
+			if len(actual) != 1 {
+				for _, a := range actual {
+					t.Log("   ", a)
+				}
+				t.Fatalf("unexpected package count: %d!=1", len(actual))
+			}
+
+			for _, d := range deep.Equal(actual[0], test.ExpectedPkg) {
+
+				t.Errorf("diff: %+v", d)
+			}
+		})
+	}
+}

syft/cataloger/javascript/parse_yarn_lock.go

@@ -40,7 +40,7 @@ func parseYarnLock(_ string, reader io.Reader) ([]pkg.Package, error) {
 		case composedNameExp.MatchString(line):
 			name := composedNameExp.FindString(line)
 			if len(name) == 0 {
-				log.Errorf("unable to parse line: '%s'", line)
+				log.Warnf("unable to parse yarn.lock line: %q", line)
 			}
 			currentName = strings.TrimLeft(name, "\"")
 		case simpleNameExp.MatchString(line):
@@ -66,7 +66,7 @@ func parseYarnLock(_ string, reader io.Reader) ([]pkg.Package, error) {
 				Name:     currentName,
 				Version:  strings.Trim(version, "\""),
 				Language: pkg.JavaScript,
-				Type:     pkg.YarnPkg,
+				Type:     pkg.NpmPkg,
 			})
 		}
 	}

syft/cataloger/javascript/parse_yarn_lock_test.go

@@ -13,43 +13,43 @@ func TestParseYarnLock(t *testing.T) {
 			Name:     "@babel/code-frame",
 			Version:  "7.10.4",
 			Language: pkg.JavaScript,
-			Type:     pkg.YarnPkg,
+			Type:     pkg.NpmPkg,
 		},
 		"@types/minimatch": {
 			Name:     "@types/minimatch",
 			Version:  "3.0.3",
 			Language: pkg.JavaScript,
-			Type:     pkg.YarnPkg,
+			Type:     pkg.NpmPkg,
 		},
 		"@types/qs": {
 			Name:     "@types/qs",
 			Version:  "6.9.4",
 			Language: pkg.JavaScript,
-			Type:     pkg.YarnPkg,
+			Type:     pkg.NpmPkg,
 		},
 		"ajv": {
 			Name:     "ajv",
 			Version:  "6.12.3",
 			Language: pkg.JavaScript,
-			Type:     pkg.YarnPkg,
+			Type:     pkg.NpmPkg,
 		},
 		"atob": {
 			Name:     "atob",
 			Version:  "2.1.2",
 			Language: pkg.JavaScript,
-			Type:     pkg.YarnPkg,
+			Type:     pkg.NpmPkg,
 		},
 		"aws-sdk": {
 			Name:     "aws-sdk",
 			Version:  "2.706.0",
 			Language: pkg.JavaScript,
-			Type:     pkg.YarnPkg,
+			Type:     pkg.NpmPkg,
 		},
 		"jhipster-core": {
 			Name:     "jhipster-core",
 			Version:  "7.3.4",
 			Language: pkg.JavaScript,
-			Type:     pkg.YarnPkg,
+			Type:     pkg.NpmPkg,
 		},
 	}
 	fixture, err := os.Open("test-fixtures/yarn/yarn.lock")

syft/cataloger/javascript/test-fixtures/pkg-json/package-nested-author.json

@@ -0,0 +1,16 @@
+{
+  "version": "6.14.6",
+  "name": "npm",
+  "description": "a package manager for JavaScript",
+  "homepage": "https://docs.npmjs.com/",
+  "author": {
+    "name": "Isaac Z. Schlueter",
+    "email": "i@izs.me",
+    "url": "http://blog.izs.me"
+  },
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/npm/cli"
+  },
+  "license": "Artistic-2.0"
+}

syft/cataloger/javascript/test-fixtures/pkg-json/package-repo-string.json

@@ -0,0 +1,66 @@
+{
+  "name": "function-bind",
+  "version": "1.1.1",
+  "description": "Implementation of Function.prototype.bind",
+  "keywords": [
+    "function",
+    "bind",
+    "shim",
+    "es5"
+  ],
+  "author": "Raynos <raynos2@gmail.com>",
+  "repository": "git://github.com/Raynos/function-bind.git",
+  "main": "index",
+  "homepage": "https://github.com/Raynos/function-bind",
+  "contributors": [
+    {
+      "name": "Raynos"
+    },
+    {
+      "name": "Jordan Harband",
+      "url": "https://github.com/ljharb"
+    }
+  ],
+  "bugs": {
+    "url": "https://github.com/Raynos/function-bind/issues",
+    "email": "raynos2@gmail.com"
+  },
+  "dependencies": {},
+  "devDependencies": {
+    "@ljharb/eslint-config": "^12.2.1",
+    "covert": "^1.1.0",
+    "eslint": "^4.5.0",
+    "jscs": "^3.0.7",
+    "tape": "^4.8.0"
+  },
+  "license": "MIT",
+  "scripts": {
+    "pretest": "npm run lint",
+    "test": "npm run tests-only",
+    "posttest": "npm run coverage -- --quiet",
+    "tests-only": "node test",
+    "coverage": "covert test/*.js",
+    "lint": "npm run jscs && npm run eslint",
+    "jscs": "jscs *.js */*.js",
+    "eslint": "eslint *.js */*.js"
+  },
+  "testling": {
+    "files": "test/index.js",
+    "browsers": [
+      "ie/8..latest",
+      "firefox/16..latest",
+      "firefox/nightly",
+      "chrome/22..latest",
+      "chrome/canary",
+      "opera/12..latest",
+      "opera/next",
+      "safari/5.1..latest",
+      "ipad/6.0..latest",
+      "iphone/6.0..latest",
+      "android-browser/4.2..latest"
+    ]
+  }
+,"_resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.1.tgz"
+,"_integrity": "sha512-yIovAzMX49sF8Yl58fSCWJ5svSLuaibPxXQJFLmBObTuCr0Mf1KiPopGM9NiFjiYBCbfaa2Fh6breQ6ANVTI0A=="
+,"_from": "function-bind@1.1.1"
+}

syft/cataloger/javascript/test-fixtures/pkg-json/package.json

@@ -0,0 +1,314 @@
+{
+  "version": "6.14.6",
+  "name": "npm",
+  "description": "a package manager for JavaScript",
+  "keywords": [
+    "install",
+    "modules",
+    "package manager",
+    "package.json"
+  ],
+  "preferGlobal": true,
+  "config": {
+    "publishtest": false
+  },
+  "homepage": "https://docs.npmjs.com/",
+  "author": "Isaac Z. Schlueter <i@izs.me> (http://blog.izs.me)",
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/npm/cli"
+  },
+  "bugs": {
+    "url": "https://npm.community/c/bugs"
+  },
+  "directories": {
+    "bin": "./bin",
+    "doc": "./doc",
+    "lib": "./lib",
+    "man": "./man"
+  },
+  "main": "./lib/npm.js",
+  "bin": {
+    "npm": "./bin/npm-cli.js",
+    "npx": "./bin/npx-cli.js"
+  },
+  "dependencies": {
+    "JSONStream": "^1.3.5",
+    "abbrev": "~1.1.1",
+    "ansicolors": "~0.3.2",
+    "ansistyles": "~0.1.3",
+    "aproba": "^2.0.0",
+    "archy": "~1.0.0",
+    "bin-links": "^1.1.7",
+    "bluebird": "^3.5.5",
+    "byte-size": "^5.0.1",
+    "cacache": "^12.0.3",
+    "call-limit": "^1.1.1",
+    "chownr": "^1.1.4",
+    "ci-info": "^2.0.0",
+    "cli-columns": "^3.1.2",
+    "cli-table3": "^0.5.1",
+    "cmd-shim": "^3.0.3",
+    "columnify": "~1.5.4",
+    "config-chain": "^1.1.12",
+    "detect-indent": "~5.0.0",
+    "detect-newline": "^2.1.0",
+    "dezalgo": "~1.0.3",
+    "editor": "~1.0.0",
+    "figgy-pudding": "^3.5.1",
+    "find-npm-prefix": "^1.0.2",
+    "fs-vacuum": "~1.2.10",
+    "fs-write-stream-atomic": "~1.0.10",
+    "gentle-fs": "^2.3.0",
+    "glob": "^7.1.6",
+    "graceful-fs": "^4.2.4",
+    "has-unicode": "~2.0.1",
+    "hosted-git-info": "^2.8.8",
+    "iferr": "^1.0.2",
+    "infer-owner": "^1.0.4",
+    "inflight": "~1.0.6",
+    "inherits": "^2.0.4",
+    "ini": "^1.3.5",
+    "init-package-json": "^1.10.3",
+    "is-cidr": "^3.0.0",
+    "json-parse-better-errors": "^1.0.2",
+    "lazy-property": "~1.0.0",
+    "libcipm": "^4.0.7",
+    "libnpm": "^3.0.1",
+    "libnpmaccess": "^3.0.2",
+    "libnpmhook": "^5.0.3",
+    "libnpmorg": "^1.0.1",
+    "libnpmsearch": "^2.0.2",
+    "libnpmteam": "^1.0.2",
+    "libnpx": "^10.2.2",
+    "lock-verify": "^2.1.0",
+    "lockfile": "^1.0.4",
+    "lodash._baseuniq": "~4.6.0",
+    "lodash.clonedeep": "~4.5.0",
+    "lodash.union": "~4.6.0",
+    "lodash.uniq": "~4.5.0",
+    "lodash.without": "~4.4.0",
+    "lru-cache": "^5.1.1",
+    "meant": "~1.0.1",
+    "mississippi": "^3.0.0",
+    "mkdirp": "^0.5.5",
+    "move-concurrently": "^1.0.1",
+    "node-gyp": "^5.1.0",
+    "nopt": "^4.0.3",
+    "normalize-package-data": "^2.5.0",
+    "npm-audit-report": "^1.3.2",
+    "npm-cache-filename": "~1.0.2",
+    "npm-install-checks": "^3.0.2",
+    "npm-lifecycle": "^3.1.4",
+    "npm-package-arg": "^6.1.1",
+    "npm-packlist": "^1.4.8",
+    "npm-pick-manifest": "^3.0.2",
+    "npm-profile": "^4.0.4",
+    "npm-registry-fetch": "^4.0.5",
+    "npm-user-validate": "~1.0.0",
+    "npmlog": "~4.1.2",
+    "once": "~1.4.0",
+    "opener": "^1.5.1",
+    "osenv": "^0.1.5",
+    "pacote": "^9.5.12",
+    "path-is-inside": "~1.0.2",
+    "promise-inflight": "~1.0.1",
+    "qrcode-terminal": "^0.12.0",
+    "query-string": "^6.8.2",
+    "qw": "~1.0.1",
+    "read": "~1.0.7",
+    "read-cmd-shim": "^1.0.5",
+    "read-installed": "~4.0.3",
+    "read-package-json": "^2.1.1",
+    "read-package-tree": "^5.3.1",
+    "readable-stream": "^3.6.0",
+    "readdir-scoped-modules": "^1.1.0",
+    "request": "^2.88.0",
+    "retry": "^0.12.0",
+    "rimraf": "^2.7.1",
+    "safe-buffer": "^5.1.2",
+    "semver": "^5.7.1",
+    "sha": "^3.0.0",
+    "slide": "~1.1.6",
+    "sorted-object": "~2.0.1",
+    "sorted-union-stream": "~2.1.3",
+    "ssri": "^6.0.1",
+    "stringify-package": "^1.0.1",
+    "tar": "^4.4.13",
+    "text-table": "~0.2.0",
+    "tiny-relative-date": "^1.3.0",
+    "uid-number": "0.0.6",
+    "umask": "~1.1.0",
+    "unique-filename": "^1.1.1",
+    "unpipe": "~1.0.0",
+    "update-notifier": "^2.5.0",
+    "uuid": "^3.3.3",
+    "validate-npm-package-license": "^3.0.4",
+    "validate-npm-package-name": "~3.0.0",
+    "which": "^1.3.1",
+    "worker-farm": "^1.7.0",
+    "write-file-atomic": "^2.4.3"
+  },
+  "bundleDependencies": [
+    "abbrev",
+    "ansicolors",
+    "ansistyles",
+    "aproba",
+    "archy",
+    "bin-links",
+    "bluebird",
+    "byte-size",
+    "cacache",
+    "call-limit",
+    "chownr",
+    "ci-info",
+    "cli-columns",
+    "cli-table3",
+    "cmd-shim",
+    "columnify",
+    "config-chain",
+    "debuglog",
+    "detect-indent",
+    "detect-newline",
+    "dezalgo",
+    "editor",
+    "figgy-pudding",
+    "find-npm-prefix",
+    "fs-vacuum",
+    "fs-write-stream-atomic",
+    "gentle-fs",
+    "glob",
+    "graceful-fs",
+    "has-unicode",
+    "hosted-git-info",
+    "iferr",
+    "imurmurhash",
+    "infer-owner",
+    "inflight",
+    "inherits",
+    "ini",
+    "init-package-json",
+    "is-cidr",
+    "json-parse-better-errors",
+    "JSONStream",
+    "lazy-property",
+    "libcipm",
+    "libnpm",
+    "libnpmaccess",
+    "libnpmhook",
+    "libnpmorg",
+    "libnpmsearch",
+    "libnpmteam",
+    "libnpx",
+    "lock-verify",
+    "lockfile",
+    "lodash._baseindexof",
+    "lodash._baseuniq",
+    "lodash._bindcallback",
+    "lodash._cacheindexof",
+    "lodash._createcache",
+    "lodash._getnative",
+    "lodash.clonedeep",
+    "lodash.restparam",
+    "lodash.union",
+    "lodash.uniq",
+    "lodash.without",
+    "lru-cache",
+    "meant",
+    "mississippi",
+    "mkdirp",
+    "move-concurrently",
+    "node-gyp",
+    "nopt",
+    "normalize-package-data",
+    "npm-audit-report",
+    "npm-cache-filename",
+    "npm-install-checks",
+    "npm-lifecycle",
+    "npm-package-arg",
+    "npm-packlist",
+    "npm-pick-manifest",
+    "npm-profile",
+    "npm-registry-fetch",
+    "npm-user-validate",
+    "npmlog",
+    "once",
+    "opener",
+    "osenv",
+    "pacote",
+    "path-is-inside",
+    "promise-inflight",
+    "qrcode-terminal",
+    "query-string",
+    "qw",
+    "read-cmd-shim",
+    "read-installed",
+    "read-package-json",
+    "read-package-tree",
+    "read",
+    "readable-stream",
+    "readdir-scoped-modules",
+    "request",
+    "retry",
+    "rimraf",
+    "safe-buffer",
+    "semver",
+    "sha",
+    "slide",
+    "sorted-object",
+    "sorted-union-stream",
+    "ssri",
+    "stringify-package",
+    "tar",
+    "text-table",
+    "tiny-relative-date",
+    "uid-number",
+    "umask",
+    "unique-filename",
+    "unpipe",
+    "update-notifier",
+    "uuid",
+    "validate-npm-package-license",
+    "validate-npm-package-name",
+    "which",
+    "worker-farm",
+    "write-file-atomic"
+  ],
+  "devDependencies": {
+    "deep-equal": "^1.0.1",
+    "get-stream": "^4.1.0",
+    "licensee": "^7.0.3",
+    "marked": "^0.6.3",
+    "marked-man": "^0.6.0",
+    "npm-registry-couchapp": "^2.7.4",
+    "npm-registry-mock": "^1.3.1",
+    "require-inject": "^1.4.4",
+    "sprintf-js": "^1.1.2",
+    "standard": "^11.0.1",
+    "tacks": "^1.3.0",
+    "tap": "^12.7.0",
+    "tar-stream": "^2.1.0"
+  },
+  "scripts": {
+    "dumpconf": "env | grep npm | sort | uniq",
+    "prepare": "node bin/npm-cli.js rebuild && node bin/npm-cli.js --no-audit --no-timing prune --prefix=. --no-global && rimraf test/*/*/node_modules && make -j4 mandocs",
+    "preversion": "bash scripts/update-authors.sh && git add AUTHORS && git commit -m \"update AUTHORS\" || true",
+    "licenses": "licensee --production --errors-only",
+    "tap": "tap -J --timeout 300 --no-esm",
+    "tap-cover": "tap -J --nyc-arg=--cache --coverage --timeout 600 --no-esm",
+    "lint": "standard",
+    "pretest": "npm run lint",
+    "test": "npm run test-tap --",
+    "test:nocleanup": "NO_TEST_CLEANUP=1 npm run test --",
+    "sudotest": "sudo npm run tap -- \"test/tap/*.js\"",
+    "sudotest:nocleanup": "sudo NO_TEST_CLEANUP=1 npm run tap -- \"test/tap/*.js\"",
+    "posttest": "rimraf test/npm_cache*",
+    "test-coverage": "npm run tap-cover -- \"test/tap/*.js\" \"test/network/*.js\"",
+    "test-tap": "npm run tap -- \"test/tap/*.js\" \"test/network/*.js\"",
+    "test-node": "tap --timeout 240 \"test/tap/*.js\" \"test/network/*.js\""
+  },
+  "license": "Artistic-2.0",
+  "engines": {
+    "node": "6 >=6.2.0 || 8 || >=9.3.0"
+  }
+}

syft/cataloger/python/cataloger.go

@@ -1,21 +0,0 @@
-/*
-Package python provides a concrete Cataloger implementation for Python ecosystem files (egg, wheel, requirements.txt).
-*/
-package python
-
-import (
-	"github.com/anchore/syft/syft/cataloger/common"
-)
-
-// NewPythonCataloger returns a new Python cataloger object.
-func NewPythonCataloger() *common.GenericCataloger {
-	globParsers := map[string]common.ParserFn{
-		"**/*egg-info/PKG-INFO":  parseEggMetadata,
-		"**/*dist-info/METADATA": parseWheelMetadata,
-		"**/*requirements*.txt":  parseRequirementsTxt,
-		"**/poetry.lock":         parsePoetryLock,
-		"**/setup.py":            parseSetup,
-	}
-
-	return common.NewGenericCataloger(nil, globParsers, "python-cataloger")
-}

syft/cataloger/python/index_cataloger.go

@@ -0,0 +1,19 @@
+/*
+Package python provides a concrete Cataloger implementation for Python ecosystem files (egg, wheel, requirements.txt).
+*/
+package python
+
+import (
+	"github.com/anchore/syft/syft/cataloger/common"
+)
+
+// NewPythonIndexCataloger returns a new cataloger for python packages referenced from poetry lock files, requirements.txt files, and setup.py files.
+func NewPythonIndexCataloger() *common.GenericCataloger {
+	globParsers := map[string]common.ParserFn{
+		"**/*requirements*.txt": parseRequirementsTxt,
+		"**/poetry.lock":        parsePoetryLock,
+		"**/setup.py":           parseSetup,
+	}
+
+	return common.NewGenericCataloger(nil, globParsers, "python-index-cataloger")
+}

syft/cataloger/python/package_cataloger.go

@@ -0,0 +1,182 @@
+package python
+
+import (
+	"bufio"
+	"fmt"
+	"path/filepath"
+	"strings"
+
+	"github.com/anchore/syft/internal/log"
+
+	"github.com/anchore/stereoscope/pkg/file"
+
+	"github.com/anchore/syft/syft/pkg"
+
+	"github.com/anchore/syft/syft/scope"
+)
+
+const (
+	eggMetadataGlob   = "**/*egg-info/PKG-INFO"
+	wheelMetadataGlob = "**/*dist-info/METADATA"
+)
+
+type PackageCataloger struct{}
+
+// NewPythonPackageCataloger returns a new cataloger for python packages within egg or wheel installation directories.
+func NewPythonPackageCataloger() *PackageCataloger {
+	return &PackageCataloger{}
+}
+
+// Name returns a string that uniquely describes a cataloger
+func (c *PackageCataloger) Name() string {
+	return "python-package-cataloger"
+}
+
+// Catalog is given an object to resolve file references and content, this function returns any discovered Packages after analyzing python egg and wheel installations.
+func (c *PackageCataloger) Catalog(resolver scope.Resolver) ([]pkg.Package, error) {
+	// nolint:prealloc
+	var fileMatches []file.Reference
+
+	for _, glob := range []string{eggMetadataGlob, wheelMetadataGlob} {
+		matches, err := resolver.FilesByGlob(glob)
+		if err != nil {
+			return nil, fmt.Errorf("failed to find files by glob: %s", glob)
+		}
+		fileMatches = append(fileMatches, matches...)
+	}
+
+	var pkgs []pkg.Package
+	for _, ref := range fileMatches {
+		p, err := c.catalogEggOrWheel(resolver, ref)
+		if err != nil {
+			return nil, fmt.Errorf("unable to catalog python package=%+v: %w", ref.Path, err)
+		}
+		if p != nil {
+			pkgs = append(pkgs, *p)
+		}
+	}
+	return pkgs, nil
+}
+
+// catalogEggOrWheel takes the primary metadata file reference and returns the python package it represents.
+func (c *PackageCataloger) catalogEggOrWheel(resolver scope.Resolver, metadataRef file.Reference) (*pkg.Package, error) {
+	metadata, sources, err := c.assembleEggOrWheelMetadata(resolver, metadataRef)
+	if err != nil {
+		return nil, err
+	}
+
+	var licenses []string
+	if metadata.License != "" {
+		licenses = []string{metadata.License}
+	}
+
+	return &pkg.Package{
+		Name:         metadata.Name,
+		Version:      metadata.Version,
+		FoundBy:      c.Name(),
+		Source:       sources,
+		Licenses:     licenses,
+		Language:     pkg.Python,
+		Type:         pkg.PythonPkg,
+		MetadataType: pkg.PythonPackageMetadataType,
+		Metadata:     *metadata,
+	}, nil
+}
+
+// fetchRecordFiles finds a corresponding RECORD file for the given python package metadata file and returns the set of file records contained.
+func (c *PackageCataloger) fetchRecordFiles(resolver scope.Resolver, metadataRef file.Reference) (files []pkg.PythonFileRecord, sources []file.Reference, err error) {
+	// we've been given a file reference to a specific wheel METADATA file. note: this may be for a directory
+	// or for an image... for an image the METADATA file may be present within multiple layers, so it is important
+	// to reconcile the RECORD path to the same layer (or the next adjacent lower layer).
+
+	// lets find the RECORD file relative to the directory where the METADATA file resides (in path AND layer structure)
+	recordPath := filepath.Join(filepath.Dir(string(metadataRef.Path)), "RECORD")
+	recordRef, err := resolver.RelativeFileByPath(metadataRef, recordPath)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	if recordRef != nil {
+		sources = append(sources, *recordRef)
+
+		recordContents, err := resolver.FileContentsByRef(*recordRef)
+		if err != nil {
+			return nil, nil, err
+		}
+
+		// parse the record contents
+		records, err := parseWheelOrEggRecord(strings.NewReader(recordContents))
+		if err != nil {
+			return nil, nil, err
+		}
+
+		files = append(files, records...)
+	}
+	return files, sources, nil
+}
+
+// fetchTopLevelPackages finds a corresponding top_level.txt file for the given python package metadata file and returns the set of package names contained.
+func (c *PackageCataloger) fetchTopLevelPackages(resolver scope.Resolver, metadataRef file.Reference) (pkgs []string, sources []file.Reference, err error) {
+	// a top_level.txt file specifies the python top-level packages (provided by this python package) installed into site-packages
+	parentDir := filepath.Dir(string(metadataRef.Path))
+	topLevelPath := filepath.Join(parentDir, "top_level.txt")
+	topLevelRef, err := resolver.RelativeFileByPath(metadataRef, topLevelPath)
+	if err != nil {
+		return nil, nil, err
+	}
+	if topLevelRef == nil {
+		log.Warnf("missing python package top_level.txt (package=%q)", string(metadataRef.Path))
+		return nil, nil, nil
+	}
+
+	sources = append(sources, *topLevelRef)
+
+	topLevelContents, err := resolver.FileContentsByRef(*topLevelRef)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	scanner := bufio.NewScanner(strings.NewReader(topLevelContents))
+	for scanner.Scan() {
+		pkgs = append(pkgs, scanner.Text())
+	}
+
+	if err := scanner.Err(); err != nil {
+		return nil, nil, fmt.Errorf("could not read python package top_level.txt: %w", err)
+	}
+
+	return pkgs, sources, nil
+}
+
+// assembleEggOrWheelMetadata discovers and accumulates python package metadata from multiple file sources and returns a single metadata object as well as a list of files where the metadata was derived from.
+func (c *PackageCataloger) assembleEggOrWheelMetadata(resolver scope.Resolver, metadataRef file.Reference) (*pkg.PythonPackageMetadata, []file.Reference, error) {
+	var sources = []file.Reference{metadataRef}
+
+	metadataContents, err := resolver.FileContentsByRef(metadataRef)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	metadata, err := parseWheelOrEggMetadata(metadataRef.Path, strings.NewReader(metadataContents))
+	if err != nil {
+		return nil, nil, err
+	}
+
+	// attach any python files found for the given wheel/egg installation
+	r, s, err := c.fetchRecordFiles(resolver, metadataRef)
+	if err != nil {
+		return nil, nil, err
+	}
+	sources = append(sources, s...)
+	metadata.Files = r
+
+	// attach any top-level package names found for the given wheel/egg installation
+	p, s, err := c.fetchTopLevelPackages(resolver, metadataRef)
+	if err != nil {
+		return nil, nil, err
+	}
+	sources = append(sources, s...)
+	metadata.TopLevelPackages = p
+
+	return &metadata, sources, nil
+}

syft/cataloger/python/package_cataloger_test.go

@@ -0,0 +1,240 @@
+package python
+
+import (
+	"fmt"
+	"io"
+	"io/ioutil"
+	"os"
+	"strings"
+	"testing"
+
+	"github.com/anchore/stereoscope/pkg/file"
+
+	"github.com/anchore/syft/syft/pkg"
+	"github.com/go-test/deep"
+)
+
+type pythonTestResolverMock struct {
+	metadataReader io.Reader
+	recordReader   io.Reader
+	topLevelReader io.Reader
+	metadataRef    *file.Reference
+	recordRef      *file.Reference
+	topLevelRef    *file.Reference
+	contents       map[file.Reference]string
+}
+
+func newTestResolver(metaPath, recordPath, topPath string) *pythonTestResolverMock {
+	metadataReader, err := os.Open(metaPath)
+	if err != nil {
+		panic(fmt.Errorf("failed to open metadata: %+v", err))
+	}
+
+	var recordReader io.Reader
+	if recordPath != "" {
+		recordReader, err = os.Open(recordPath)
+		if err != nil {
+			panic(fmt.Errorf("failed to open record: %+v", err))
+		}
+	}
+
+	var topLevelReader io.Reader
+	if topPath != "" {
+		topLevelReader, err = os.Open(topPath)
+		if err != nil {
+			panic(fmt.Errorf("failed to open top level: %+v", err))
+		}
+	}
+
+	var recordRef *file.Reference
+	if recordReader != nil {
+		ref := file.NewFileReference("test-fixtures/dist-info/RECORD")
+		recordRef = &ref
+	}
+	var topLevelRef *file.Reference
+	if topLevelReader != nil {
+		ref := file.NewFileReference("test-fixtures/dist-info/top_level.txt")
+		topLevelRef = &ref
+	}
+	metadataRef := file.NewFileReference("test-fixtures/dist-info/METADATA")
+	return &pythonTestResolverMock{
+		recordReader:   recordReader,
+		metadataReader: metadataReader,
+		topLevelReader: topLevelReader,
+		metadataRef:    &metadataRef,
+		recordRef:      recordRef,
+		topLevelRef:    topLevelRef,
+		contents:       make(map[file.Reference]string),
+	}
+}
+
+func (r *pythonTestResolverMock) FileContentsByRef(ref file.Reference) (string, error) {
+	switch {
+	case r.topLevelRef != nil && ref.Path == r.topLevelRef.Path:
+		b, err := ioutil.ReadAll(r.topLevelReader)
+		if err != nil {
+			return "", err
+		}
+		return string(b), nil
+	case ref.Path == r.metadataRef.Path:
+		b, err := ioutil.ReadAll(r.metadataReader)
+		if err != nil {
+			return "", err
+		}
+		return string(b), nil
+	case ref.Path == r.recordRef.Path:
+		b, err := ioutil.ReadAll(r.recordReader)
+		if err != nil {
+			return "", err
+		}
+		return string(b), nil
+	}
+	return "", fmt.Errorf("invalid value given")
+}
+
+func (r *pythonTestResolverMock) MultipleFileContentsByRef(_ ...file.Reference) (map[file.Reference]string, error) {
+	return nil, fmt.Errorf("not implemented")
+}
+
+func (r *pythonTestResolverMock) FilesByPath(_ ...file.Path) ([]file.Reference, error) {
+	return nil, fmt.Errorf("not implemented")
+}
+
+func (r *pythonTestResolverMock) FilesByGlob(_ ...string) ([]file.Reference, error) {
+	return nil, fmt.Errorf("not implemented")
+}
+func (r *pythonTestResolverMock) RelativeFileByPath(_ file.Reference, path string) (*file.Reference, error) {
+	switch {
+	case strings.Contains(path, "RECORD"):
+		return r.recordRef, nil
+	case strings.Contains(path, "top_level.txt"):
+		return r.topLevelRef, nil
+	default:
+		return nil, fmt.Errorf("invalid RelativeFileByPath value given: %q", path)
+	}
+}
+
+func TestPythonPackageWheelCataloger(t *testing.T) {
+	tests := []struct {
+		MetadataFixture string
+		RecordFixture   string
+		TopLevelFixture string
+		ExpectedPackage pkg.Package
+	}{
+		{
+			MetadataFixture: "test-fixtures/egg-info/PKG-INFO",
+			RecordFixture:   "test-fixtures/egg-info/RECORD",
+			TopLevelFixture: "test-fixtures/egg-info/top_level.txt",
+			ExpectedPackage: pkg.Package{
+				Name:         "requests",
+				Version:      "2.22.0",
+				Type:         pkg.PythonPkg,
+				Language:     pkg.Python,
+				Licenses:     []string{"Apache 2.0"},
+				FoundBy:      "python-package-cataloger",
+				MetadataType: pkg.PythonPackageMetadataType,
+				Metadata: pkg.PythonPackageMetadata{
+					Name:                 "requests",
+					Version:              "2.22.0",
+					License:              "Apache 2.0",
+					Platform:             "UNKNOWN",
+					Author:               "Kenneth Reitz",
+					AuthorEmail:          "me@kennethreitz.org",
+					SitePackagesRootPath: "test-fixtures",
+					Files: []pkg.PythonFileRecord{
+						{Path: "requests-2.22.0.dist-info/INSTALLER", Digest: &pkg.Digest{"sha256", "zuuue4knoyJ-UwPPXg8fezS7VCrXJQrAP7zeNuwvFQg"}, Size: "4"},
+						{Path: "requests/__init__.py", Digest: &pkg.Digest{"sha256", "PnKCgjcTq44LaAMzB-7--B2FdewRrE8F_vjZeaG9NhA"}, Size: "3921"},
+						{Path: "requests/__pycache__/__version__.cpython-38.pyc"},
+						{Path: "requests/__pycache__/utils.cpython-38.pyc"},
+						{Path: "requests/__version__.py", Digest: &pkg.Digest{"sha256", "Bm-GFstQaFezsFlnmEMrJDe8JNROz9n2XXYtODdvjjc"}, Size: "436"},
+						{Path: "requests/utils.py", Digest: &pkg.Digest{"sha256", "LtPJ1db6mJff2TJSJWKi7rBpzjPS3mSOrjC9zRhoD3A"}, Size: "30049"},
+					},
+					TopLevelPackages: []string{"requests"},
+				},
+			},
+		},
+		{
+			MetadataFixture: "test-fixtures/dist-info/METADATA",
+			RecordFixture:   "test-fixtures/dist-info/RECORD",
+			TopLevelFixture: "test-fixtures/dist-info/top_level.txt",
+			ExpectedPackage: pkg.Package{
+				Name:         "Pygments",
+				Version:      "2.6.1",
+				Type:         pkg.PythonPkg,
+				Language:     pkg.Python,
+				Licenses:     []string{"BSD License"},
+				FoundBy:      "python-package-cataloger",
+				MetadataType: pkg.PythonPackageMetadataType,
+				Metadata: pkg.PythonPackageMetadata{
+					Name:                 "Pygments",
+					Version:              "2.6.1",
+					License:              "BSD License",
+					Platform:             "any",
+					Author:               "Georg Brandl",
+					AuthorEmail:          "georg@python.org",
+					SitePackagesRootPath: "test-fixtures",
+					Files: []pkg.PythonFileRecord{
+						{Path: "../../../bin/pygmentize", Digest: &pkg.Digest{"sha256", "dDhv_U2jiCpmFQwIRHpFRLAHUO4R1jIJPEvT_QYTFp8"}, Size: "220"},
+						{Path: "Pygments-2.6.1.dist-info/AUTHORS", Digest: &pkg.Digest{"sha256", "PVpa2_Oku6BGuiUvutvuPnWGpzxqFy2I8-NIrqCvqUY"}, Size: "8449"},
+						{Path: "Pygments-2.6.1.dist-info/RECORD"},
+						{Path: "pygments/__pycache__/__init__.cpython-38.pyc"},
+						{Path: "pygments/util.py", Digest: &pkg.Digest{"sha256", "586xXHiJGGZxqk5PMBu3vBhE68DLuAe5MBARWrSPGxA"}, Size: "10778"},
+					},
+					TopLevelPackages: []string{"pygments", "something_else"},
+				},
+			},
+		},
+		{
+			// in cases where the metadata file is available and the record is not we should still record there is a package
+			// additionally empty top_level.txt files should not result in an error
+			MetadataFixture: "test-fixtures/partial.dist-info/METADATA",
+			ExpectedPackage: pkg.Package{
+				Name:         "Pygments",
+				Version:      "2.6.1",
+				Type:         pkg.PythonPkg,
+				Language:     pkg.Python,
+				Licenses:     []string{"BSD License"},
+				FoundBy:      "python-package-cataloger",
+				MetadataType: pkg.PythonPackageMetadataType,
+				Metadata: pkg.PythonPackageMetadata{
+					Name:                 "Pygments",
+					Version:              "2.6.1",
+					License:              "BSD License",
+					Platform:             "any",
+					Author:               "Georg Brandl",
+					AuthorEmail:          "georg@python.org",
+					SitePackagesRootPath: "test-fixtures",
+				},
+			},
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.MetadataFixture, func(t *testing.T) {
+			resolver := newTestResolver(test.MetadataFixture, test.RecordFixture, test.TopLevelFixture)
+
+			// note that the source is the record ref created by the resolver mock... attach the expected values
+			test.ExpectedPackage.Source = []file.Reference{*resolver.metadataRef}
+			if resolver.recordRef != nil {
+				test.ExpectedPackage.Source = append(test.ExpectedPackage.Source, *resolver.recordRef)
+			}
+
+			if resolver.topLevelRef != nil {
+				test.ExpectedPackage.Source = append(test.ExpectedPackage.Source, *resolver.topLevelRef)
+			}
+			// end patching expected values with runtime data...
+
+			pyPkgCataloger := NewPythonPackageCataloger()
+
+			actual, err := pyPkgCataloger.catalogEggOrWheel(resolver, *resolver.metadataRef)
+			if err != nil {
+				t.Fatalf("failed to catalog python package: %+v", err)
+			}
+
+			for _, d := range deep.Equal(actual, &test.ExpectedPackage) {
+				t.Errorf("diff: %+v", d)
+			}
+		})
+	}
+
+}

syft/cataloger/python/parse_poetry_lock_test.go

@@ -1,10 +1,11 @@
 package python
 
 import (
-	"github.com/anchore/syft/syft/pkg"
-	"github.com/go-test/deep"
 	"os"
 	"testing"
+
+	"github.com/anchore/syft/syft/pkg"
+	"github.com/go-test/deep"
 )
 
 func TestParsePoetryLock(t *testing.T) {
@@ -13,28 +14,28 @@ func TestParsePoetryLock(t *testing.T) {
 			Name:     "added-value",
 			Version:  "0.14.2",
 			Language: pkg.Python,
-			Type:     pkg.PoetryPkg,
+			Type:     pkg.PythonPkg,
 			Licenses: nil,
 		},
 		{
 			Name:     "alabaster",
 			Version:  "0.7.12",
 			Language: pkg.Python,
-			Type:     pkg.PoetryPkg,
+			Type:     pkg.PythonPkg,
 			Licenses: nil,
 		},
 		{
 			Name:     "appnope",
 			Version:  "0.1.0",
 			Language: pkg.Python,
-			Type:     pkg.PoetryPkg,
+			Type:     pkg.PythonPkg,
 			Licenses: nil,
 		},
 		{
 			Name:     "asciitree",
 			Version:  "0.3.3",
 			Language: pkg.Python,
-			Type:     pkg.PoetryPkg,
+			Type:     pkg.PythonPkg,
 			Licenses: nil,
 		},
 	}

syft/cataloger/python/parse_requirements.go

@@ -47,7 +47,7 @@ func parseRequirementsTxt(_ string, reader io.Reader) ([]pkg.Package, error) {
 				Name:     name,
 				Version:  version,
 				Language: pkg.Python,
-				Type:     pkg.PythonRequirementsPkg,
+				Type:     pkg.PythonPkg,
 			})
 		default:
 			continue

syft/cataloger/python/parse_requirements_test.go

@@ -4,24 +4,45 @@ import (
 	"os"
 	"testing"
 
+	"github.com/go-test/deep"
+
 	"github.com/anchore/syft/syft/pkg"
 )
 
+func assertPackagesEqual(t *testing.T, actual []pkg.Package, expected map[string]pkg.Package) {
+	t.Helper()
+	if len(actual) != len(expected) {
+		for _, a := range actual {
+			t.Log("   ", a)
+		}
+		t.Fatalf("unexpected package count: %d!=%d", len(actual), len(expected))
+	}
+
+	for _, a := range actual {
+		expectedPkg, ok := expected[a.Name]
+		if !ok {
+			t.Errorf("unexpected package found: '%s'", a.Name)
+		}
+
+		for _, d := range deep.Equal(a, expectedPkg) {
+			t.Errorf("diff: %+v", d)
+		}
+	}
+}
+
 func TestParseRequirementsTxt(t *testing.T) {
 	expected := map[string]pkg.Package{
 		"foo": {
 			Name:     "foo",
 			Version:  "1.0.0",
 			Language: pkg.Python,
-			Type:     pkg.PythonRequirementsPkg,
-			Licenses: []string{},
+			Type:     pkg.PythonPkg,
 		},
 		"flask": {
 			Name:     "flask",
 			Version:  "4.0.0",
 			Language: pkg.Python,
-			Type:     pkg.PythonRequirementsPkg,
-			Licenses: []string{},
+			Type:     pkg.PythonPkg,
 		},
 	}
 	fixture, err := os.Open("test-fixtures/requires/requirements.txt")
@@ -34,6 +55,6 @@ func TestParseRequirementsTxt(t *testing.T) {
 		t.Fatalf("failed to parse requirements: %+v", err)
 	}
 
-	assertPkgsEqual(t, actual, expected)
+	assertPackagesEqual(t, actual, expected)
 
 }

syft/cataloger/python/parse_setup.go

@@ -41,7 +41,7 @@ func parseSetup(_ string, reader io.Reader) ([]pkg.Package, error) {
 				Name:     strings.Trim(name, "'\""),
 				Version:  strings.Trim(version, "'\""),
 				Language: pkg.Python,
-				Type:     pkg.PythonSetupPkg,
+				Type:     pkg.PythonPkg,
 			})
 		}
 	}

syft/cataloger/python/parse_setup_test.go

@@ -13,36 +13,31 @@ func TestParseSetup(t *testing.T) {
 			Name:     "pathlib3",
 			Version:  "2.2.0",
 			Language: pkg.Python,
-			Type:     pkg.PythonSetupPkg,
-			Licenses: []string{},
+			Type:     pkg.PythonPkg,
 		},
 		"mypy": {
 			Name:     "mypy",
 			Version:  "v0.770",
 			Language: pkg.Python,
-			Type:     pkg.PythonSetupPkg,
-			Licenses: []string{},
+			Type:     pkg.PythonPkg,
 		},
 		"mypy1": {
 			Name:     "mypy1",
 			Version:  "v0.770",
 			Language: pkg.Python,
-			Type:     pkg.PythonSetupPkg,
-			Licenses: []string{},
+			Type:     pkg.PythonPkg,
 		},
 		"mypy2": {
 			Name:     "mypy2",
 			Version:  "v0.770",
 			Language: pkg.Python,
-			Type:     pkg.PythonSetupPkg,
-			Licenses: []string{},
+			Type:     pkg.PythonPkg,
 		},
 		"mypy3": {
 			Name:     "mypy3",
 			Version:  "v0.770",
 			Language: pkg.Python,
-			Type:     pkg.PythonSetupPkg,
-			Licenses: []string{},
+			Type:     pkg.PythonPkg,
 		},
 	}
 	fixture, err := os.Open("test-fixtures/setup/setup.py")
@@ -55,6 +50,6 @@ func TestParseSetup(t *testing.T) {
 		t.Fatalf("failed to parse requirements: %+v", err)
 	}
 
-	assertPkgsEqual(t, actual, expected)
+	assertPackagesEqual(t, actual, expected)
 
 }

syft/cataloger/python/parse_wheel_egg.go

@@ -1,100 +0,0 @@
-package python
-
-import (
-	"bufio"
-	"fmt"
-	"io"
-	"strings"
-
-	"github.com/anchore/syft/syft/cataloger/common"
-	"github.com/anchore/syft/syft/pkg"
-)
-
-// integrity check
-var _ common.ParserFn = parseWheelMetadata
-var _ common.ParserFn = parseEggMetadata
-
-// parseWheelMetadata is a parser function for individual Python Wheel metadata file contents, returning all Python
-// packages listed.
-func parseWheelMetadata(_ string, reader io.Reader) ([]pkg.Package, error) {
-	packages, err := parseWheelOrEggMetadata(reader)
-	for idx := range packages {
-		packages[idx].Type = pkg.WheelPkg
-	}
-	return packages, err
-}
-
-// parseEggMetadata is a parser function for individual Python Egg metadata file contents, returning all Python
-// packages listed.
-func parseEggMetadata(_ string, reader io.Reader) ([]pkg.Package, error) {
-	packages, err := parseWheelOrEggMetadata(reader)
-	for idx := range packages {
-		packages[idx].Type = pkg.EggPkg
-	}
-	return packages, err
-}
-
-// parseWheelOrEggMetadata takes a Python Egg or Wheel (which share the same format and values for our purposes),
-// returning all Python packages listed.
-func parseWheelOrEggMetadata(reader io.Reader) ([]pkg.Package, error) {
-	fields := make(map[string]string)
-	var key string
-
-	scanner := bufio.NewScanner(reader)
-	for scanner.Scan() {
-		line := scanner.Text()
-
-		line = strings.TrimRight(line, "\n")
-
-		// empty line indicates end of entry
-		if len(line) == 0 {
-			// if the entry has not started, keep parsing lines
-			if len(fields) == 0 {
-				continue
-			}
-			break
-		}
-
-		switch {
-		case strings.HasPrefix(line, " "):
-			// a field-body continuation
-			if len(key) == 0 {
-				return nil, fmt.Errorf("no match for continuation: line: '%s'", line)
-			}
-
-			val, ok := fields[key]
-			if !ok {
-				return nil, fmt.Errorf("no previous key exists, expecting: %s", key)
-			}
-			// concatenate onto previous value
-			val = fmt.Sprintf("%s\n %s", val, strings.TrimSpace(line))
-			fields[key] = val
-		default:
-			// parse a new key (note, duplicate keys are overridden)
-			if i := strings.Index(line, ":"); i > 0 {
-				key = strings.TrimSpace(line[0:i])
-				val := strings.TrimSpace(line[i+1:])
-
-				fields[key] = val
-			} else {
-				return nil, fmt.Errorf("cannot parse field from line: '%s'", line)
-			}
-		}
-	}
-
-	if err := scanner.Err(); err != nil {
-		return nil, fmt.Errorf("failed to parse python wheel/egg: %w", err)
-	}
-
-	p := pkg.Package{
-		Name:     fields["Name"],
-		Version:  fields["Version"],
-		Language: pkg.Python,
-	}
-
-	if license, ok := fields["License"]; ok && license != "" {
-		p.Licenses = []string{license}
-	}
-
-	return []pkg.Package{p}, nil
-}

syft/cataloger/python/parse_wheel_egg_metadata.go

@@ -0,0 +1,80 @@
+package python
+
+import (
+	"bufio"
+	"fmt"
+	"io"
+	"path/filepath"
+	"strings"
+
+	"github.com/anchore/stereoscope/pkg/file"
+
+	"github.com/mitchellh/mapstructure"
+
+	"github.com/anchore/syft/syft/pkg"
+)
+
+// parseWheelOrEggMetadata takes a Python Egg or Wheel (which share the same format and values for our purposes),
+// returning all Python packages listed.
+func parseWheelOrEggMetadata(path file.Path, reader io.Reader) (pkg.PythonPackageMetadata, error) {
+	fields := make(map[string]string)
+	var key string
+
+	scanner := bufio.NewScanner(reader)
+	for scanner.Scan() {
+		line := scanner.Text()
+		line = strings.TrimRight(line, "\n")
+
+		// empty line indicates end of entry
+		if len(line) == 0 {
+			// if the entry has not started, keep parsing lines
+			if len(fields) == 0 {
+				continue
+			}
+			break
+		}
+
+		switch {
+		case strings.HasPrefix(line, " "):
+			// a field-body continuation
+			if len(key) == 0 {
+				return pkg.PythonPackageMetadata{}, fmt.Errorf("no match for continuation: line: '%s'", line)
+			}
+
+			val, ok := fields[key]
+			if !ok {
+				return pkg.PythonPackageMetadata{}, fmt.Errorf("no previous key exists, expecting: %s", key)
+			}
+			// concatenate onto previous value
+			val = fmt.Sprintf("%s\n %s", val, strings.TrimSpace(line))
+			fields[key] = val
+		default:
+			// parse a new key (note, duplicate keys are overridden)
+			if i := strings.Index(line, ":"); i > 0 {
+				// mapstruct cannot map keys with dashes, and we are expected to persist the "Author-email" field
+				key = strings.ReplaceAll(strings.TrimSpace(line[0:i]), "-", "")
+				val := strings.TrimSpace(line[i+1:])
+
+				fields[key] = val
+			} else {
+				return pkg.PythonPackageMetadata{}, fmt.Errorf("cannot parse field from line: '%s'", line)
+			}
+		}
+	}
+
+	if err := scanner.Err(); err != nil {
+		return pkg.PythonPackageMetadata{}, fmt.Errorf("failed to parse python wheel/egg: %w", err)
+	}
+
+	var metadata pkg.PythonPackageMetadata
+	if err := mapstructure.Decode(fields, &metadata); err != nil {
+		return pkg.PythonPackageMetadata{}, fmt.Errorf("unable to parse APK metadata: %w", err)
+	}
+
+	// add additional metadata not stored in the egg/wheel metadata file
+
+	sitePackagesRoot := filepath.Clean(filepath.Join(filepath.Dir(string(path)), ".."))
+	metadata.SitePackagesRootPath = sitePackagesRoot
+
+	return metadata, nil
+}

syft/cataloger/python/parse_wheel_egg_metadata_test.go

@@ -0,0 +1,62 @@
+package python
+
+import (
+	"os"
+	"testing"
+
+	"github.com/anchore/stereoscope/pkg/file"
+
+	"github.com/anchore/syft/syft/pkg"
+	"github.com/go-test/deep"
+)
+
+func TestParseWheelEggMetadata(t *testing.T) {
+	tests := []struct {
+		Fixture          string
+		ExpectedMetadata pkg.PythonPackageMetadata
+	}{
+		{
+			Fixture: "test-fixtures/egg-info/PKG-INFO",
+			ExpectedMetadata: pkg.PythonPackageMetadata{
+				Name:                 "requests",
+				Version:              "2.22.0",
+				License:              "Apache 2.0",
+				Platform:             "UNKNOWN",
+				Author:               "Kenneth Reitz",
+				AuthorEmail:          "me@kennethreitz.org",
+				SitePackagesRootPath: "test-fixtures",
+			},
+		},
+		{
+			Fixture: "test-fixtures/dist-info/METADATA",
+			ExpectedMetadata: pkg.PythonPackageMetadata{
+				Name:                 "Pygments",
+				Version:              "2.6.1",
+				License:              "BSD License",
+				Platform:             "any",
+				Author:               "Georg Brandl",
+				AuthorEmail:          "georg@python.org",
+				SitePackagesRootPath: "test-fixtures",
+			},
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.Fixture, func(t *testing.T) {
+			fixture, err := os.Open(test.Fixture)
+			if err != nil {
+				t.Fatalf("failed to open fixture: %+v", err)
+			}
+
+			actual, err := parseWheelOrEggMetadata(file.Path(test.Fixture), fixture)
+			if err != nil {
+				t.Fatalf("failed to parse: %+v", err)
+			}
+
+			for _, d := range deep.Equal(actual, test.ExpectedMetadata) {
+				t.Errorf("diff: %+v", d)
+			}
+		})
+	}
+
+}

syft/cataloger/python/parse_wheel_egg_record.go

@@ -0,0 +1,60 @@
+package python
+
+import (
+	"encoding/csv"
+	"fmt"
+	"io"
+	"strings"
+
+	"github.com/anchore/syft/syft/pkg"
+)
+
+// parseWheelOrEggMetadata takes a Python Egg or Wheel (which share the same format and values for our purposes),
+// returning all Python packages listed.
+func parseWheelOrEggRecord(reader io.Reader) ([]pkg.PythonFileRecord, error) {
+	var records []pkg.PythonFileRecord
+	r := csv.NewReader(reader)
+
+	for {
+		recordList, err := r.Read()
+		if err == io.EOF {
+			break
+		}
+		if err != nil {
+			return nil, fmt.Errorf("unable to read python record file: %w", err)
+		}
+
+		if len(recordList) != 3 {
+			return nil, fmt.Errorf("python record an unexpected length=%d: %q", len(recordList), recordList)
+		}
+
+		var record pkg.PythonFileRecord
+
+		for idx, item := range recordList {
+			switch idx {
+			case 0:
+				record.Path = item
+			case 1:
+				if item == "" {
+					continue
+				}
+				fields := strings.Split(item, "=")
+
+				if len(fields) != 2 {
+					return nil, fmt.Errorf("unexpected python record digest: %q", item)
+				}
+
+				record.Digest = &pkg.Digest{
+					Algorithm: fields[0],
+					Value:     fields[1],
+				}
+			case 2:
+				record.Size = item
+			}
+		}
+
+		records = append(records, record)
+	}
+
+	return records, nil
+}

syft/cataloger/python/parse_wheel_egg_record_test.go

@@ -0,0 +1,57 @@
+package python
+
+import (
+	"os"
+	"testing"
+
+	"github.com/anchore/syft/syft/pkg"
+	"github.com/go-test/deep"
+)
+
+func TestParseWheelEggRecord(t *testing.T) {
+	tests := []struct {
+		Fixture          string
+		ExpectedMetadata []pkg.PythonFileRecord
+	}{
+		{
+			Fixture: "test-fixtures/egg-info/RECORD",
+			ExpectedMetadata: []pkg.PythonFileRecord{
+				{Path: "requests-2.22.0.dist-info/INSTALLER", Digest: &pkg.Digest{"sha256", "zuuue4knoyJ-UwPPXg8fezS7VCrXJQrAP7zeNuwvFQg"}, Size: "4"},
+				{Path: "requests/__init__.py", Digest: &pkg.Digest{"sha256", "PnKCgjcTq44LaAMzB-7--B2FdewRrE8F_vjZeaG9NhA"}, Size: "3921"},
+				{Path: "requests/__pycache__/__version__.cpython-38.pyc"},
+				{Path: "requests/__pycache__/utils.cpython-38.pyc"},
+				{Path: "requests/__version__.py", Digest: &pkg.Digest{"sha256", "Bm-GFstQaFezsFlnmEMrJDe8JNROz9n2XXYtODdvjjc"}, Size: "436"},
+				{Path: "requests/utils.py", Digest: &pkg.Digest{"sha256", "LtPJ1db6mJff2TJSJWKi7rBpzjPS3mSOrjC9zRhoD3A"}, Size: "30049"},
+			},
+		},
+		{
+			Fixture: "test-fixtures/dist-info/RECORD",
+			ExpectedMetadata: []pkg.PythonFileRecord{
+				{Path: "../../../bin/pygmentize", Digest: &pkg.Digest{"sha256", "dDhv_U2jiCpmFQwIRHpFRLAHUO4R1jIJPEvT_QYTFp8"}, Size: "220"},
+				{Path: "Pygments-2.6.1.dist-info/AUTHORS", Digest: &pkg.Digest{"sha256", "PVpa2_Oku6BGuiUvutvuPnWGpzxqFy2I8-NIrqCvqUY"}, Size: "8449"},
+				{Path: "Pygments-2.6.1.dist-info/RECORD"},
+				{Path: "pygments/__pycache__/__init__.cpython-38.pyc"},
+				{Path: "pygments/util.py", Digest: &pkg.Digest{"sha256", "586xXHiJGGZxqk5PMBu3vBhE68DLuAe5MBARWrSPGxA"}, Size: "10778"},
+			},
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.Fixture, func(t *testing.T) {
+			fixture, err := os.Open(test.Fixture)
+			if err != nil {
+				t.Fatalf("failed to open fixture: %+v", err)
+			}
+
+			actual, err := parseWheelOrEggRecord(fixture)
+			if err != nil {
+				t.Fatalf("failed to parse: %+v", err)
+			}
+
+			for _, d := range deep.Equal(actual, test.ExpectedMetadata) {
+				t.Errorf("diff: %+v", d)
+			}
+		})
+	}
+
+}

syft/cataloger/python/parse_wheel_egg_test.go

@@ -1,93 +0,0 @@
-package python
-
-import (
-	"os"
-	"testing"
-
-	"github.com/anchore/syft/syft/pkg"
-)
-
-func assertPkgsEqual(t *testing.T, actual []pkg.Package, expected map[string]pkg.Package) {
-	t.Helper()
-	if len(actual) != len(expected) {
-		for _, a := range actual {
-			t.Log("   ", a)
-		}
-		t.Fatalf("unexpected package count: %d!=%d", len(actual), len(expected))
-	}
-
-	for _, a := range actual {
-		expectedPkg, ok := expected[a.Name]
-		if !ok {
-			t.Errorf("unexpected package found: '%s'", a.Name)
-		}
-
-		if expectedPkg.Version != a.Version {
-			t.Errorf("unexpected package version: '%s'", a.Version)
-		}
-
-		if a.Language != expectedPkg.Language {
-			t.Errorf("bad language: '%+v'", a.Language)
-		}
-
-		if a.Type != expectedPkg.Type {
-			t.Errorf("bad package type: %+v", a.Type)
-		}
-
-		if len(a.Licenses) < len(expectedPkg.Licenses) {
-			t.Errorf("bad package licenses count: '%+v'", a.Licenses)
-		}
-		if len(a.Licenses) > 0 {
-			if a.Licenses[0] != expectedPkg.Licenses[0] {
-				t.Errorf("bad package licenses: '%+v'", a.Licenses)
-			}
-		}
-
-	}
-}
-
-func TestParseEggMetadata(t *testing.T) {
-	expected := map[string]pkg.Package{
-		"requests": {
-			Name:     "requests",
-			Version:  "2.22.0",
-			Language: pkg.Python,
-			Type:     pkg.EggPkg,
-			Licenses: []string{"Apache 2.0"},
-		},
-	}
-	fixture, err := os.Open("test-fixtures/egg-info/PKG-INFO")
-	if err != nil {
-		t.Fatalf("failed to open fixture: %+v", err)
-	}
-
-	actual, err := parseEggMetadata(fixture.Name(), fixture)
-	if err != nil {
-		t.Fatalf("failed to parse egg-info: %+v", err)
-	}
-
-	assertPkgsEqual(t, actual, expected)
-}
-
-func TestParseWheelMetadata(t *testing.T) {
-	expected := map[string]pkg.Package{
-		"Pygments": {
-			Name:     "Pygments",
-			Version:  "2.6.1",
-			Language: pkg.Python,
-			Type:     pkg.WheelPkg,
-			Licenses: []string{"BSD License"},
-		},
-	}
-	fixture, err := os.Open("test-fixtures/dist-info/METADATA")
-	if err != nil {
-		t.Fatalf("failed to open fixture: %+v", err)
-	}
-
-	actual, err := parseWheelMetadata(fixture.Name(), fixture)
-	if err != nil {
-		t.Fatalf("failed to parse dist-info: %+v", err)
-	}
-
-	assertPkgsEqual(t, actual, expected)
-}

syft/cataloger/python/poetry_metadata_package.go

@@ -16,6 +16,6 @@ func (p PoetryMetadataPackage) Pkg() pkg.Package {
 		Name:     p.Name,
 		Version:  p.Version,
 		Language: pkg.Python,
-		Type:     pkg.PoetryPkg,
+		Type:     pkg.PythonPkg,
 	}
 }

syft/cataloger/python/test-fixtures/dist-info/RECORD

@@ -0,0 +1,5 @@
+../../../bin/pygmentize,sha256=dDhv_U2jiCpmFQwIRHpFRLAHUO4R1jIJPEvT_QYTFp8,220
+Pygments-2.6.1.dist-info/AUTHORS,sha256=PVpa2_Oku6BGuiUvutvuPnWGpzxqFy2I8-NIrqCvqUY,8449
+Pygments-2.6.1.dist-info/RECORD,,
+pygments/__pycache__/__init__.cpython-38.pyc,,
+pygments/util.py,sha256=586xXHiJGGZxqk5PMBu3vBhE68DLuAe5MBARWrSPGxA,10778

syft/cataloger/python/test-fixtures/dist-info/top_level.txt

@@ -0,0 +1,2 @@
+pygments
+something_else

syft/cataloger/python/test-fixtures/egg-info/RECORD

@@ -0,0 +1,6 @@
+requests-2.22.0.dist-info/INSTALLER,sha256=zuuue4knoyJ-UwPPXg8fezS7VCrXJQrAP7zeNuwvFQg,4
+requests/__init__.py,sha256=PnKCgjcTq44LaAMzB-7--B2FdewRrE8F_vjZeaG9NhA,3921
+requests/__pycache__/__version__.cpython-38.pyc,,
+requests/__pycache__/utils.cpython-38.pyc,,
+requests/__version__.py,sha256=Bm-GFstQaFezsFlnmEMrJDe8JNROz9n2XXYtODdvjjc,436
+requests/utils.py,sha256=LtPJ1db6mJff2TJSJWKi7rBpzjPS3mSOrjC9zRhoD3A,30049

syft/cataloger/python/test-fixtures/egg-info/top_level.txt

@@ -0,0 +1 @@
+requests

syft/cataloger/python/test-fixtures/partial.dist-info/METADATA

@@ -0,0 +1,47 @@
+Metadata-Version: 2.1
+Name: Pygments
+Version: 2.6.1
+Summary: Pygments is a syntax highlighting package written in Python.
+Home-page: https://pygments.org/
+Author: Georg Brandl
+Author-email: georg@python.org
+License: BSD License
+Keywords: syntax highlighting
+Platform: any
+Classifier: License :: OSI Approved :: BSD License
+Classifier: Intended Audience :: Developers
+Classifier: Intended Audience :: End Users/Desktop
+Classifier: Intended Audience :: System Administrators
+Classifier: Development Status :: 6 - Mature
+Classifier: Programming Language :: Python
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.5
+Classifier: Programming Language :: Python :: 3.6
+Classifier: Programming Language :: Python :: 3.7
+Classifier: Programming Language :: Python :: 3.8
+Classifier: Programming Language :: Python :: Implementation :: CPython
+Classifier: Programming Language :: Python :: Implementation :: PyPy
+Classifier: Operating System :: OS Independent
+Classifier: Topic :: Text Processing :: Filters
+Classifier: Topic :: Utilities
+Requires-Python: >=3.5
+
+
+Pygments
+~~~~~~~~
+
+Pygments is a syntax highlighting package written in Python.
+
+It is a generic syntax highlighter suitable for use in code hosting, forums,
+wikis or other applications that need to prettify source code.  Highlights
+are:
+
+* a wide range of over 500 languages and other text formats is supported
+* special attention is paid to details, increasing quality by a fair amount
+* support for new languages and formats are added easily
+* a number of output formats, presently HTML, LaTeX, RTF, SVG, all image     formats that PIL supports and ANSI sequences
+* it is usable as a command-line tool and as a library
+
+:copyright: Copyright 2006-2019 by the Pygments team, see AUTHORS.
+:license: BSD, see LICENSE for details.
+

syft/cataloger/rpmdb/cataloger.go

@@ -4,13 +4,47 @@ Package rpmdb provides a concrete Cataloger implementation for RPM "Package" DB
 package rpmdb
 
 import (
-	"github.com/anchore/syft/syft/cataloger/common"
+	"fmt"
+	"strings"
+
+	"github.com/anchore/syft/syft/pkg"
+	"github.com/anchore/syft/syft/scope"
 )
 
+const (
+	packagesGlob = "**/var/lib/rpm/Packages"
+)
+
+type Cataloger struct{}
+
 // NewRpmdbCataloger returns a new RPM DB cataloger object.
-func NewRpmdbCataloger() *common.GenericCataloger {
-	globParsers := map[string]common.ParserFn{
-		"**/var/lib/rpm/Packages": parseRpmDB,
+func NewRpmdbCataloger() *Cataloger {
+	return &Cataloger{}
+}
+
+// Name returns a string that uniquely describes a cataloger
+func (c *Cataloger) Name() string {
+	return "rpmdb-cataloger"
+}
+
+// Catalog is given an object to resolve file references and content, this function returns any discovered Packages after analyzing rpm db installation.
+func (c *Cataloger) Catalog(resolver scope.Resolver) ([]pkg.Package, error) {
+	fileMatches, err := resolver.FilesByGlob(packagesGlob)
+	if err != nil {
+		return nil, fmt.Errorf("failed to find rpmdb's by glob: %w", err)
 	}
-	return common.NewGenericCataloger(nil, globParsers, "rpmdb-cataloger")
+
+	var pkgs []pkg.Package
+	for _, ref := range fileMatches {
+		dbContents, err := resolver.FileContentsByRef(ref)
+		if err != nil {
+			return nil, err
+		}
+
+		pkgs, err = parseRpmDB(resolver, strings.NewReader(dbContents))
+		if err != nil {
+			return nil, fmt.Errorf("unable to catalog rpmdb package=%+v: %w", ref.Path, err)
+		}
+	}
+	return pkgs, nil
 }

syft/cataloger/rpmdb/parse_rpmdb.go

@@ -6,18 +6,18 @@ import (
 	"io/ioutil"
 	"os"
 
+	"github.com/anchore/stereoscope/pkg/file"
+
+	"github.com/anchore/syft/syft/scope"
+
 	rpmdb "github.com/anchore/go-rpmdb/pkg"
 	"github.com/anchore/syft/internal"
 	"github.com/anchore/syft/internal/log"
-	"github.com/anchore/syft/syft/cataloger/common"
 	"github.com/anchore/syft/syft/pkg"
 )
 
-// integrity check
-var _ common.ParserFn = parseRpmDB
-
 // parseApkDb parses an "Packages" RPM DB and returns the Packages listed within it.
-func parseRpmDB(_ string, reader io.Reader) ([]pkg.Package, error) {
+func parseRpmDB(resolver scope.FileResolver, reader io.Reader) ([]pkg.Package, error) {
 	f, err := ioutil.TempFile("", internal.ApplicationName+"-rpmdb")
 	if err != nil {
 		return nil, fmt.Errorf("failed to create temp rpmdb file: %w", err)
@@ -48,12 +48,18 @@ func parseRpmDB(_ string, reader io.Reader) ([]pkg.Package, error) {
 	allPkgs := make([]pkg.Package, 0)
 
 	for _, entry := range pkgList {
+		records, err := extractRpmdbFileRecords(resolver, entry)
+		if err != nil {
+			return nil, err
+		}
+
 		p := pkg.Package{
 			Name:    entry.Name,
 			Version: fmt.Sprintf("%s-%s", entry.Version, entry.Release), // this is what engine does
 			//Version: fmt.Sprintf("%d:%s-%s.%s", entry.Epoch, entry.Version, entry.Release, entry.Arch),
-			Type: pkg.RpmPkg,
-			Metadata: pkg.RpmMetadata{
+			Type:         pkg.RpmPkg,
+			MetadataType: pkg.RpmdbMetadataType,
+			Metadata: pkg.RpmdbMetadata{
 				Name:      entry.Name,
 				Version:   entry.Version,
 				Epoch:     entry.Epoch,
@@ -63,6 +69,7 @@ func parseRpmDB(_ string, reader io.Reader) ([]pkg.Package, error) {
 				Vendor:    entry.Vendor,
 				License:   entry.License,
 				Size:      entry.Size,
+				Files:     records,
 			},
 		}
 
@@ -71,3 +78,26 @@ func parseRpmDB(_ string, reader io.Reader) ([]pkg.Package, error) {
 
 	return allPkgs, nil
 }
+
+func extractRpmdbFileRecords(resolver scope.FileResolver, entry *rpmdb.PackageInfo) ([]pkg.RpmdbFileRecord, error) {
+	var records = make([]pkg.RpmdbFileRecord, 0)
+
+	for _, record := range entry.Files {
+		refs, err := resolver.FilesByPath(file.Path(record.Path))
+		if err != nil {
+			return nil, fmt.Errorf("failed to resolve path=%+v: %w", record.Path, err)
+		}
+		//only persist RPMDB file records which exist in the image/directory, otherwise ignore them
+		if len(refs) == 0 {
+			continue
+		}
+
+		records = append(records, pkg.RpmdbFileRecord{
+			Path:   record.Path,
+			Mode:   pkg.RpmdbFileMode(record.Mode),
+			Size:   int(record.Size),
+			SHA256: record.SHA256,
+		})
+	}
+	return records, nil
+}

syft/cataloger/rpmdb/parse_rpmdb_test.go

@@ -1,56 +1,141 @@
 package rpmdb
 
 import (
-	"github.com/anchore/syft/syft/pkg"
-	"github.com/go-test/deep"
+	"fmt"
 	"os"
 	"testing"
+
+	"github.com/anchore/stereoscope/pkg/file"
+	"github.com/anchore/syft/syft/pkg"
+	"github.com/go-test/deep"
 )
 
+type rpmdbTestFileResolverMock struct {
+	ignorePaths bool
+}
+
+func newTestFileResolver(ignorePaths bool) *rpmdbTestFileResolverMock {
+	return &rpmdbTestFileResolverMock{
+		ignorePaths: ignorePaths,
+	}
+}
+
+func (r *rpmdbTestFileResolverMock) FilesByPath(paths ...file.Path) ([]file.Reference, error) {
+	if r.ignorePaths {
+		// act as if no paths exist
+		return nil, nil
+	}
+	// act as if all files exist
+	var refs = make([]file.Reference, len(paths))
+	for i, p := range paths {
+		refs[i] = file.NewFileReference(p)
+	}
+	return refs, nil
+}
+
+func (r *rpmdbTestFileResolverMock) FilesByGlob(...string) ([]file.Reference, error) {
+	return nil, fmt.Errorf("not implemented")
+}
+func (r *rpmdbTestFileResolverMock) RelativeFileByPath(file.Reference, string) (*file.Reference, error) {
+	return nil, fmt.Errorf("not implemented")
+}
+
 func TestParseRpmDB(t *testing.T) {
-	expected := map[string]pkg.Package{
-		"dive": {
-			Name:    "dive",
-			Version: "0.9.2-1",
-			Type:    pkg.RpmPkg,
-			Metadata: pkg.RpmMetadata{
-				Name:      "dive",
-				Epoch:     0,
-				Arch:      "x86_64",
-				Release:   "1",
-				Version:   "0.9.2",
-				SourceRpm: "dive-0.9.2-1.src.rpm",
-				Size:      12406784,
-				License:   "MIT",
-				Vendor:    "",
+	tests := []struct {
+		fixture     string
+		expected    map[string]pkg.Package
+		ignorePaths bool
+	}{
+		{
+			fixture: "test-fixtures/Packages",
+			// we only surface package paths for files that exist (here we DO NOT expect a path)
+			ignorePaths: true,
+			expected: map[string]pkg.Package{
+				"dive": {
+					Name:         "dive",
+					Version:      "0.9.2-1",
+					Type:         pkg.RpmPkg,
+					MetadataType: pkg.RpmdbMetadataType,
+					Metadata: pkg.RpmdbMetadata{
+						Name:      "dive",
+						Epoch:     0,
+						Arch:      "x86_64",
+						Release:   "1",
+						Version:   "0.9.2",
+						SourceRpm: "dive-0.9.2-1.src.rpm",
+						Size:      12406784,
+						License:   "MIT",
+						Vendor:    "",
+						Files:     []pkg.RpmdbFileRecord{},
+					},
+				},
+			},
+		},
+		{
+			fixture: "test-fixtures/Packages",
+			// we only surface package paths for files that exist (here we expect a path)
+			ignorePaths: false,
+			expected: map[string]pkg.Package{
+				"dive": {
+					Name:         "dive",
+					Version:      "0.9.2-1",
+					Type:         pkg.RpmPkg,
+					MetadataType: pkg.RpmdbMetadataType,
+					Metadata: pkg.RpmdbMetadata{
+						Name:      "dive",
+						Epoch:     0,
+						Arch:      "x86_64",
+						Release:   "1",
+						Version:   "0.9.2",
+						SourceRpm: "dive-0.9.2-1.src.rpm",
+						Size:      12406784,
+						License:   "MIT",
+						Vendor:    "",
+						Files: []pkg.RpmdbFileRecord{
+							{
+								Path:   "/usr/local/bin/dive",
+								Mode:   33261,
+								Size:   12406784,
+								SHA256: "81d29f327ba23096b3c52ff6fe1c425641e618bc87b5c05ee377edc650afaa55",
+							},
+						},
+					},
+				},
 			},
 		},
 	}
 
-	fixture, err := os.Open("test-fixtures/Packages")
-	if err != nil {
-		t.Fatalf("failed to open fixture: %+v", err)
-	}
-
-	actual, err := parseRpmDB(fixture.Name(), fixture)
-	if err != nil {
-		t.Fatalf("failed to parse rpmdb: %+v", err)
-	}
-
-	if len(actual) != len(expected) {
-		for _, a := range actual {
-			t.Log("   ", a)
-		}
-		t.Fatalf("unexpected package count: %d!=%d", len(actual), len(expected))
-	}
-
-	for _, a := range actual {
-		e := expected[a.Name]
-		diffs := deep.Equal(a, e)
-		if len(diffs) > 0 {
-			for _, d := range diffs {
-				t.Errorf("diff: %+v", d)
+	for _, test := range tests {
+		t.Run(test.fixture, func(t *testing.T) {
+			fixture, err := os.Open(test.fixture)
+			if err != nil {
+				t.Fatalf("failed to open fixture: %+v", err)
 			}
-		}
+
+			fileResolver := newTestFileResolver(test.ignorePaths)
+
+			actual, err := parseRpmDB(fileResolver, fixture)
+			if err != nil {
+				t.Fatalf("failed to parse rpmdb: %+v", err)
+			}
+
+			if len(actual) != len(test.expected) {
+				for _, a := range actual {
+					t.Log("   ", a)
+				}
+				t.Fatalf("unexpected package count: %d!=%d", len(actual), len(test.expected))
+			}
+
+			for _, a := range actual {
+				e := test.expected[a.Name]
+				diffs := deep.Equal(a, e)
+				if len(diffs) > 0 {
+					for _, d := range diffs {
+						t.Errorf("diff: %+v", d)
+					}
+				}
+			}
+		})
 	}
+
 }

syft/cataloger/ruby/parse_gemspec.go

@@ -8,6 +8,8 @@ import (
 	"regexp"
 	"strings"
 
+	"github.com/anchore/syft/internal"
+
 	"github.com/mitchellh/mapstructure"
 
 	"github.com/anchore/syft/syft/cataloger/common"
@@ -76,7 +78,7 @@ func parseGemSpecEntries(_ string, reader io.Reader) ([]pkg.Package, error) {
 		}
 
 		for field, pattern := range patterns {
-			matchMap := matchCaptureGroups(pattern, sanitizedLine)
+			matchMap := internal.MatchCaptureGroups(pattern, sanitizedLine)
 			if value := matchMap[field]; value != "" {
 				if postProcessor := postProcessors[field]; postProcessor != nil {
 					fields[field] = postProcessor(value)
@@ -96,12 +98,13 @@ func parseGemSpecEntries(_ string, reader io.Reader) ([]pkg.Package, error) {
 		}
 
 		pkgs = append(pkgs, pkg.Package{
-			Name:     metadata.Name,
-			Version:  metadata.Version,
-			Licenses: metadata.Licenses,
-			Language: pkg.Ruby,
-			Type:     pkg.GemPkg,
-			Metadata: metadata,
+			Name:         metadata.Name,
+			Version:      metadata.Version,
+			Licenses:     metadata.Licenses,
+			Language:     pkg.Ruby,
+			Type:         pkg.GemPkg,
+			MetadataType: pkg.GemMetadataType,
+			Metadata:     metadata,
 		})
 	}
 
@@ -121,15 +124,3 @@ func renderUtf8(s string) string {
 	})
 	return fullReplacement
 }
-
-// matchCaptureGroups takes a regular expression and string and returns all of the named capture group results in a map.
-func matchCaptureGroups(regEx *regexp.Regexp, str string) map[string]string {
-	match := regEx.FindStringSubmatch(str)
-	results := make(map[string]string)
-	for i, name := range regEx.SubexpNames() {
-		if i > 0 && i <= len(match) {
-			results[name] = match[i]
-		}
-	}
-	return results
-}

syft/cataloger/ruby/parse_gemspec_test.go

@@ -10,11 +10,12 @@ import (
 
 func TestParseGemspec(t *testing.T) {
 	var expectedPkg = pkg.Package{
-		Name:     "bundler",
-		Version:  "2.1.4",
-		Type:     pkg.GemPkg,
-		Licenses: []string{"MIT"},
-		Language: pkg.Ruby,
+		Name:         "bundler",
+		Version:      "2.1.4",
+		Type:         pkg.GemPkg,
+		Licenses:     []string{"MIT"},
+		Language:     pkg.Ruby,
+		MetadataType: pkg.GemMetadataType,
 		Metadata: pkg.GemMetadata{
 			Name:     "bundler",
 			Version:  "2.1.4",

syft/distro/distro.go

@@ -10,6 +10,7 @@ type Distro struct {
 	Type       Type
 	Version    *hashiVer.Version
 	RawVersion string
+	IDLike     string
 }
 
 // NewUnknownDistro creates a standardized Distro object for unidentifiable distros
@@ -19,7 +20,7 @@ func NewUnknownDistro() Distro {
 	}
 }
 
-func NewDistro(t Type, ver string) (Distro, error) {
+func NewDistro(t Type, ver, like string) (Distro, error) {
 	if ver == "" {
 		return Distro{Type: t}, nil
 	}
@@ -31,6 +32,7 @@ func NewDistro(t Type, ver string) (Distro, error) {
 		Type:       t,
 		Version:    verObj,
 		RawVersion: ver,
+		IDLike:     like,
 	}, nil
 }
 

syft/distro/distro_test.go

@@ -33,7 +33,7 @@ func TestDistro_FullVersion(t *testing.T) {
 	for _, test := range tests {
 		name := fmt.Sprintf("%s:%s", test.dist, test.version)
 		t.Run(name, func(t *testing.T) {
-			d, err := NewDistro(test.dist, test.version)
+			d, err := NewDistro(test.dist, test.version, "")
 			if err != nil {
 				t.Errorf("could not create distro='%+v:%+v': %+v", test.dist, test.version, err)
 			}
@@ -53,29 +53,34 @@ func TestDistro_MajorVersion(t *testing.T) {
 		dist     Type
 		version  string
 		expected string
+		like     string
 	}{
 		{
 			version:  "8",
 			expected: "8",
+			like:     "",
 		},
 		{
 			version:  "18.04",
 			expected: "18",
+			like:     "debian",
 		},
 		{
 			version:  "0",
 			expected: "0",
+			like:     "",
 		},
 		{
 			version:  "18.1.2",
 			expected: "18",
+			like:     "debian",
 		},
 	}
 
 	for _, test := range tests {
 		name := fmt.Sprintf("%s:%s", test.dist, test.version)
 		t.Run(name, func(t *testing.T) {
-			d, err := NewDistro(test.dist, test.version)
+			d, err := NewDistro(test.dist, test.version, test.like)
 			if err != nil {
 				t.Errorf("could not create distro='%+v:%+v': %+v", test.dist, test.version, err)
 			}

syft/distro/identify.go

@@ -81,7 +81,7 @@ identifyLoop:
 	return distro
 }
 
-func assemble(name, version string) *Distro {
+func assemble(name, version, like string) *Distro {
 	distroType, ok := IDMapping[name]
 
 	// Both distro and version must be present
@@ -90,7 +90,7 @@ func assemble(name, version string) *Distro {
 	}
 
 	if ok {
-		distro, err := NewDistro(distroType, version)
+		distro, err := NewDistro(distroType, version, like)
 		if err != nil {
 			return nil
 		}
@@ -101,7 +101,7 @@ func assemble(name, version string) *Distro {
 }
 
 func parseOsRelease(contents string) *Distro {
-	id, vers := "", ""
+	id, vers, like := "", "", ""
 	for _, line := range strings.Split(contents, "\n") {
 		parts := strings.Split(line, "=")
 		prefix := parts[0]
@@ -112,10 +112,12 @@ func parseOsRelease(contents string) *Distro {
 			id = strings.TrimSpace(value)
 		case "VERSION_ID":
 			vers = strings.TrimSpace(value)
+		case "ID_LIKE":
+			like = strings.TrimSpace(value)
 		}
 	}
 
-	return assemble(id, vers)
+	return assemble(id, vers, like)
 }
 
 var busyboxVersionMatcher = regexp.MustCompile(`BusyBox v[\d\.]+`)
@@ -125,7 +127,7 @@ func parseBusyBox(contents string) *Distro {
 	for _, match := range matches {
 		parts := strings.Split(match, " ")
 		version := strings.ReplaceAll(parts[1], "v", "")
-		distro := assemble("busybox", version)
+		distro := assemble("busybox", version, "")
 		if distro != nil {
 			return distro
 		}

syft/logger/logger.go

@@ -5,6 +5,9 @@ package logger
 
 type Logger interface {
 	Errorf(format string, args ...interface{})
+	Error(args ...interface{})
+	Warnf(format string, args ...interface{})
+	Warn(args ...interface{})
 	Infof(format string, args ...interface{})
 	Info(args ...interface{})
 	Debugf(format string, args ...interface{})

syft/pkg/apk_metadata.go

@@ -29,10 +29,10 @@ type ApkMetadata struct {
 // ApkFileRecord represents a single file listing and metadata from a APK DB entry (which may have many of these file records).
 type ApkFileRecord struct {
 	Path        string `json:"path"`
-	OwnerUID    string `json:"ownerUid"`
-	OwnerGUI    string `json:"ownerGid"`
-	Permissions string `json:"permissions"`
-	Checksum    string `json:"checksum"`
+	OwnerUID    string `json:"ownerUid,omitempty"`
+	OwnerGID    string `json:"ownerGid,omitempty"`
+	Permissions string `json:"permissions,omitempty"`
+	Checksum    string `json:"checksum,omitempty"`
 }
 
 func (m ApkMetadata) PackageURL() string {