remove unstable CLI test (bad binaries) (#721)

Signed-off-by: Alex Goodman <alex.goodman@anchore.com>

.github/scripts/goreleaser-install.sh

@@ -338,7 +338,7 @@ hash_sha256_verify() {
 		return 1
 	fi
 	BASENAME=${TARGET##*/}
-	want=$(grep "${BASENAME}" "${checksums}" 2>/dev/null | tr '\t' ' ' | cut -d ' ' -f 1)
+	want=$(grep "${BASENAME}\$" "${checksums}" 2>/dev/null | tr '\t' ' ' | cut -d ' ' -f 1)
 	if [ -z "$want" ]; then
 		log_err "hash_sha256_verify unable to find checksum for '${TARGET}' in '${checksums}'"
 		return 1

Makefile

@@ -82,7 +82,7 @@ help:
 
 .PHONY: ci-bootstrap
 ci-bootstrap:
-	DEBIAN_FRONTEND=noninteractive sudo apt update && sudo -E apt install -y bc jq libxml2-utils && go install github.com/neilpa/yajsv@latest
+	DEBIAN_FRONTEND=noninteractive sudo apt update && sudo -E apt install -y bc jq libxml2-utils
 
 .PHONY:
 ci-bootstrap-mac:
@@ -100,7 +100,8 @@ bootstrap-tools: $(TEMPDIR)
 	curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(TEMPDIR)/ v1.42.1
 	curl -sSfL https://raw.githubusercontent.com/wagoodman/go-bouncer/master/bouncer.sh | sh -s -- -b $(TEMPDIR)/ v0.2.0
 	curl -sSfL https://raw.githubusercontent.com/anchore/chronicle/main/install.sh | sh -s -- -b $(TEMPDIR)/ v0.3.0
-	.github/scripts/goreleaser-install.sh -b $(TEMPDIR)/ v0.177.0
+	.github/scripts/goreleaser-install.sh -b $(TEMPDIR)/ v1.2.2
+	GOBIN="$(shell realpath $(TEMPDIR))" go install github.com/neilpa/yajsv@v1.4.0
 
 .PHONY: bootstrap-go
 bootstrap-go:

README.md

@@ -93,6 +93,23 @@ file:path/to/yourproject/file          read directly from a path on disk (any si
 registry:yourrepo/yourimage:tag        pull image directly from a registry (no container runtime required)
 ```
 
+### Excluding file paths
+
+Syft can exclude files and paths from being scanned within a source by using glob expressions
+with one or more `--exclude` parameters:
+```
+syft <source> --exclude './out/**/*.json' --exclude /etc
+```
+**Note:** in the case of _image scanning_, since the entire filesystem is scanned it is
+possible to use absolute paths like `/etc` or `/usr/**/*.txt` whereas _directory scans_
+exclude files _relative to the specified directory_. For example: scanning `/usr/foo` with
+`--exclude ./package.json` would exclude `/usr/foo/package.json` and `--exclude '**/package.json'`
+would exclude all `package.json` files under `/usr/foo`. For _directory scans_,
+it is required to begin path expressions with `./`, `*/`, or `**/`, all of which
+will be resolved _relative to the specified scan directory_. Keep in mind, your shell
+may attempt to expand wildcards, so put those parameters in single quotes, like:
+`'**/*.json'`.
+
 ### Output formats
 
 The output format for Syft is configurable as well:
@@ -104,7 +121,8 @@ syft packages <image> -o <format>
 Where the `formats` available are:
 - `json`: Use this to get as much information out of Syft as possible!
 - `text`: A row-oriented, human-and-machine-friendly output.
-- `cyclonedx`: A XML report conforming to the [CycloneDX 1.2 specification](https://cyclonedx.org/specification/overview/).
+- `cyclonedx`: A XML report conforming to the [CycloneDX 1.3 specification](https://cyclonedx.org/specification/overview/).
+- `cyclonedx-json`: A JSON report conforming to the [CycloneDX 1.3 specification](https://cyclonedx.org/specification/overview/).
 - `spdx`: A tag-value formatted report conforming to the [SPDX 2.2 specification](https://spdx.github.io/spdx-spec/).
 - `spdx-json`: A JSON report conforming to the [SPDX 2.2 JSON Schema](https://github.com/spdx/spdx-spec/blob/v2.2/schemas/spdx-schema.json).
 - `table`: A columnar summary (default).
@@ -218,6 +236,12 @@ file: ""
 # same as SYFT_CHECK_FOR_APP_UPDATE env var
 check-for-app-update: true
 
+# a list of globs to exclude from scanning. same as --exclude ; for example:
+# exclude:
+#   - '/etc/**'
+#   - './out/**/*.json'
+exclude:
+
 # cataloging packages is exposed through the packages and power-user subcommands
 package:
   cataloger:

cmd/check_for_application_update.go

@@ -11,6 +11,7 @@ import (
 
 func checkForApplicationUpdate() {
 	if appConfig.CheckForAppUpdate {
+		log.Debugf("checking if new vesion of %s is available", internal.ApplicationName)
 		isAvailable, newVersion, err := version.IsUpdateAvailable()
 		if err != nil {
 			// this should never stop the application

cmd/cmd.go

@@ -1,17 +1,19 @@
 package cmd
 
 import (
+	"encoding/json"
 	"fmt"
 	"os"
-
-	"github.com/spf13/cobra"
+	"sort"
 
 	"github.com/anchore/stereoscope"
 	"github.com/anchore/syft/internal/config"
 	"github.com/anchore/syft/internal/log"
 	"github.com/anchore/syft/internal/logger"
+	"github.com/anchore/syft/internal/version"
 	"github.com/anchore/syft/syft"
 	"github.com/gookit/color"
+	"github.com/spf13/cobra"
 	"github.com/spf13/viper"
 	"github.com/wagoodman/go-partybus"
 )
@@ -28,6 +30,8 @@ func init() {
 		initAppConfig,
 		initLogging,
 		logAppConfig,
+		checkForApplicationUpdate,
+		logAppVersion,
 		initEventBus,
 	)
 }
@@ -110,3 +114,33 @@ func initEventBus() {
 	stereoscope.SetBus(eventBus)
 	syft.SetBus(eventBus)
 }
+
+func logAppVersion() {
+	versionInfo := version.FromBuild()
+	log.Infof("syft version: %s", versionInfo.Version)
+
+	var fields map[string]interface{}
+	bytes, err := json.Marshal(versionInfo)
+	if err != nil {
+		return
+	}
+	err = json.Unmarshal(bytes, &fields)
+	if err != nil {
+		return
+	}
+
+	keys := make([]string, 0, len(fields))
+	for k := range fields {
+		keys = append(keys, k)
+	}
+	sort.Strings(keys)
+
+	for idx, field := range keys {
+		value := fields[field]
+		branch := "├──"
+		if idx == len(fields)-1 {
+			branch = "└──"
+		}
+		log.Debugf("  %s %s: %s", branch, field, value)
+	}
+}

cmd/packages.go

@@ -132,6 +132,11 @@ func setPackageFlags(flags *pflag.FlagSet) {
 		"include dockerfile for upload to Anchore Enterprise",
 	)
 
+	flags.StringArrayP(
+		"exclude", "", nil,
+		"exclude paths from being scanned using a glob expression",
+	)
+
 	flags.Bool(
 		"overwrite-existing-image", false,
 		"overwrite an existing image during the upload to Anchore Enterprise",
@@ -158,6 +163,10 @@ func bindPackagesConfigOptions(flags *pflag.FlagSet) error {
 		return err
 	}
 
+	if err := viper.BindPFlag("exclude", flags.Lookup("exclude")); err != nil {
+		return err
+	}
+
 	// Upload options //////////////////////////////////////////////////////////
 
 	if err := viper.BindPFlag("anchore.host", flags.Lookup("host")); err != nil {
@@ -251,11 +260,9 @@ func packagesExecWorker(userInput string) <-chan error {
 			return
 		}
 
-		checkForApplicationUpdate()
-
-		src, cleanup, err := source.New(userInput, appConfig.Registry.ToOptions())
+		src, cleanup, err := source.New(userInput, appConfig.Registry.ToOptions(), appConfig.Exclusions)
 		if err != nil {
-			errs <- fmt.Errorf("failed to determine image source: %w", err)
+			errs <- fmt.Errorf("failed to construct source from user input %q: %w", userInput, err)
 			return
 		}
 		if cleanup != nil {

cmd/power_user.go

@@ -111,9 +111,7 @@ func powerUserExecWorker(userInput string) <-chan error {
 			return
 		}
 
-		checkForApplicationUpdate()
-
-		src, cleanup, err := source.New(userInput, appConfig.Registry.ToOptions())
+		src, cleanup, err := source.New(userInput, appConfig.Registry.ToOptions(), appConfig.Exclusions)
 		if err != nil {
 			errs <- err
 			return

go.mod

@@ -13,7 +13,7 @@ require (
 	github.com/anchore/go-testutils v0.0.0-20200925183923-d5f45b0d3c04
 	github.com/anchore/go-version v1.2.2-0.20200701162849-18adb9c92b9b
 	github.com/anchore/packageurl-go v0.0.0-20210922164639-b3fa992ebd29
-	github.com/anchore/stereoscope v0.0.0-20211203160213-5a5e323a5c89
+	github.com/anchore/stereoscope v0.0.0-20211222141827-6e663afeef5d
 	// we are hinting brotli to latest due to warning when installing archiver v3:
 	// go: warning: github.com/andybalholm/brotli@v1.0.1: retracted by module author: occasional panics and data corruption
 	github.com/andybalholm/brotli v1.0.4 // indirect

go.sum

@@ -111,8 +111,8 @@ github.com/anchore/go-version v1.2.2-0.20200701162849-18adb9c92b9b h1:e1bmaoJfZV
 github.com/anchore/go-version v1.2.2-0.20200701162849-18adb9c92b9b/go.mod h1:Bkc+JYWjMCF8OyZ340IMSIi2Ebf3uwByOk6ho4wne1E=
 github.com/anchore/packageurl-go v0.0.0-20210922164639-b3fa992ebd29 h1:K9LfnxwhqvihqU0+MF325FNy7fsKV9EGaUxdfR4gnWk=
 github.com/anchore/packageurl-go v0.0.0-20210922164639-b3fa992ebd29/go.mod h1:Oc1UkGaJwY6ND6vtAqPSlYrptKRJngHwkwB6W7l1uP0=
-github.com/anchore/stereoscope v0.0.0-20211203160213-5a5e323a5c89 h1:pcMFN7wjIQIxdZm8NA0R3JiRRFn5Eyx9mhagHOVS4Bw=
-github.com/anchore/stereoscope v0.0.0-20211203160213-5a5e323a5c89/go.mod h1:FNm1rtauEjkC3elA3jr3bJkU+/4QiovApwJdPnHQ9x0=
+github.com/anchore/stereoscope v0.0.0-20211222141827-6e663afeef5d h1:ggdC5SRb8rAXmBkwxah8gfSxwpoAVcpp12tQ+XbDi20=
+github.com/anchore/stereoscope v0.0.0-20211222141827-6e663afeef5d/go.mod h1:FNm1rtauEjkC3elA3jr3bJkU+/4QiovApwJdPnHQ9x0=
 github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883/go.mod h1:rCTlJbsFo29Kk6CurOXKm700vrz8f0KW0JNfpkRJY/8=
 github.com/andybalholm/brotli v1.0.1/go.mod h1:loMXtMfwqflxFJPmdbJO0a3KNoPuLBgiu3qAvBg8x/Y=
 github.com/andybalholm/brotli v1.0.4 h1:V7DdXeJtZscaqfNuAdSRuRFzuiKlHSC/Zh3zl9qY3JY=

install.sh

@@ -1,7 +1,5 @@
 #!/bin/sh
 set -e
-# Code generated by godownloader on 2020-08-10T20:55:46Z.
-#
 
 usage() {
   this=$1
@@ -23,7 +21,7 @@ EOF
 }
 
 parse_args() {
-  #BINDIR is ./bin unless set be ENV
+  # BINDIR is ./bin unless set be ENV
   # over-ridden by flag below
 
   BINDIR=${BINDIR:-./bin}
@@ -67,6 +65,7 @@ execute() {
 }
 get_binaries() {
   case "$PLATFORM" in
+    darwin/arm64) BINARIES="syft" ;;
     darwin/amd64) BINARIES="syft" ;;
     linux/amd64) BINARIES="syft" ;;
     windows/amd64) BINARIES="syft" ;;
@@ -97,6 +96,9 @@ adjust_format() {
     darwin) FORMAT=dmg ;;
     windows) FORMAT=zip ;;
   esac
+  case "${PLATFORM}" in
+    darwin/arm64) FORMAT=zip ;;
+  esac
   true
 }
 adjust_os() {
@@ -244,7 +246,7 @@ extract_from_dmg() {
   dmg_file=$1
   mount_point="/Volumes/tmp-dmg"
   hdiutil attach -quiet -nobrowse -mountpoint "${mount_point}" "${dmg_file}"
-  cp -fR "${mount_point}/" ./
+  cp -fR "${mount_point}/." ./
   hdiutil detach -quiet -force "${mount_point}"
 }
 http_download_curl() {

internal/config/application.go

@@ -44,6 +44,7 @@ type Application struct {
 	FileContents       fileContents       `yaml:"file-contents" json:"file-contents" mapstructure:"file-contents"`
 	Secrets            secrets            `yaml:"secrets" json:"secrets" mapstructure:"secrets"`
 	Registry           registry           `yaml:"registry" json:"registry" mapstructure:"registry"`
+	Exclusions         []string           `yaml:"exclude" json:"exclude" mapstructure:"exclude"`
 }
 
 // PowerUserCatalogerEnabledDefault switches all catalogers to be enabled when running power-user command

internal/constants.go

@@ -6,5 +6,5 @@ const (
 
 	// JSONSchemaVersion is the current schema version output by the JSON presenter
 	// This is roughly following the "SchemaVer" guidelines for versioning the JSON schema. Please see schema/json/README.md for details on how to increment.
-	JSONSchemaVersion = "2.0.1"
+	JSONSchemaVersion = "2.0.2"
 )

internal/formats/spdx22json/test-fixtures/snapshot/TestSPDXJSONDirectoryPresenter.golden

@@ -3,7 +3,7 @@
  "name": "/some/path",
  "spdxVersion": "SPDX-2.2",
  "creationInfo": {
-  "created": "2021-12-01T15:08:29.469369Z",
+  "created": "2021-12-20T19:12:47.869816Z",
   "creators": [
    "Organization: Anchore, Inc",
    "Tool: syft-[not provided]"
@@ -11,10 +11,10 @@
   "licenseListVersion": "3.15"
  },
  "dataLicense": "CC0-1.0",
- "documentNamespace": "https://anchore.com/syft/dir/some/path-f4586501-2da6-4541-a8e9-232b32f25e9a",
+ "documentNamespace": "https://anchore.com/syft/dir/some/path-4b896ded-7852-4e31-b764-136b53bdf346",
  "packages": [
   {
-   "SPDXID": "SPDXRef-2a115ac97d018a0e",
+   "SPDXID": "SPDXRef-1d97af55efe9512f",
    "name": "package-1",
    "licenseConcluded": "MIT",
    "downloadLocation": "NOASSERTION",
@@ -36,7 +36,7 @@
    "versionInfo": "1.0.1"
   },
   {
-   "SPDXID": "SPDXRef-5e920b2bece2c3ae",
+   "SPDXID": "SPDXRef-ad3d1c4abd84bf75",
    "name": "package-2",
    "licenseConcluded": "NONE",
    "downloadLocation": "NOASSERTION",

internal/formats/spdx22json/test-fixtures/snapshot/TestSPDXJSONImagePresenter.golden

@@ -3,7 +3,7 @@
  "name": "user-image-input",
  "spdxVersion": "SPDX-2.2",
  "creationInfo": {
-  "created": "2021-12-01T15:08:29.476498Z",
+  "created": "2021-12-20T19:13:07.647486Z",
   "creators": [
    "Organization: Anchore, Inc",
    "Tool: syft-[not provided]"
@@ -11,10 +11,10 @@
   "licenseListVersion": "3.15"
  },
  "dataLicense": "CC0-1.0",
- "documentNamespace": "https://anchore.com/syft/image/user-image-input-e3b7637c-9b2f-4005-a683-58e60f979082",
+ "documentNamespace": "https://anchore.com/syft/image/user-image-input-174da656-1824-4bd3-8604-28919f8a65bc",
  "packages": [
   {
-   "SPDXID": "SPDXRef-888661d4f0362f02",
+   "SPDXID": "SPDXRef-d16127444133b5c1",
    "name": "package-1",
    "licenseConcluded": "MIT",
    "downloadLocation": "NOASSERTION",
@@ -36,7 +36,7 @@
    "versionInfo": "1.0.1"
   },
   {
-   "SPDXID": "SPDXRef-4068ff5e8926b305",
+   "SPDXID": "SPDXRef-24907357f3705420",
    "name": "package-2",
    "licenseConcluded": "NONE",
    "downloadLocation": "NOASSERTION",

internal/formats/syftjson/encoder.go

@@ -8,7 +8,7 @@ import (
 )
 
 func encoder(output io.Writer, s sbom.SBOM) error {
-	doc := ToFormatModel(s)
+	doc := toFormatModel(s)
 
 	enc := json.NewEncoder(output)
 	// prevent > and < from being escaped in the payload

internal/formats/syftjson/model/package.go

@@ -4,9 +4,10 @@ import (
 	"encoding/json"
 	"fmt"
 
+	"github.com/anchore/syft/syft/source"
+
 	"github.com/anchore/syft/internal/log"
 	"github.com/anchore/syft/syft/pkg"
-	"github.com/anchore/syft/syft/source"
 )
 
 // Package represents a pkg.Package object specialized for JSON marshaling and unmarshalling.

internal/formats/syftjson/test-fixtures/snapshot/TestDirectoryPresenter.golden

@@ -1,7 +1,7 @@
 {
  "artifacts": [
   {
-   "id": "2a115ac97d018a0e",
+   "id": "1d97af55efe9512f",
    "name": "package-1",
    "version": "1.0.1",
    "type": "python",
@@ -36,7 +36,7 @@
    }
   },
   {
-   "id": "5e920b2bece2c3ae",
+   "id": "ad3d1c4abd84bf75",
    "name": "package-2",
    "version": "2.0.1",
    "type": "deb",
@@ -83,7 +83,7 @@
   }
  },
  "schema": {
-  "version": "2.0.1",
-  "url": "https://raw.githubusercontent.com/anchore/syft/main/schema/json/schema-2.0.1.json"
+  "version": "2.0.2",
+  "url": "https://raw.githubusercontent.com/anchore/syft/main/schema/json/schema-2.0.2.json"
  }
 }

internal/formats/syftjson/test-fixtures/snapshot/TestEncodeFullJSONDocument.golden

@@ -1,7 +1,7 @@
 {
  "artifacts": [
   {
-   "id": "962403cfb7be50d7",
+   "id": "d9a7c58726ab4bef",
    "name": "package-1",
    "version": "1.0.1",
    "type": "python",
@@ -31,7 +31,7 @@
    }
   },
   {
-   "id": "b11f44847bba0ed1",
+   "id": "ac462e450060da2c",
    "name": "package-2",
    "version": "2.0.1",
    "type": "deb",
@@ -62,8 +62,8 @@
  ],
  "artifactRelationships": [
   {
-   "parent": "962403cfb7be50d7",
-   "child": "b11f44847bba0ed1",
+   "parent": "",
+   "child": "",
    "type": "ownership-by-file-overlap",
    "metadata": {
     "file": "path"
@@ -179,7 +179,7 @@
   }
  },
  "schema": {
-  "version": "2.0.1",
-  "url": "https://raw.githubusercontent.com/anchore/syft/main/schema/json/schema-2.0.1.json"
+  "version": "2.0.2",
+  "url": "https://raw.githubusercontent.com/anchore/syft/main/schema/json/schema-2.0.2.json"
  }
 }

internal/formats/syftjson/test-fixtures/snapshot/TestImagePresenter.golden

@@ -1,7 +1,7 @@
 {
  "artifacts": [
   {
-   "id": "888661d4f0362f02",
+   "id": "d16127444133b5c1",
    "name": "package-1",
    "version": "1.0.1",
    "type": "python",
@@ -9,7 +9,7 @@
    "locations": [
     {
      "path": "/somefile-1.txt",
-     "layerID": "sha256:fb6beecb75b39f4bb813dbf177e501edd5ddb3e69bb45cedeb78c676ee1b7a59"
+     "layerID": "sha256:16e64541f2ddf59a90391ce7bb8af90313f7d373f2105d88f3d3267b72e0ebab"
     }
    ],
    "licenses": [
@@ -32,7 +32,7 @@
    }
   },
   {
-   "id": "4068ff5e8926b305",
+   "id": "24907357f3705420",
    "name": "package-2",
    "version": "2.0.1",
    "type": "deb",
@@ -40,7 +40,7 @@
    "locations": [
     {
      "path": "/somefile-2.txt",
-     "layerID": "sha256:319b588ce64253a87b533c8ed01cf0025e0eac98e7b516e12532957e1244fdec"
+     "layerID": "sha256:de6c235f76ea24c8503ec08891445b5d6a8bdf8249117ed8d8b0b6fb3ebe4f67"
     }
    ],
    "licenses": [],
@@ -67,7 +67,7 @@
   "type": "image",
   "target": {
    "userInput": "user-image-input",
-   "imageID": "sha256:2480160b55bec40c44d3b145c7b2c1c47160db8575c3dcae086d76b9370ae7ca",
+   "imageID": "sha256:9624b89704d23fa5f61b427379d172dac91dc7a508c4d7dea7aed0e04a4cf39e",
    "manifestDigest": "sha256:2731251dc34951c0e50fcc643b4c5f74922dad1a5d98f302b504cf46cd5d9368",
    "mediaType": "application/vnd.docker.distribution.manifest.v2+json",
    "tags": [
@@ -77,17 +77,17 @@
    "layers": [
     {
      "mediaType": "application/vnd.docker.image.rootfs.diff.tar.gzip",
-     "digest": "sha256:fb6beecb75b39f4bb813dbf177e501edd5ddb3e69bb45cedeb78c676ee1b7a59",
+     "digest": "sha256:16e64541f2ddf59a90391ce7bb8af90313f7d373f2105d88f3d3267b72e0ebab",
      "size": 22
     },
     {
      "mediaType": "application/vnd.docker.image.rootfs.diff.tar.gzip",
-     "digest": "sha256:319b588ce64253a87b533c8ed01cf0025e0eac98e7b516e12532957e1244fdec",
+     "digest": "sha256:de6c235f76ea24c8503ec08891445b5d6a8bdf8249117ed8d8b0b6fb3ebe4f67",
      "size": 16
     }
    ],
-   "manifest": "eyJzY2hlbWFWZXJzaW9uIjoyLCJtZWRpYVR5cGUiOiJhcHBsaWNhdGlvbi92bmQuZG9ja2VyLmRpc3RyaWJ1dGlvbi5tYW5pZmVzdC52Mitqc29uIiwiY29uZmlnIjp7Im1lZGlhVHlwZSI6ImFwcGxpY2F0aW9uL3ZuZC5kb2NrZXIuY29udGFpbmVyLmltYWdlLnYxK2pzb24iLCJzaXplIjo2NjcsImRpZ2VzdCI6InNoYTI1NjoyNDgwMTYwYjU1YmVjNDBjNDRkM2IxNDVjN2IyYzFjNDcxNjBkYjg1NzVjM2RjYWUwODZkNzZiOTM3MGFlN2NhIn0sImxheWVycyI6W3sibWVkaWFUeXBlIjoiYXBwbGljYXRpb24vdm5kLmRvY2tlci5pbWFnZS5yb290ZnMuZGlmZi50YXIuZ3ppcCIsInNpemUiOjIwNDgsImRpZ2VzdCI6InNoYTI1NjpmYjZiZWVjYjc1YjM5ZjRiYjgxM2RiZjE3N2U1MDFlZGQ1ZGRiM2U2OWJiNDVjZWRlYjc4YzY3NmVlMWI3YTU5In0seyJtZWRpYVR5cGUiOiJhcHBsaWNhdGlvbi92bmQuZG9ja2VyLmltYWdlLnJvb3Rmcy5kaWZmLnRhci5nemlwIiwic2l6ZSI6MjA0OCwiZGlnZXN0Ijoic2hhMjU2OjMxOWI1ODhjZTY0MjUzYTg3YjUzM2M4ZWQwMWNmMDAyNWUwZWFjOThlN2I1MTZlMTI1MzI5NTdlMTI0NGZkZWMifV19",
-   "config": "eyJhcmNoaXRlY3R1cmUiOiJhbWQ2NCIsImNvbmZpZyI6eyJFbnYiOlsiUEFUSD0vdXNyL2xvY2FsL3NiaW46L3Vzci9sb2NhbC9iaW46L3Vzci9zYmluOi91c3IvYmluOi9zYmluOi9iaW4iXSwiV29ya2luZ0RpciI6Ii8iLCJPbkJ1aWxkIjpudWxsfSwiY3JlYXRlZCI6IjIwMjEtMTAtMDRUMTE6NDA6MDAuNjM4Mzk0NVoiLCJoaXN0b3J5IjpbeyJjcmVhdGVkIjoiMjAyMS0xMC0wNFQxMTo0MDowMC41OTA3MzE2WiIsImNyZWF0ZWRfYnkiOiJBREQgZmlsZS0xLnR4dCAvc29tZWZpbGUtMS50eHQgIyBidWlsZGtpdCIsImNvbW1lbnQiOiJidWlsZGtpdC5kb2NrZXJmaWxlLnYwIn0seyJjcmVhdGVkIjoiMjAyMS0xMC0wNFQxMTo0MDowMC42MzgzOTQ1WiIsImNyZWF0ZWRfYnkiOiJBREQgZmlsZS0yLnR4dCAvc29tZWZpbGUtMi50eHQgIyBidWlsZGtpdCIsImNvbW1lbnQiOiJidWlsZGtpdC5kb2NrZXJmaWxlLnYwIn1dLCJvcyI6ImxpbnV4Iiwicm9vdGZzIjp7InR5cGUiOiJsYXllcnMiLCJkaWZmX2lkcyI6WyJzaGEyNTY6ZmI2YmVlY2I3NWIzOWY0YmI4MTNkYmYxNzdlNTAxZWRkNWRkYjNlNjliYjQ1Y2VkZWI3OGM2NzZlZTFiN2E1OSIsInNoYTI1NjozMTliNTg4Y2U2NDI1M2E4N2I1MzNjOGVkMDFjZjAwMjVlMGVhYzk4ZTdiNTE2ZTEyNTMyOTU3ZTEyNDRmZGVjIl19fQ==",
+   "manifest": "eyJzY2hlbWFWZXJzaW9uIjoyLCJtZWRpYVR5cGUiOiJhcHBsaWNhdGlvbi92bmQuZG9ja2VyLmRpc3RyaWJ1dGlvbi5tYW5pZmVzdC52Mitqc29uIiwiY29uZmlnIjp7Im1lZGlhVHlwZSI6ImFwcGxpY2F0aW9uL3ZuZC5kb2NrZXIuY29udGFpbmVyLmltYWdlLnYxK2pzb24iLCJzaXplIjo2NjcsImRpZ2VzdCI6InNoYTI1Njo5NjI0Yjg5NzA0ZDIzZmE1ZjYxYjQyNzM3OWQxNzJkYWM5MWRjN2E1MDhjNGQ3ZGVhN2FlZDBlMDRhNGNmMzllIn0sImxheWVycyI6W3sibWVkaWFUeXBlIjoiYXBwbGljYXRpb24vdm5kLmRvY2tlci5pbWFnZS5yb290ZnMuZGlmZi50YXIuZ3ppcCIsInNpemUiOjIwNDgsImRpZ2VzdCI6InNoYTI1NjoxNmU2NDU0MWYyZGRmNTlhOTAzOTFjZTdiYjhhZjkwMzEzZjdkMzczZjIxMDVkODhmM2QzMjY3YjcyZTBlYmFiIn0seyJtZWRpYVR5cGUiOiJhcHBsaWNhdGlvbi92bmQuZG9ja2VyLmltYWdlLnJvb3Rmcy5kaWZmLnRhci5nemlwIiwic2l6ZSI6MjA0OCwiZGlnZXN0Ijoic2hhMjU2OmRlNmMyMzVmNzZlYTI0Yzg1MDNlYzA4ODkxNDQ1YjVkNmE4YmRmODI0OTExN2VkOGQ4YjBiNmZiM2ViZTRmNjcifV19",
+   "config": "eyJhcmNoaXRlY3R1cmUiOiJhbWQ2NCIsImNvbmZpZyI6eyJFbnYiOlsiUEFUSD0vdXNyL2xvY2FsL3NiaW46L3Vzci9sb2NhbC9iaW46L3Vzci9zYmluOi91c3IvYmluOi9zYmluOi9iaW4iXSwiV29ya2luZ0RpciI6Ii8iLCJPbkJ1aWxkIjpudWxsfSwiY3JlYXRlZCI6IjIwMjEtMTItMDFUMTI6MTM6NDMuNDAxMzkxNFoiLCJoaXN0b3J5IjpbeyJjcmVhdGVkIjoiMjAyMS0xMi0wMVQxMjoxMzo0My4zNDM2MzQyWiIsImNyZWF0ZWRfYnkiOiJBREQgZmlsZS0xLnR4dCAvc29tZWZpbGUtMS50eHQgIyBidWlsZGtpdCIsImNvbW1lbnQiOiJidWlsZGtpdC5kb2NrZXJmaWxlLnYwIn0seyJjcmVhdGVkIjoiMjAyMS0xMi0wMVQxMjoxMzo0My40MDEzOTE0WiIsImNyZWF0ZWRfYnkiOiJBREQgZmlsZS0yLnR4dCAvc29tZWZpbGUtMi50eHQgIyBidWlsZGtpdCIsImNvbW1lbnQiOiJidWlsZGtpdC5kb2NrZXJmaWxlLnYwIn1dLCJvcyI6ImxpbnV4Iiwicm9vdGZzIjp7InR5cGUiOiJsYXllcnMiLCJkaWZmX2lkcyI6WyJzaGEyNTY6MTZlNjQ1NDFmMmRkZjU5YTkwMzkxY2U3YmI4YWY5MDMxM2Y3ZDM3M2YyMTA1ZDg4ZjNkMzI2N2I3MmUwZWJhYiIsInNoYTI1NjpkZTZjMjM1Zjc2ZWEyNGM4NTAzZWMwODg5MTQ0NWI1ZDZhOGJkZjgyNDkxMTdlZDhkOGIwYjZmYjNlYmU0ZjY3Il19fQ==",
    "repoDigests": []
   }
  },
@@ -104,7 +104,7 @@
   }
  },
  "schema": {
-  "version": "2.0.1",
-  "url": "https://raw.githubusercontent.com/anchore/syft/main/schema/json/schema-2.0.1.json"
+  "version": "2.0.2",
+  "url": "https://raw.githubusercontent.com/anchore/syft/main/schema/json/schema-2.0.2.json"
  }
 }

internal/formats/syftjson/to_format_model.go

@@ -19,8 +19,7 @@ import (
 	"github.com/anchore/syft/syft/source"
 )
 
-// TODO: this is exported for the use of the power-user command (temp)
-func ToFormatModel(s sbom.SBOM) model.Document {
+func toFormatModel(s sbom.SBOM) model.Document {
 	src, err := toSourceModel(s.Source)
 	if err != nil {
 		log.Warnf("unable to create syft-json source object: %+v", err)

schema/cyclonedx/Makefile

@@ -4,4 +4,4 @@ validate-schema:
 	go run ../../main.go ubuntu:latest -vv -o cyclonedx > bom.xml
 	xmllint --noout --schema ./cyclonedx.xsd bom.xml
 	go run ../../main.go ubuntu:latest -vv -o cyclonedx-json > bom.json
-	yajsv -s bom-1.3.schema.json bom.json
+	../../.tmp/yajsv -s bom-1.3.schema.json bom.json

syft/artifact/id.go

@@ -13,7 +13,7 @@ type Identifiable interface {
 	ID() ID
 }
 
-func IDFromHash(obj interface{}) (ID, error) {
+func IDByHash(obj interface{}) (ID, error) {
 	f, err := hashstructure.Hash(obj, hashstructure.FormatV2, &hashstructure.HashOptions{
 		ZeroNil:      true,
 		SlicesAsSets: true,

syft/file/classification_cataloger_test.go

@@ -18,7 +18,7 @@ func TestClassifierCataloger_DefaultClassifiers_PositiveCases(t *testing.T) {
 		{
 			name:       "positive-libpython3.7.so",
 			fixtureDir: "test-fixtures/classifiers/positive",
-			location:   "test-fixtures/classifiers/positive/libpython3.7.so",
+			location:   "libpython3.7.so",
 			expected: []Classification{
 				{
 					Class: "python-binary",
@@ -32,7 +32,7 @@ func TestClassifierCataloger_DefaultClassifiers_PositiveCases(t *testing.T) {
 		{
 			name:       "positive-python3.6",
 			fixtureDir: "test-fixtures/classifiers/positive",
-			location:   "test-fixtures/classifiers/positive/python3.6",
+			location:   "python3.6",
 			expected: []Classification{
 				{
 					Class: "python-binary",
@@ -46,7 +46,7 @@ func TestClassifierCataloger_DefaultClassifiers_PositiveCases(t *testing.T) {
 		{
 			name:       "positive-patchlevel.h",
 			fixtureDir: "test-fixtures/classifiers/positive",
-			location:   "test-fixtures/classifiers/positive/patchlevel.h",
+			location:   "patchlevel.h",
 			expected: []Classification{
 				{
 					Class: "cpython-source",
@@ -60,7 +60,7 @@ func TestClassifierCataloger_DefaultClassifiers_PositiveCases(t *testing.T) {
 		{
 			name:       "positive-go",
 			fixtureDir: "test-fixtures/classifiers/positive",
-			location:   "test-fixtures/classifiers/positive/go",
+			location:   "go",
 			expected: []Classification{
 				{
 					Class: "go-binary",
@@ -74,7 +74,7 @@ func TestClassifierCataloger_DefaultClassifiers_PositiveCases(t *testing.T) {
 		{
 			name:       "positive-go-hint",
 			fixtureDir: "test-fixtures/classifiers/positive",
-			location:   "test-fixtures/classifiers/positive/VERSION",
+			location:   "VERSION",
 			expected: []Classification{
 				{
 					Class: "go-binary-hint",
@@ -88,7 +88,7 @@ func TestClassifierCataloger_DefaultClassifiers_PositiveCases(t *testing.T) {
 		{
 			name:       "positive-busybox",
 			fixtureDir: "test-fixtures/classifiers/positive",
-			location:   "test-fixtures/classifiers/positive/busybox",
+			location:   "busybox",
 			expected: []Classification{
 				{
 					Class: "busybox-binary",

syft/pkg/cargo_package_metadata.go

@@ -9,8 +9,8 @@ type CargoPackageMetadata struct {
 }
 
 // Pkg returns the standard `pkg.Package` representation of the package referenced within the Cargo.lock metadata.
-func (p CargoPackageMetadata) Pkg() Package {
-	return Package{
+func (p CargoPackageMetadata) Pkg() *Package {
+	return &Package{
 		Name:         p.Name,
 		Version:      p.Version,
 		Language:     Rust,

syft/pkg/catalog.go

@@ -51,6 +51,7 @@ func (c *Catalog) Package(id artifact.ID) *Package {
 		log.Warnf("unable to copy package id=%q name=%q: %+v", id, v.Name, err)
 		return nil
 	}
+	p.id = v.id
 	return &p
 }
 
@@ -75,8 +76,12 @@ func (c *Catalog) Add(p Package) {
 	c.lock.Lock()
 	defer c.lock.Unlock()
 
-	// note: since we are capturing the ID, we cannot modify the package being added from this point forward
 	id := p.ID()
+	if id == "" {
+		log.Warnf("found package with empty ID while adding to the catalog: %+v", p)
+		p.SetID()
+		id = p.ID()
+	}
 
 	// store by package ID
 	c.byID[id] = p
@@ -103,6 +108,10 @@ func (c *Catalog) Enumerate(types ...Type) <-chan Package {
 	channel := make(chan Package)
 	go func() {
 		defer close(channel)
+		if c == nil {
+			// we should allow enumerating from a catalog that was never created (which will result in no packages enumerated)
+			return
+		}
 		for ty, ids := range c.idsByType {
 			if len(types) != 0 {
 				found := false
@@ -138,7 +147,7 @@ func (c *Catalog) Sorted(types ...Type) (pkgs []Package) {
 	sort.SliceStable(pkgs, func(i, j int) bool {
 		if pkgs[i].Name == pkgs[j].Name {
 			if pkgs[i].Version == pkgs[j].Version {
-				if pkgs[i].Type == pkgs[j].Type {
+				if pkgs[i].Type == pkgs[j].Type && len(pkgs[i].Locations) > 0 && len(pkgs[j].Locations) > 0 {
 					return pkgs[i].Locations[0].String() < pkgs[j].Locations[0].String()
 				}
 				return pkgs[i].Type < pkgs[j].Type

syft/pkg/catalog_test.go

@@ -3,28 +3,13 @@ package pkg
 import (
 	"testing"
 
+	"github.com/stretchr/testify/assert"
+
 	"github.com/scylladb/go-set/strset"
 
 	"github.com/anchore/syft/syft/source"
 )
 
-var catalogAddAndRemoveTestPkgs = []Package{
-	{
-		Locations: []source.Location{
-			source.NewVirtualLocation("/a/path", "/another/path"),
-			source.NewVirtualLocation("/b/path", "/bee/path"),
-		},
-		Type: RpmPkg,
-	},
-	{
-		Locations: []source.Location{
-			source.NewVirtualLocation("/c/path", "/another/path"),
-			source.NewVirtualLocation("/d/path", "/another/path"),
-		},
-		Type: NpmPkg,
-	},
-}
-
 type expectedIndexes struct {
 	byType map[Type]*strset.Set
 	byPath map[string]*strset.Set
@@ -32,18 +17,38 @@ type expectedIndexes struct {
 
 func TestCatalogAddPopulatesIndex(t *testing.T) {
 
+	var pkgs = []Package{
+		{
+			Locations: []source.Location{
+				source.NewVirtualLocation("/a/path", "/another/path"),
+				source.NewVirtualLocation("/b/path", "/bee/path"),
+			},
+			Type: RpmPkg,
+		},
+		{
+			Locations: []source.Location{
+				source.NewVirtualLocation("/c/path", "/another/path"),
+				source.NewVirtualLocation("/d/path", "/another/path"),
+			},
+			Type: NpmPkg,
+		},
+	}
+
+	for i := range pkgs {
+		p := &pkgs[i]
+		p.SetID()
+	}
+
 	fixtureID := func(i int) string {
-		return string(catalogAddAndRemoveTestPkgs[i].ID())
+		return string(pkgs[i].ID())
 	}
 
 	tests := []struct {
 		name            string
-		pkgs            []Package
 		expectedIndexes expectedIndexes
 	}{
 		{
 			name: "vanilla-add",
-			pkgs: catalogAddAndRemoveTestPkgs,
 			expectedIndexes: expectedIndexes{
 				byType: map[Type]*strset.Set{
 					RpmPkg: strset.New(fixtureID(0)),
@@ -63,7 +68,7 @@ func TestCatalogAddPopulatesIndex(t *testing.T) {
 
 	for _, test := range tests {
 		t.Run(test.name, func(t *testing.T) {
-			c := NewCatalog(test.pkgs...)
+			c := NewCatalog(pkgs...)
 
 			assertIndexes(t, c, test.expectedIndexes)
 
@@ -73,9 +78,7 @@ func TestCatalogAddPopulatesIndex(t *testing.T) {
 
 func assertIndexes(t *testing.T, c *Catalog, expectedIndexes expectedIndexes) {
 	// assert path index
-	if len(c.idsByPath) != len(expectedIndexes.byPath) {
-		t.Errorf("unexpected path index length: %d != %d", len(c.idsByPath), len(expectedIndexes.byPath))
-	}
+	assert.Len(t, c.idsByPath, len(expectedIndexes.byPath), "unexpected path index length")
 	for path, expectedIds := range expectedIndexes.byPath {
 		actualIds := strset.New()
 		for _, p := range c.PackagesByPath(path) {
@@ -88,9 +91,7 @@ func assertIndexes(t *testing.T, c *Catalog, expectedIndexes expectedIndexes) {
 	}
 
 	// assert type index
-	if len(c.idsByType) != len(expectedIndexes.byType) {
-		t.Errorf("unexpected type index length: %d != %d", len(c.idsByType), len(expectedIndexes.byType))
-	}
+	assert.Len(t, c.idsByType, len(expectedIndexes.byType), "unexpected type index length")
 	for ty, expectedIds := range expectedIndexes.byType {
 		actualIds := strset.New()
 		for p := range c.Enumerate(ty) {
@@ -144,3 +145,8 @@ func TestCatalog_PathIndexDeduplicatesRealVsVirtualPaths(t *testing.T) {
 	}
 
 }
+
+func TestCatalog_EnumerateNilCatalog(t *testing.T) {
+	var c *Catalog
+	assert.Empty(t, c.Enumerate())
+}

syft/pkg/cataloger/apkdb/parse_apk_db.go

@@ -21,14 +21,25 @@ import (
 // integrity check
 var _ common.ParserFn = parseApkDB
 
+func newApkDBPackage(d *pkg.ApkMetadata) *pkg.Package {
+	return &pkg.Package{
+		Name:         d.Package,
+		Version:      d.Version,
+		Licenses:     strings.Split(d.License, " "),
+		Type:         pkg.ApkPkg,
+		MetadataType: pkg.ApkMetadataType,
+		Metadata:     *d,
+	}
+}
+
 // parseApkDb parses individual packages from a given Alpine DB file. For more information on specific fields
 // see https://wiki.alpinelinux.org/wiki/Apk_spec .
-func parseApkDB(_ string, reader io.Reader) ([]pkg.Package, []artifact.Relationship, error) {
+func parseApkDB(_ string, reader io.Reader) ([]*pkg.Package, []artifact.Relationship, error) {
 	// larger capacity for the scanner.
 	const maxScannerCapacity = 1024 * 1024
 	// a new larger buffer for the scanner
 	bufScan := make([]byte, maxScannerCapacity)
-	packages := make([]pkg.Package, 0)
+	packages := make([]*pkg.Package, 0)
 
 	scanner := bufio.NewScanner(reader)
 	scanner.Buffer(bufScan, maxScannerCapacity)
@@ -52,14 +63,7 @@ func parseApkDB(_ string, reader io.Reader) ([]pkg.Package, []artifact.Relations
 			return nil, nil, err
 		}
 		if metadata != nil {
-			packages = append(packages, pkg.Package{
-				Name:         metadata.Package,
-				Version:      metadata.Version,
-				Licenses:     strings.Split(metadata.License, " "),
-				Type:         pkg.ApkPkg,
-				MetadataType: pkg.ApkMetadataType,
-				Metadata:     *metadata,
-			})
+			packages = append(packages, newApkDBPackage(metadata))
 		}
 	}
 

syft/pkg/cataloger/apkdb/parse_apk_db_test.go

@@ -647,11 +647,11 @@ func TestSinglePackageDetails(t *testing.T) {
 func TestMultiplePackages(t *testing.T) {
 	tests := []struct {
 		fixture  string
-		expected []pkg.Package
+		expected []*pkg.Package
 	}{
 		{
 			fixture: "test-fixtures/multiple",
-			expected: []pkg.Package{
+			expected: []*pkg.Package{
 				{
 					Name:         "libc-utils",
 					Version:      "0.7.2-r0",

syft/pkg/cataloger/catalog.go

@@ -49,9 +49,10 @@ func Catalog(resolver source.FileResolver, theDistro *distro.Distro, catalogers
 
 	// perform analysis, accumulating errors for each failed analysis
 	var errs error
-	for _, theCataloger := range catalogers {
+	for _, c := range catalogers {
 		// find packages from the underlying raw data
-		packages, relationships, err := theCataloger.Catalog(resolver)
+		log.Debugf("cataloging with %q", c.Name())
+		packages, relationships, err := c.Catalog(resolver)
 		if err != nil {
 			errs = multierror.Append(errs, err)
 			continue
@@ -59,14 +60,14 @@ func Catalog(resolver source.FileResolver, theDistro *distro.Distro, catalogers
 
 		catalogedPackages := len(packages)
 
-		log.Debugf("package cataloger %q discovered %d packages", theCataloger.Name(), catalogedPackages)
+		log.Debugf("discovered %d packages", catalogedPackages)
 		packagesDiscovered.N += int64(catalogedPackages)
 
 		for _, p := range packages {
-			// generate CPEs
+			// generate CPEs (note: this is excluded from package ID, so is safe to mutate)
 			p.CPEs = cpe.Generate(p)
 
-			// generate PURL
+			// generate PURL (note: this is excluded from package ID, so is safe to mutate)
 			p.PURL = generatePackageURL(p, theDistro)
 
 			// create file-to-package relationships for files owned by the package

syft/pkg/cataloger/common/generic_cataloger.go

@@ -45,22 +45,23 @@ func (c *GenericCataloger) Catalog(resolver source.FileResolver) ([]pkg.Package,
 		contentReader, err := resolver.FileContentsByLocation(location)
 		if err != nil {
 			// TODO: fail or log?
-			return nil, nil, fmt.Errorf("unable to fetch contents for location=%v : %w", location, err)
+			return nil, nil, fmt.Errorf("unable to fetch contents at location=%v: %w", location, err)
 		}
 
 		discoveredPackages, discoveredRelationships, err := parser(location.RealPath, contentReader)
 		internal.CloseAndLogError(contentReader, location.VirtualPath)
 		if err != nil {
 			// TODO: should we fail? or only log?
-			log.Warnf("cataloger '%s' failed to parse entries (location=%+v): %+v", c.upstreamCataloger, location, err)
+			log.Warnf("cataloger '%s' failed to parse entries at location=%+v: %+v", c.upstreamCataloger, location, err)
 			continue
 		}
 
 		for _, p := range discoveredPackages {
 			p.FoundBy = c.upstreamCataloger
 			p.Locations = append(p.Locations, location)
+			p.SetID()
 
-			packages = append(packages, p)
+			packages = append(packages, *p)
 		}
 
 		relationships = append(relationships, discoveredRelationships...)

syft/pkg/cataloger/common/generic_cataloger_test.go

@@ -13,12 +13,12 @@ import (
 	"github.com/anchore/syft/syft/source"
 )
 
-func parser(_ string, reader io.Reader) ([]pkg.Package, []artifact.Relationship, error) {
+func parser(_ string, reader io.Reader) ([]*pkg.Package, []artifact.Relationship, error) {
 	contents, err := ioutil.ReadAll(reader)
 	if err != nil {
 		panic(err)
 	}
-	return []pkg.Package{
+	return []*pkg.Package{
 		{
 			Name: string(contents),
 		},

syft/pkg/cataloger/common/parser.go

@@ -8,4 +8,4 @@ import (
 )
 
 // ParserFn standardizes a function signature for parser functions that accept the virtual file path (not usable for file reads) and contents and return any discovered packages from that file
-type ParserFn func(string, io.Reader) ([]pkg.Package, []artifact.Relationship, error)
+type ParserFn func(string, io.Reader) ([]*pkg.Package, []artifact.Relationship, error)

syft/pkg/cataloger/deb/cataloger.go

@@ -68,6 +68,8 @@ func (c *Cataloger) Catalog(resolver source.FileResolver) ([]pkg.Package, []arti
 
 			// fetch additional data from the copyright file to derive the license information
 			addLicenses(resolver, dbLocation, p)
+
+			p.SetID()
 		}
 
 		allPackages = append(allPackages, pkgs...)

syft/pkg/cataloger/deb/parse_dpkg_status.go

@@ -20,6 +20,16 @@ var (
 	sourceRegexp     = regexp.MustCompile(`(?P<name>\S+)( \((?P<version>.*)\))?`)
 )
 
+func newDpkgPackage(d pkg.DpkgMetadata) pkg.Package {
+	return pkg.Package{
+		Name:         d.Package,
+		Version:      d.Version,
+		Type:         pkg.DebPkg,
+		MetadataType: pkg.DpkgMetadataType,
+		Metadata:     d,
+	}
+}
+
 // parseDpkgStatus is a parser function for Debian DB status contents, returning all Debian packages listed.
 func parseDpkgStatus(reader io.Reader) ([]pkg.Package, error) {
 	buffedReader := bufio.NewReader(reader)
@@ -37,13 +47,7 @@ func parseDpkgStatus(reader io.Reader) ([]pkg.Package, error) {
 		}
 
 		if entry.Package != "" {
-			packages = append(packages, pkg.Package{
-				Name:         entry.Package,
-				Version:      entry.Version,
-				Type:         pkg.DebPkg,
-				MetadataType: pkg.DpkgMetadataType,
-				Metadata:     entry,
-			})
+			packages = append(packages, newDpkgPackage(entry))
 		}
 	}
 

syft/pkg/cataloger/golang/binary_cataloger.go

@@ -40,12 +40,12 @@ func (c *Cataloger) Catalog(resolver source.FileResolver) ([]pkg.Package, []arti
 	for _, location := range fileMatches {
 		r, err := resolver.FileContentsByLocation(location)
 		if err != nil {
-			return pkgs, nil, fmt.Errorf("failed to resolve file contents by location: %w", err)
+			return pkgs, nil, fmt.Errorf("failed to resolve file contents by location=%q: %w", location.RealPath, err)
 		}
 
-		goPkgs, err := parseGoBin(location, r)
+		goPkgs, err := parseGoBin(location, r, openExe)
 		if err != nil {
-			log.Warnf("could not parse possible go binary: %+v", err)
+			log.Warnf("could not parse possible go binary at %q: %+v", location.RealPath, err)
 		}
 
 		internal.CloseAndLogError(r, location.RealPath)

syft/pkg/cataloger/golang/parse_go_bin.go

@@ -2,6 +2,7 @@ package golang
 
 import (
 	"bufio"
+	"fmt"
 	"io"
 	"strings"
 
@@ -14,19 +15,51 @@ const (
 	replaceIdentifier = "=>"
 )
 
-func parseGoBin(location source.Location, reader io.ReadCloser) ([]pkg.Package, error) {
+type exeOpener func(file io.ReadCloser) ([]exe, error)
+
+func newGoBinaryPackage(name, version, h1Digest, goVersion, architecture string, location source.Location) pkg.Package {
+	p := pkg.Package{
+		Name:     name,
+		Version:  version,
+		Language: pkg.Go,
+		Type:     pkg.GoModulePkg,
+		Locations: []source.Location{
+			location,
+		},
+		MetadataType: pkg.GolangBinMetadataType,
+		Metadata: pkg.GolangBinMetadata{
+			GoCompiledVersion: goVersion,
+			H1Digest:          h1Digest,
+			Architecture:      architecture,
+		},
+	}
+
+	p.SetID()
+
+	return p
+}
+
+func parseGoBin(location source.Location, reader io.ReadCloser, opener exeOpener) (pkgs []pkg.Package, err error) {
+	var exes []exe
+	// it has been found that there are stdlib paths within openExe that can panic. We want to prevent this behavior
+	// bubbling up and halting execution. For this reason we try to recover from any panic and return an error.
+	defer func() {
+		if r := recover(); r != nil {
+			err = fmt.Errorf("recovered from panic while parse go binary at %q: %+v", location.RealPath, r)
+		}
+	}()
+
 	// Identify if bin was compiled by go
-	exes, err := openExe(reader)
+	exes, err = opener(reader)
 	if err != nil {
-		return nil, err
+		return pkgs, err
 	}
 
-	var pkgs []pkg.Package
 	for _, x := range exes {
 		goVersion, mod := findVers(x)
 		pkgs = append(pkgs, buildGoPkgInfo(location, mod, goVersion, x.ArchName())...)
 	}
-	return pkgs, nil
+	return pkgs, err
 }
 
 func buildGoPkgInfo(location source.Location, mod, goVersion, arch string) []pkg.Package {
@@ -43,21 +76,10 @@ func buildGoPkgInfo(location source.Location, mod, goVersion, arch string) []pkg
 		}
 
 		if fields[0] == packageIdentifier || fields[0] == replaceIdentifier {
-			pkgsSlice = append(pkgsSlice, pkg.Package{
-				Name:     fields[1],
-				Version:  fields[2],
-				Language: pkg.Go,
-				Type:     pkg.GoModulePkg,
-				Locations: []source.Location{
-					location,
-				},
-				MetadataType: pkg.GolangBinMetadataType,
-				Metadata: pkg.GolangBinMetadata{
-					GoCompiledVersion: goVersion,
-					H1Digest:          fields[3],
-					Architecture:      arch,
-				},
-			})
+			name := fields[1]
+			version := fields[2]
+			h1Digest := fields[3]
+			pkgsSlice = append(pkgsSlice, newGoBinaryPackage(name, version, h1Digest, goVersion, arch, location))
 		}
 	}
 

syft/pkg/cataloger/golang/parse_go_bin_test.go

@@ -1,6 +1,7 @@
 package golang
 
 import (
+	"io"
 	"testing"
 
 	"github.com/anchore/syft/syft/pkg"
@@ -144,17 +145,42 @@ func TestBuildGoPkgInfo(t *testing.T) {
 		},
 	}
 
-	for _, tt := range tests {
-		tt := tt
-		t.Run(tt.name, func(t *testing.T) {
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			for i := range test.expected {
+				p := &test.expected[i]
+				p.SetID()
+			}
 			location := source.Location{
 				Coordinates: source.Coordinates{
 					RealPath:     "/a-path",
 					FileSystemID: "layer-id",
 				},
 			}
-			pkgs := buildGoPkgInfo(location, tt.mod, goCompiledVersion, archDetails)
-			assert.Equal(t, tt.expected, pkgs)
+			pkgs := buildGoPkgInfo(location, test.mod, goCompiledVersion, archDetails)
+			assert.Equal(t, test.expected, pkgs)
+		})
+	}
+}
+
+func Test_parseGoBin_recoversFromPanic(t *testing.T) {
+	freakOut := func(file io.ReadCloser) ([]exe, error) {
+		panic("baaahhh!")
+	}
+	tests := []struct {
+		name     string
+		wantPkgs []pkg.Package
+		wantErr  assert.ErrorAssertionFunc
+	}{
+		{
+			name: "recovers from panic",
+		},
+	}
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			pkgs, err := parseGoBin(source.NewLocation("some/path"), nil, freakOut)
+			assert.Error(t, err)
+			assert.Nil(t, pkgs)
 		})
 	}
 }

syft/pkg/cataloger/golang/parse_go_mod.go

@@ -12,8 +12,8 @@ import (
 )
 
 // parseGoMod takes a go.mod and lists all packages discovered.
-func parseGoMod(path string, reader io.Reader) ([]pkg.Package, []artifact.Relationship, error) {
-	packages := make(map[string]pkg.Package)
+func parseGoMod(path string, reader io.Reader) ([]*pkg.Package, []artifact.Relationship, error) {
+	packages := make(map[string]*pkg.Package)
 
 	contents, err := ioutil.ReadAll(reader)
 	if err != nil {
@@ -26,7 +26,7 @@ func parseGoMod(path string, reader io.Reader) ([]pkg.Package, []artifact.Relati
 	}
 
 	for _, m := range file.Require {
-		packages[m.Mod.Path] = pkg.Package{
+		packages[m.Mod.Path] = &pkg.Package{
 			Name:     m.Mod.Path,
 			Version:  m.Mod.Version,
 			Language: pkg.Go,
@@ -36,7 +36,7 @@ func parseGoMod(path string, reader io.Reader) ([]pkg.Package, []artifact.Relati
 
 	// remove any old packages and replace with new ones...
 	for _, m := range file.Replace {
-		packages[m.New.Path] = pkg.Package{
+		packages[m.New.Path] = &pkg.Package{
 			Name:     m.New.Path,
 			Version:  m.New.Version,
 			Language: pkg.Go,
@@ -49,7 +49,7 @@ func parseGoMod(path string, reader io.Reader) ([]pkg.Package, []artifact.Relati
 		delete(packages, m.Mod.Path)
 	}
 
-	pkgsSlice := make([]pkg.Package, len(packages))
+	pkgsSlice := make([]*pkg.Package, len(packages))
 	idx := 0
 	for _, p := range packages {
 		pkgsSlice[idx] = p

syft/pkg/cataloger/golang/parse_go_mod_test.go

@@ -87,7 +87,7 @@ func TestParseGoMod(t *testing.T) {
 					continue
 				}
 
-				diffs := deep.Equal(a, e)
+				diffs := deep.Equal(a, &e)
 				if len(diffs) > 0 {
 					t.Errorf("diffs found for %q", a.Name)
 					for _, d := range diffs {

syft/pkg/cataloger/java/archive_parser.go

@@ -35,7 +35,7 @@ type archiveParser struct {
 }
 
 // parseJavaArchive is a parser function for java archive contents, returning all Java libraries and nested archives.
-func parseJavaArchive(virtualPath string, reader io.Reader) ([]pkg.Package, []artifact.Relationship, error) {
+func parseJavaArchive(virtualPath string, reader io.Reader) ([]*pkg.Package, []artifact.Relationship, error) {
 	parser, cleanupFn, err := newJavaArchiveParser(virtualPath, reader, true)
 	// note: even on error, we should always run cleanup functions
 	defer cleanupFn()
@@ -81,8 +81,8 @@ func newJavaArchiveParser(virtualPath string, reader io.Reader, detectNested boo
 }
 
 // parse the loaded archive and return all packages found.
-func (j *archiveParser) parse() ([]pkg.Package, []artifact.Relationship, error) {
-	var pkgs []pkg.Package
+func (j *archiveParser) parse() ([]*pkg.Package, []artifact.Relationship, error) {
+	var pkgs []*pkg.Package
 	var relationships []artifact.Relationship
 
 	// find the parent package from the java manifest
@@ -110,7 +110,7 @@ func (j *archiveParser) parse() ([]pkg.Package, []artifact.Relationship, error)
 
 	// lastly, add the parent package to the list (assuming the parent exists)
 	if parentPkg != nil {
-		pkgs = append([]pkg.Package{*parentPkg}, pkgs...)
+		pkgs = append([]*pkg.Package{parentPkg}, pkgs...)
 	}
 
 	return pkgs, relationships, nil
@@ -158,12 +158,12 @@ func (j *archiveParser) discoverMainPackage() (*pkg.Package, error) {
 // parent package, returning all listed Java packages found for each pom
 // properties discovered and potentially updating the given parentPkg with new
 // data.
-func (j *archiveParser) discoverPkgsFromAllMavenFiles(parentPkg *pkg.Package) ([]pkg.Package, error) {
+func (j *archiveParser) discoverPkgsFromAllMavenFiles(parentPkg *pkg.Package) ([]*pkg.Package, error) {
 	if parentPkg == nil {
 		return nil, nil
 	}
 
-	var pkgs []pkg.Package
+	var pkgs []*pkg.Package
 
 	properties, err := pomPropertiesByParentPath(j.archivePath, j.fileManifest.GlobMatch(pomPropertiesGlob), j.virtualPath)
 	if err != nil {
@@ -183,7 +183,7 @@ func (j *archiveParser) discoverPkgsFromAllMavenFiles(parentPkg *pkg.Package) ([
 
 		pkgFromPom := newPackageFromMavenData(propertiesObj, pomProject, parentPkg, j.virtualPath)
 		if pkgFromPom != nil {
-			pkgs = append(pkgs, *pkgFromPom)
+			pkgs = append(pkgs, pkgFromPom)
 		}
 	}
 
@@ -192,8 +192,8 @@ func (j *archiveParser) discoverPkgsFromAllMavenFiles(parentPkg *pkg.Package) ([
 
 // discoverPkgsFromNestedArchives finds Java archives within Java archives, returning all listed Java packages found and
 // associating each discovered package to the given parent package.
-func (j *archiveParser) discoverPkgsFromNestedArchives(parentPkg *pkg.Package) ([]pkg.Package, []artifact.Relationship, error) {
-	var pkgs []pkg.Package
+func (j *archiveParser) discoverPkgsFromNestedArchives(parentPkg *pkg.Package) ([]*pkg.Package, []artifact.Relationship, error) {
+	var pkgs []*pkg.Package
 	var relationships []artifact.Relationship
 
 	// search and parse pom.properties files & fetch the contents

syft/pkg/cataloger/java/archive_parser_test.go

@@ -257,7 +257,7 @@ func TestParseJar(t *testing.T) {
 			var parent *pkg.Package
 			for _, a := range actual {
 				if strings.Contains(a.Name, "example-") {
-					parent = &a
+					parent = a
 				}
 			}
 
@@ -292,7 +292,7 @@ func TestParseJar(t *testing.T) {
 				// write censored data back
 				a.Metadata = metadata
 
-				diffs := deep.Equal(e, a)
+				diffs := deep.Equal(&e, a)
 				if len(diffs) > 0 {
 					t.Errorf("diffs found for %q", a.Name)
 					for _, d := range diffs {
@@ -527,7 +527,7 @@ func TestParseNestedJar(t *testing.T) {
 
 			actualNameVersionPairSet := internal.NewStringSet()
 			for _, a := range actual {
-				key := makeKey(&a)
+				key := makeKey(a)
 				actualNameVersionPairSet.Add(key)
 				if !expectedNameVersionPairSet.Contains(key) {
 					t.Errorf("extra package: %s", a)
@@ -545,7 +545,7 @@ func TestParseNestedJar(t *testing.T) {
 			}
 
 			for _, a := range actual {
-				actualKey := makeKey(&a)
+				actualKey := makeKey(a)
 
 				metadata := a.Metadata.(pkg.JavaMetadata)
 				if actualKey == "spring-boot|0.0.1-SNAPSHOT" {

syft/pkg/cataloger/java/parse_java_manifest.go

@@ -19,48 +19,66 @@ const manifestGlob = "/META-INF/MANIFEST.MF"
 // For more information: https://docs.oracle.com/en/java/javase/11/docs/specs/jar/jar.html#jar-manifest
 func parseJavaManifest(path string, reader io.Reader) (*pkg.JavaManifest, error) {
 	var manifest pkg.JavaManifest
-	sections := []map[string]string{
-		make(map[string]string),
+	var sections []map[string]string
+
+	currentSection := func() int {
+		return len(sections) - 1
 	}
-	currentSection := 0
-	scanner := bufio.NewScanner(reader)
+
 	var lastKey string
+	scanner := bufio.NewScanner(reader)
+
 	for scanner.Scan() {
 		line := scanner.Text()
 
 		// empty lines denote section separators
 		if strings.TrimSpace(line) == "" {
-			currentSection++
-			// we don't want to allocate a new section map that wont necessarily be used, do that once there is
+			// we don't want to allocate a new section map that won't necessarily be used, do that once there is
 			// a non-empty line to process
 
 			// do not process line continuations after this
 			lastKey = ""
+
 			continue
-		} else if currentSection >= len(sections) {
-			sections = append(sections, make(map[string]string))
 		}
 
 		if line[0] == ' ' {
 			// this is a continuation
+
 			if lastKey == "" {
-				return nil, fmt.Errorf("found continuation with no previous key (%s)", line)
-			}
-			sections[currentSection][lastKey] += strings.TrimSpace(line)
-		} else {
-			// this is a new key-value pair
-			idx := strings.Index(line, ":")
-			if idx == -1 {
-				return nil, fmt.Errorf("unable to split java manifest key-value pairs: %q", line)
+				log.Warnf("java manifest %q: found continuation with no previous key: %q", path, line)
+				continue
 			}
 
-			key := strings.TrimSpace(line[0:idx])
-			value := strings.TrimSpace(line[idx+1:])
-			sections[currentSection][key] = value
+			sections[currentSection()][lastKey] += strings.TrimSpace(line)
 
-			// keep track of key for potential future continuations
-			lastKey = key
+			continue
 		}
+
+		// this is a new key-value pair
+		idx := strings.Index(line, ":")
+		if idx == -1 {
+			log.Warnf("java manifest %q: unable to split java manifest key-value pairs: %q", path, line)
+			continue
+		}
+
+		key := strings.TrimSpace(line[0:idx])
+		value := strings.TrimSpace(line[idx+1:])
+
+		if key == "" {
+			// don't attempt to add new keys or sections unless there is a non-empty key
+			continue
+		}
+
+		if lastKey == "" {
+			// we're entering a new section
+			sections = append(sections, make(map[string]string))
+		}
+
+		sections[currentSection()][key] = value
+
+		// keep track of key for potential future continuations
+		lastKey = key
 	}
 
 	if err := scanner.Err(); err != nil {
@@ -75,7 +93,7 @@ func parseJavaManifest(path string, reader io.Reader) (*pkg.JavaManifest, error)
 				name, ok := s["Name"]
 				if !ok {
 					// per the manifest spec (https://docs.oracle.com/en/java/javase/11/docs/specs/jar/jar.html#jar-manifest)
-					// this should never happen. If it does we want to know about it, but not necessarily stop
+					// this should never happen. If it does, we want to know about it, but not necessarily stop
 					// cataloging entirely... for this reason we only log.
 					log.Warnf("java manifest section found without a name: %s", path)
 					name = strconv.Itoa(i)
@@ -115,16 +133,40 @@ func selectName(manifest *pkg.JavaManifest, filenameObj archiveFilename) string
 }
 
 func selectVersion(manifest *pkg.JavaManifest, filenameObj archiveFilename) string {
-	var version string
-	switch {
-	case filenameObj.version != "":
-		version = filenameObj.version
-	case manifest.Main["Implementation-Version"] != "":
-		version = manifest.Main["Implementation-Version"]
-	case manifest.Main["Specification-Version"] != "":
-		version = manifest.Main["Specification-Version"]
-	case manifest.Main["Plugin-Version"] != "":
-		version = manifest.Main["Plugin-Version"]
+	if v := filenameObj.version; v != "" {
+		return v
 	}
-	return version
+
+	if manifest == nil {
+		return ""
+	}
+
+	fieldNames := []string{
+		"Implementation-Version",
+		"Specification-Version",
+		"Plugin-Version",
+		"Bundle-Version",
+	}
+
+	for _, fieldName := range fieldNames {
+		if v := fieldValueFromManifest(*manifest, fieldName); v != "" {
+			return v
+		}
+	}
+
+	return ""
+}
+
+func fieldValueFromManifest(manifest pkg.JavaManifest, fieldName string) string {
+	if value := manifest.Main[fieldName]; value != "" {
+		return value
+	}
+
+	for _, section := range manifest.NamedSections {
+		if value := section[fieldName]; value != "" {
+			return value
+		}
+	}
+
+	return ""
 }

syft/pkg/cataloger/java/parse_java_manifest_test.go

@@ -5,6 +5,8 @@ import (
 	"os"
 	"testing"
 
+	"github.com/stretchr/testify/assert"
+
 	"github.com/anchore/syft/syft/pkg"
 	"github.com/go-test/deep"
 )
@@ -56,6 +58,30 @@ func TestParseJavaManifest(t *testing.T) {
 				},
 			},
 		},
+		{
+			fixture: "test-fixtures/manifest/extra-empty-lines",
+			expected: pkg.JavaManifest{
+				Main: map[string]string{
+					"Manifest-Version": "1.0",
+					"Archiver-Version": "Plexus Archiver",
+					"Created-By":       "Apache Maven 3.6.3",
+				},
+				NamedSections: map[string]map[string]string{
+					"thing-1": {
+						"Built-By": "?",
+					},
+					"thing-2": {
+						"Built-By": "someone!",
+					},
+					"2": {
+						"Other": "things",
+					},
+					"3": {
+						"Last": "item",
+					},
+				},
+			},
+		},
 		{
 			fixture: "test-fixtures/manifest/continuation",
 			expected: pkg.JavaManifest{
@@ -145,5 +171,94 @@ func TestSelectName(t *testing.T) {
 			}
 		})
 	}
+}
 
+func TestSelectVersion(t *testing.T) {
+	tests := []struct {
+		name     string
+		manifest pkg.JavaManifest
+		archive  archiveFilename
+		expected string
+	}{
+		{
+			name:    "Get name from Implementation-Version",
+			archive: archiveFilename{},
+			manifest: pkg.JavaManifest{
+				Main: map[string]string{
+					"Implementation-Version": "1.8.2",
+				},
+			},
+			expected: "1.8.2",
+		},
+		{
+			name: "Implementation-Version takes precedence over Specification-Version",
+			manifest: pkg.JavaManifest{
+				Main: map[string]string{
+					"Implementation-Version": "1.8.2",
+					"Specification-Version":  "1.0",
+				},
+			},
+			expected: "1.8.2",
+		},
+		{
+			name: "Implementation-Version found outside the main section",
+			manifest: pkg.JavaManifest{
+				Main: map[string]string{
+					"Manifest-Version": "1.0",
+					"Ant-Version":      "Apache Ant 1.8.2",
+					"Created-By":       "1.5.0_22-b03 (Sun Microsystems Inc.)",
+				},
+				NamedSections: map[string]map[string]string{
+					"org/apache/tools/ant/taskdefs/optional/": {
+						"Implementation-Version": "1.8.2",
+					},
+				},
+			},
+			expected: "1.8.2",
+		},
+		{
+			name: "Implementation-Version takes precedence over Specification-Version in subsequent section",
+			manifest: pkg.JavaManifest{
+				Main: map[string]string{
+					"Manifest-Version":      "1.0",
+					"Ant-Version":           "Apache Ant 1.8.2",
+					"Created-By":            "1.5.0_22-b03 (Sun Microsystems Inc.)",
+					"Specification-Version": "2.0",
+				},
+				NamedSections: map[string]map[string]string{
+					"org/apache/tools/ant/taskdefs/optional/": {
+						"Specification-Version": "1.8",
+					},
+					"some-other-section": {
+						"Implementation-Version": "1.8.2",
+					},
+				},
+			},
+			expected: "1.8.2",
+		},
+		{
+			name: "Implementation-Version takes precedence over Specification-Version in subsequent section",
+			manifest: pkg.JavaManifest{
+				Main: map[string]string{
+					"Manifest-Version": "1.0",
+					"Ant-Version":      "Apache Ant 1.8.2",
+					"Created-By":       "1.5.0_22-b03 (Sun Microsystems Inc.)",
+				},
+				NamedSections: map[string]map[string]string{
+					"some-other-section": {
+						"Bundle-Version": "1.11.28",
+					},
+				},
+			},
+			expected: "1.11.28",
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			result := selectVersion(&test.manifest, test.archive)
+
+			assert.Equal(t, test.expected, result)
+		})
+	}
 }

syft/pkg/cataloger/java/test-fixtures/manifest/extra-empty-lines

@@ -0,0 +1,22 @@
+Manifest-Version: 1.0
+Archiver-Version: Plexus Archiver
+Created-By: Apache Maven 3.6.3
+
+
+Name: thing-1
+Built-By: ?
+
+.
+
+Name: thing-2
+Built-By: someone!
+
+
+ :
+
+Other: things
+junk
+
+:
+
+Last: item

syft/pkg/cataloger/javascript/parse_package_json.go

@@ -50,7 +50,6 @@ type Repository struct {
 // ---> name: "Isaac Z. Schlueter" email: "i@izs.me" url: "http://blog.izs.me"
 var authorPattern = regexp.MustCompile(`^\s*(?P<name>[^<(]*)(\s+<(?P<email>.*)>)?(\s\((?P<url>.*)\))?\s*$`)
 
-// Exports Author.UnmarshalJSON interface to help normalize the json structure.
 func (a *Author) UnmarshalJSON(b []byte) error {
 	var authorStr string
 	var fields map[string]string
@@ -135,7 +134,7 @@ func licenseFromJSON(b []byte) (string, error) {
 	return "", errors.New("unable to unmarshal license field as either string or object")
 }
 
-func licensesFromJSON(p PackageJSON) ([]string, error) {
+func (p PackageJSON) licensesFromJSON() ([]string, error) {
 	if p.License == nil && p.Licenses == nil {
 		// This package.json doesn't specify any licenses whatsoever
 		return []string{}, nil
@@ -163,8 +162,8 @@ func licensesFromJSON(p PackageJSON) ([]string, error) {
 }
 
 // parsePackageJSON parses a package.json and returns the discovered JavaScript packages.
-func parsePackageJSON(_ string, reader io.Reader) ([]pkg.Package, []artifact.Relationship, error) {
-	var packages []pkg.Package
+func parsePackageJSON(_ string, reader io.Reader) ([]*pkg.Package, []artifact.Relationship, error) {
+	var packages []*pkg.Package
 	dec := json.NewDecoder(reader)
 
 	for {
@@ -180,35 +179,46 @@ func parsePackageJSON(_ string, reader io.Reader) ([]pkg.Package, []artifact.Rel
 			return nil, nil, nil
 		}
 
-		licenses, err := licensesFromJSON(p)
-		if err != nil {
-			return nil, nil, fmt.Errorf("failed to parse package.json file: %w", err)
-		}
-
-		packages = append(packages, pkg.Package{
-			Name:         p.Name,
-			Version:      p.Version,
-			Licenses:     licenses,
-			Language:     pkg.JavaScript,
-			Type:         pkg.NpmPkg,
-			MetadataType: pkg.NpmPackageJSONMetadataType,
-			Metadata: pkg.NpmPackageJSONMetadata{
-				Author:   p.Author.AuthorString(),
-				Homepage: p.Homepage,
-				URL:      p.Repository.URL,
-				Licenses: licenses,
-			},
-		})
+		packages = append(packages, newPackageJSONPackage(p))
 	}
 
 	return packages, nil, nil
 }
 
+func newPackageJSONPackage(p PackageJSON) *pkg.Package {
+	licenses, err := p.licensesFromJSON()
+	if err != nil {
+		log.Warnf("unable to extract licenses from javascript package.json: %+v", err)
+	}
+
+	return &pkg.Package{
+		Name:         p.Name,
+		Version:      p.Version,
+		Licenses:     licenses,
+		Language:     pkg.JavaScript,
+		Type:         pkg.NpmPkg,
+		MetadataType: pkg.NpmPackageJSONMetadataType,
+		Metadata: pkg.NpmPackageJSONMetadata{
+			Author:   p.Author.AuthorString(),
+			Homepage: p.Homepage,
+			URL:      p.Repository.URL,
+			Licenses: licenses,
+		},
+	}
+}
+
 func (p PackageJSON) hasNameAndVersionValues() bool {
 	return p.Name != "" && p.Version != ""
 }
 
-func pathContainsNodeModulesDirectory(path string) bool {
-	isNodeModulesPath, _ := regexp.MatchString("[\\/]node_modules[\\/]", path)
-	return isNodeModulesPath
+// this supports both windows and unix paths
+var filepathSeparator = regexp.MustCompile(`[\\/]`)
+
+func pathContainsNodeModulesDirectory(p string) bool {
+	for _, subPath := range filepathSeparator.Split(p, -1) {
+		if subPath == "node_modules" {
+			return true
+		}
+	}
+	return false
 }

syft/pkg/cataloger/javascript/parse_package_json_test.go

@@ -6,6 +6,7 @@ import (
 
 	"github.com/anchore/syft/syft/pkg"
 	"github.com/go-test/deep"
+	"github.com/stretchr/testify/assert"
 )
 
 func TestParsePackageJSON(t *testing.T) {
@@ -135,7 +136,7 @@ func TestParsePackageJSON(t *testing.T) {
 				t.Fatalf("unexpected package count: %d!=1", len(actual))
 			}
 
-			for _, d := range deep.Equal(actual[0], test.ExpectedPkg) {
+			for _, d := range deep.Equal(actual[0], &test.ExpectedPkg) {
 
 				t.Errorf("diff: %+v", d)
 			}
@@ -160,3 +161,50 @@ func TestParsePackageJSON_Partial(t *testing.T) { // see https://github.com/anch
 		t.Errorf("no packages should've been returned (but got %d packages)", actualCount)
 	}
 }
+
+func Test_pathContainsNodeModulesDirectory(t *testing.T) {
+	tests := []struct {
+		path     string
+		expected bool
+	}{
+		// positive
+		{
+			path:     "something/node_modules/package",
+			expected: true,
+		},
+		{
+			path:     "node_modules/package",
+			expected: true,
+		},
+		{
+			path:     "something/node_modules",
+			expected: true,
+		},
+		{
+			path:     "\\something\\node_modules\\",
+			expected: true,
+		},
+		{
+			path:     "\\something\\node_modules",
+			expected: true,
+		},
+		// negative
+		{
+			path:     "something/node_bogus_modules",
+			expected: false,
+		},
+		{
+			path:     "something/node_modules_bogus",
+			expected: false,
+		},
+		{
+			path:     "something/node_bogus_modules/package",
+			expected: false,
+		},
+	}
+	for _, test := range tests {
+		t.Run(test.path, func(t *testing.T) {
+			assert.Equal(t, test.expected, pathContainsNodeModulesDirectory(test.path))
+		})
+	}
+}

syft/pkg/cataloger/javascript/parse_package_lock.go

@@ -29,14 +29,14 @@ type Dependency struct {
 }
 
 // parsePackageLock parses a package-lock.json and returns the discovered JavaScript packages.
-func parsePackageLock(path string, reader io.Reader) ([]pkg.Package, []artifact.Relationship, error) {
+func parsePackageLock(path string, reader io.Reader) ([]*pkg.Package, []artifact.Relationship, error) {
 	// in the case we find package-lock.json files in the node_modules directories, skip those
 	// as the whole purpose of the lock file is for the specific dependencies of the root project
 	if pathContainsNodeModulesDirectory(path) {
 		return nil, nil, nil
 	}
 
-	var packages []pkg.Package
+	var packages []*pkg.Package
 	dec := json.NewDecoder(reader)
 
 	for {
@@ -47,7 +47,7 @@ func parsePackageLock(path string, reader io.Reader) ([]pkg.Package, []artifact.
 			return nil, nil, fmt.Errorf("failed to parse package-lock.json file: %w", err)
 		}
 		for name, pkgMeta := range lock.Dependencies {
-			packages = append(packages, pkg.Package{
+			packages = append(packages, &pkg.Package{
 				Name:     name,
 				Version:  pkgMeta.Version,
 				Language: pkg.JavaScript,

syft/pkg/cataloger/javascript/parse_package_lock_test.go

@@ -4,10 +4,12 @@ import (
 	"os"
 	"testing"
 
+	"github.com/stretchr/testify/assert"
+
 	"github.com/anchore/syft/syft/pkg"
 )
 
-func assertPkgsEqual(t *testing.T, actual []pkg.Package, expected map[string]pkg.Package) {
+func assertPkgsEqual(t *testing.T, actual []*pkg.Package, expected map[string]pkg.Package) {
 	t.Helper()
 	if len(actual) != len(expected) {
 		for _, a := range actual {
@@ -18,26 +20,11 @@ func assertPkgsEqual(t *testing.T, actual []pkg.Package, expected map[string]pkg
 
 	for _, a := range actual {
 		expectedPkg, ok := expected[a.Name]
-		if !ok {
-			t.Errorf("unexpected package found: '%s'", a.Name)
-		}
-
-		if expectedPkg.Version != a.Version {
-			t.Errorf("%s : unexpected package version: '%s', expected: '%s'", a.Name, a.Version, expectedPkg.Version)
-		}
-
-		if a.Language != expectedPkg.Language {
-			t.Errorf("%s : bad language: '%+v', expected: '%+v'", a.Name, a.Language, expectedPkg.Language)
-		}
-
-		if a.Type != expectedPkg.Type {
-			t.Errorf("%s : bad package type: %+v, expected: %+v", a.Name, a.Type, expectedPkg.Type)
-		}
-
-		if len(a.Licenses) < len(expectedPkg.Licenses) {
-			t.Errorf("%s : bad package licenses count: '%+v'", a.Name, a.Licenses)
-		}
-
+		assert.True(t, ok)
+		assert.Equal(t, expectedPkg.Version, a.Version, "bad version")
+		assert.Equal(t, expectedPkg.Language, a.Language, "bad language")
+		assert.Equal(t, expectedPkg.Type, a.Type, "bad type")
+		assert.Equal(t, expectedPkg.Licenses, a.Licenses, "bad license count")
 	}
 }
 

syft/pkg/cataloger/javascript/parse_yarn_lock.go

@@ -35,14 +35,14 @@ const (
 	noVersion = ""
 )
 
-func parseYarnLock(path string, reader io.Reader) ([]pkg.Package, []artifact.Relationship, error) {
+func parseYarnLock(path string, reader io.Reader) ([]*pkg.Package, []artifact.Relationship, error) {
 	// in the case we find yarn.lock files in the node_modules directories, skip those
 	// as the whole purpose of the lock file is for the specific dependencies of the project
 	if pathContainsNodeModulesDirectory(path) {
 		return nil, nil, nil
 	}
 
-	var packages []pkg.Package
+	var packages []*pkg.Package
 	scanner := bufio.NewScanner(reader)
 	parsedPackages := internal.NewStringSet()
 	currentPackage := noPackage
@@ -106,8 +106,8 @@ func findPackageVersion(line string) string {
 	return noVersion
 }
 
-func newYarnLockPackage(name, version string) pkg.Package {
-	return pkg.Package{
+func newYarnLockPackage(name, version string) *pkg.Package {
+	return &pkg.Package{
 		Name:     name,
 		Version:  version,
 		Language: pkg.JavaScript,

syft/pkg/cataloger/package_url_test.go

@@ -10,11 +10,13 @@ import (
 
 func TestPackageURL(t *testing.T) {
 	tests := []struct {
+		name     string
 		pkg      pkg.Package
 		distro   *distro.Distro
 		expected string
 	}{
 		{
+			name: "golang",
 			pkg: pkg.Package{
 				Name:    "github.com/anchore/syft",
 				Version: "v0.1.0",
@@ -23,14 +25,38 @@ func TestPackageURL(t *testing.T) {
 			expected: "pkg:golang/github.com/anchore/syft@v0.1.0",
 		},
 		{
+			name: "pip with vcs url",
 			pkg: pkg.Package{
-				Name:    "name",
-				Version: "v0.1.0",
+				Name:    "bad-name",
+				Version: "bad-v0.1.0",
 				Type:    pkg.PythonPkg,
+				Metadata: pkg.PythonPackageMetadata{
+					Name:    "name",
+					Version: "v0.1.0",
+					DirectURLOrigin: &pkg.PythonDirectURLOriginInfo{
+						VCS:      "git",
+						URL:      "https://github.com/test/test.git",
+						CommitID: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
+					},
+				},
+			},
+			expected: "pkg:pypi/name@v0.1.0?vcs_url=git+https:%2F%2Fgithub.com%2Ftest%2Ftest.git@aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
+		},
+		{
+			name: "pip without vcs url",
+			pkg: pkg.Package{
+				Name:    "bad-name",
+				Version: "bad-v0.1.0",
+				Type:    pkg.PythonPkg,
+				Metadata: pkg.PythonPackageMetadata{
+					Name:    "name",
+					Version: "v0.1.0",
+				},
 			},
 			expected: "pkg:pypi/name@v0.1.0",
 		},
 		{
+			name: "gem",
 			pkg: pkg.Package{
 				Name:    "name",
 				Version: "v0.1.0",
@@ -39,6 +65,7 @@ func TestPackageURL(t *testing.T) {
 			expected: "pkg:gem/name@v0.1.0",
 		},
 		{
+			name: "npm",
 			pkg: pkg.Package{
 				Name:    "name",
 				Version: "v0.1.0",
@@ -47,6 +74,7 @@ func TestPackageURL(t *testing.T) {
 			expected: "pkg:npm/name@v0.1.0",
 		},
 		{
+			name: "deb with arch",
 			distro: &distro.Distro{
 				Type: distro.Ubuntu,
 			},
@@ -63,6 +91,7 @@ func TestPackageURL(t *testing.T) {
 			expected: "pkg:deb/ubuntu/name@v0.1.0?arch=amd64",
 		},
 		{
+			name: "deb with epoch",
 			distro: &distro.Distro{
 				Type: distro.CentOS,
 			},
@@ -81,6 +110,7 @@ func TestPackageURL(t *testing.T) {
 			expected: "pkg:rpm/centos/name@0.1.0-3?arch=amd64&epoch=2",
 		},
 		{
+			name: "deb with nil epoch",
 			distro: &distro.Distro{
 				Type: distro.CentOS,
 			},
@@ -99,6 +129,7 @@ func TestPackageURL(t *testing.T) {
 			expected: "pkg:rpm/centos/name@0.1.0-3?arch=amd64",
 		},
 		{
+			name: "deb with unknown distro",
 			distro: &distro.Distro{
 				Type: distro.UnknownDistroType,
 			},
@@ -110,6 +141,7 @@ func TestPackageURL(t *testing.T) {
 			expected: "pkg:deb/name@v0.1.0",
 		},
 		{
+			name: "cargo",
 			pkg: pkg.Package{
 				Name:    "name",
 				Version: "v0.1.0",
@@ -120,7 +152,7 @@ func TestPackageURL(t *testing.T) {
 	}
 
 	for _, test := range tests {
-		t.Run(string(test.pkg.Type)+"|"+test.expected, func(t *testing.T) {
+		t.Run(test.name, func(t *testing.T) {
 			actual := generatePackageURL(test.pkg, test.distro)
 			if actual != test.expected {
 				dmp := diffmatchpatch.New()

syft/pkg/cataloger/php/parse_composer_lock.go

@@ -8,7 +8,6 @@ import (
 	"github.com/anchore/syft/syft/artifact"
 
 	"github.com/anchore/syft/syft/pkg"
-	"github.com/anchore/syft/syft/pkg/cataloger/common"
 )
 
 type ComposerLock struct {
@@ -21,12 +20,9 @@ type Dependency struct {
 	Version string `json:"version"`
 }
 
-// integrity check
-var _ common.ParserFn = parseComposerLock
-
 // parseComposerLock is a parser function for Composer.lock contents, returning "Default" php packages discovered.
-func parseComposerLock(_ string, reader io.Reader) ([]pkg.Package, []artifact.Relationship, error) {
-	packages := make([]pkg.Package, 0)
+func parseComposerLock(_ string, reader io.Reader) ([]*pkg.Package, []artifact.Relationship, error) {
+	packages := make([]*pkg.Package, 0)
 	dec := json.NewDecoder(reader)
 
 	for {
@@ -39,7 +35,7 @@ func parseComposerLock(_ string, reader io.Reader) ([]pkg.Package, []artifact.Re
 		for _, pkgMeta := range lock.Packages {
 			version := pkgMeta.Version
 			name := pkgMeta.Name
-			packages = append(packages, pkg.Package{
+			packages = append(packages, &pkg.Package{
 				Name:     name,
 				Version:  version,
 				Language: pkg.PHP,

syft/pkg/cataloger/php/parse_composer_lock_test.go

@@ -9,7 +9,7 @@ import (
 )
 
 func TestParseComposerFileLock(t *testing.T) {
-	expected := []pkg.Package{
+	expected := []*pkg.Package{
 		{
 			Name:     "adoy/fastcgi-client",
 			Version:  "1.0.2",

syft/pkg/cataloger/php/parse_installed_json.go

@@ -40,8 +40,8 @@ func (w *installedJSONComposerV2) UnmarshalJSON(data []byte) error {
 var _ common.ParserFn = parseComposerLock
 
 // parseComposerLock is a parser function for Composer.lock contents, returning "Default" php packages discovered.
-func parseInstalledJSON(_ string, reader io.Reader) ([]pkg.Package, []artifact.Relationship, error) {
-	packages := make([]pkg.Package, 0)
+func parseInstalledJSON(_ string, reader io.Reader) ([]*pkg.Package, []artifact.Relationship, error) {
+	packages := make([]*pkg.Package, 0)
 	dec := json.NewDecoder(reader)
 
 	for {
@@ -54,7 +54,7 @@ func parseInstalledJSON(_ string, reader io.Reader) ([]pkg.Package, []artifact.R
 		for _, pkgMeta := range lock.Packages {
 			version := pkgMeta.Version
 			name := pkgMeta.Name
-			packages = append(packages, pkg.Package{
+			packages = append(packages, &pkg.Package{
 				Name:     name,
 				Version:  version,
 				Language: pkg.PHP,

syft/pkg/cataloger/php/parse_installed_json_test.go

@@ -9,7 +9,7 @@ import (
 )
 
 func TestParseInstalledJsonComposerV1(t *testing.T) {
-	expected := []pkg.Package{
+	expected := []*pkg.Package{
 		{
 			Name:     "asm89/stack-cors",
 			Version:  "1.3.0",
@@ -41,7 +41,7 @@ func TestParseInstalledJsonComposerV1(t *testing.T) {
 }
 
 func TestParseInstalledJsonComposerV2(t *testing.T) {
-	expected := []pkg.Package{
+	expected := []*pkg.Package{
 		{
 			Name:     "asm89/stack-cors",
 			Version:  "1.3.0",

syft/pkg/cataloger/python/package_cataloger.go

@@ -2,7 +2,9 @@ package python
 
 import (
 	"bufio"
+	"encoding/json"
 	"fmt"
+	"io/ioutil"
 	"path/filepath"
 
 	"github.com/anchore/syft/internal"
@@ -74,7 +76,7 @@ func (c *PackageCataloger) catalogEggOrWheel(resolver source.FileResolver, metad
 		licenses = []string{metadata.License}
 	}
 
-	return &pkg.Package{
+	p := &pkg.Package{
 		Name:         metadata.Name,
 		Version:      metadata.Version,
 		FoundBy:      c.Name(),
@@ -84,7 +86,11 @@ func (c *PackageCataloger) catalogEggOrWheel(resolver source.FileResolver, metad
 		Type:         pkg.PythonPkg,
 		MetadataType: pkg.PythonPackageMetadataType,
 		Metadata:     *metadata,
-	}, nil
+	}
+
+	p.SetID()
+
+	return p, nil
 }
 
 // fetchRecordFiles finds a corresponding RECORD file for the given python package metadata file and returns the set of file records contained.
@@ -148,6 +154,40 @@ func (c *PackageCataloger) fetchTopLevelPackages(resolver source.FileResolver, m
 	return pkgs, sources, nil
 }
 
+func (c *PackageCataloger) fetchDirectURLData(resolver source.FileResolver, metadataLocation source.Location) (d *pkg.PythonDirectURLOriginInfo, sources []source.Location, err error) {
+	parentDir := filepath.Dir(metadataLocation.RealPath)
+	directURLPath := filepath.Join(parentDir, "direct_url.json")
+	directURLLocation := resolver.RelativeFileByPath(metadataLocation, directURLPath)
+
+	if directURLLocation == nil {
+		return nil, nil, nil
+	}
+
+	sources = append(sources, *directURLLocation)
+
+	directURLContents, err := resolver.FileContentsByLocation(*directURLLocation)
+	if err != nil {
+		return nil, nil, err
+	}
+	defer internal.CloseAndLogError(directURLContents, directURLLocation.VirtualPath)
+
+	buffer, err := ioutil.ReadAll(directURLContents)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	var directURLJson pkg.DirectURLOrigin
+	if err := json.Unmarshal(buffer, &directURLJson); err != nil {
+		return nil, nil, err
+	}
+
+	return &pkg.PythonDirectURLOriginInfo{
+		URL:      directURLJson.URL,
+		CommitID: directURLJson.VCSInfo.CommitID,
+		VCS:      directURLJson.VCSInfo.VCS,
+	}, sources, nil
+}
+
 // assembleEggOrWheelMetadata discovers and accumulates python package metadata from multiple file sources and returns a single metadata object as well as a list of files where the metadata was derived from.
 func (c *PackageCataloger) assembleEggOrWheelMetadata(resolver source.FileResolver, metadataLocation source.Location) (*pkg.PythonPackageMetadata, []source.Location, error) {
 	var sources = []source.Location{metadataLocation}
@@ -179,5 +219,13 @@ func (c *PackageCataloger) assembleEggOrWheelMetadata(resolver source.FileResolv
 	sources = append(sources, s...)
 	metadata.TopLevelPackages = p
 
+	// attach any direct-url package data found for the given wheel/egg installation
+	d, s, err := c.fetchDirectURLData(resolver, metadataLocation)
+	if err != nil {
+		return nil, nil, err
+	}
+	sources = append(sources, s...)
+	metadata.DirectURLOrigin = d
+
 	return &metadata, sources, nil
 }

syft/pkg/cataloger/python/package_cataloger_test.go

@@ -55,6 +55,7 @@ func TestPythonPackageWheelCataloger(t *testing.T) {
 				"test-fixtures/dist-info/METADATA",
 				"test-fixtures/dist-info/RECORD",
 				"test-fixtures/dist-info/top_level.txt",
+				"test-fixtures/dist-info/direct_url.json",
 			},
 			expectedPackage: pkg.Package{
 				Name:         "Pygments",
@@ -82,6 +83,7 @@ func TestPythonPackageWheelCataloger(t *testing.T) {
 						{Path: "pygments/x_util.py", Digest: &pkg.PythonFileDigest{"sha256", "qpzzsOW31KT955agi-7NS--90I0iNiJCyLJQnRCHgKI="}, Size: "10778"},
 					},
 					TopLevelPackages: []string{"pygments", "something_else"},
+					DirectURLOrigin:  &pkg.PythonDirectURLOriginInfo{URL: "https://github.com/python-test/test.git", VCS: "git", CommitID: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"},
 				},
 			},
 		},

syft/pkg/cataloger/python/parse_pipfile_lock.go

@@ -38,8 +38,8 @@ type Dependency struct {
 var _ common.ParserFn = parsePipfileLock
 
 // parsePipfileLock is a parser function for Pipfile.lock contents, returning "Default" python packages discovered.
-func parsePipfileLock(_ string, reader io.Reader) ([]pkg.Package, []artifact.Relationship, error) {
-	packages := make([]pkg.Package, 0)
+func parsePipfileLock(_ string, reader io.Reader) ([]*pkg.Package, []artifact.Relationship, error) {
+	packages := make([]*pkg.Package, 0)
 	dec := json.NewDecoder(reader)
 
 	for {
@@ -51,7 +51,7 @@ func parsePipfileLock(_ string, reader io.Reader) ([]pkg.Package, []artifact.Rel
 		}
 		for name, pkgMeta := range lock.Default {
 			version := strings.TrimPrefix(pkgMeta.Version, "==")
-			packages = append(packages, pkg.Package{
+			packages = append(packages, &pkg.Package{
 				Name:     name,
 				Version:  version,
 				Language: pkg.Python,

syft/pkg/cataloger/python/parse_poetry_lock.go

@@ -14,7 +14,7 @@ import (
 var _ common.ParserFn = parsePoetryLock
 
 // parsePoetryLock is a parser function for poetry.lock contents, returning all python packages discovered.
-func parsePoetryLock(_ string, reader io.Reader) ([]pkg.Package, []artifact.Relationship, error) {
+func parsePoetryLock(_ string, reader io.Reader) ([]*pkg.Package, []artifact.Relationship, error) {
 	tree, err := toml.LoadReader(reader)
 	if err != nil {
 		return nil, nil, fmt.Errorf("unable to load poetry.lock for parsing: %v", err)

syft/pkg/cataloger/python/parse_poetry_lock_test.go

@@ -9,7 +9,7 @@ import (
 )
 
 func TestParsePoetryLock(t *testing.T) {
-	expected := []pkg.Package{
+	expected := []*pkg.Package{
 		{
 			Name:     "added-value",
 			Version:  "0.14.2",

syft/pkg/cataloger/python/parse_requirements.go

@@ -16,8 +16,8 @@ var _ common.ParserFn = parseRequirementsTxt
 
 // parseRequirementsTxt takes a Python requirements.txt file, returning all Python packages that are locked to a
 // specific version.
-func parseRequirementsTxt(_ string, reader io.Reader) ([]pkg.Package, []artifact.Relationship, error) {
-	packages := make([]pkg.Package, 0)
+func parseRequirementsTxt(_ string, reader io.Reader) ([]*pkg.Package, []artifact.Relationship, error) {
+	packages := make([]*pkg.Package, 0)
 
 	scanner := bufio.NewScanner(reader)
 	for scanner.Scan() {
@@ -44,7 +44,7 @@ func parseRequirementsTxt(_ string, reader io.Reader) ([]pkg.Package, []artifact
 			parts := strings.Split(uncommented, "==")
 			name := strings.TrimSpace(parts[0])
 			version := strings.TrimSpace(parts[1])
-			packages = append(packages, pkg.Package{
+			packages = append(packages, &pkg.Package{
 				Name:     name,
 				Version:  version,
 				Language: pkg.Python,

syft/pkg/cataloger/python/parse_requirements_test.go

@@ -4,12 +4,14 @@ import (
 	"os"
 	"testing"
 
+	"github.com/stretchr/testify/assert"
+
 	"github.com/go-test/deep"
 
 	"github.com/anchore/syft/syft/pkg"
 )
 
-func assertPackagesEqual(t *testing.T, actual []pkg.Package, expected map[string]pkg.Package) {
+func assertPackagesEqual(t *testing.T, actual []*pkg.Package, expected map[string]pkg.Package) {
 	t.Helper()
 	if len(actual) != len(expected) {
 		for _, a := range actual {
@@ -20,11 +22,9 @@ func assertPackagesEqual(t *testing.T, actual []pkg.Package, expected map[string
 
 	for _, a := range actual {
 		expectedPkg, ok := expected[a.Name]
-		if !ok {
-			t.Errorf("unexpected package found: '%s'", a.Name)
-		}
+		assert.True(t, ok)
 
-		for _, d := range deep.Equal(a, expectedPkg) {
+		for _, d := range deep.Equal(a, &expectedPkg) {
 			t.Errorf("diff: %+v", d)
 		}
 	}

syft/pkg/cataloger/python/parse_setup.go

@@ -20,8 +20,8 @@ var _ common.ParserFn = parseSetup
 //		" mypy2 == v0.770", ' mypy3== v0.770',  --> match(name=mypy2 version=v0.770), match(name=mypy3, version=v0.770)
 var pinnedDependency = regexp.MustCompile(`['"]\W?(\w+\W?==\W?[\w\.]*)`)
 
-func parseSetup(_ string, reader io.Reader) ([]pkg.Package, []artifact.Relationship, error) {
-	packages := make([]pkg.Package, 0)
+func parseSetup(_ string, reader io.Reader) ([]*pkg.Package, []artifact.Relationship, error) {
+	packages := make([]*pkg.Package, 0)
 
 	scanner := bufio.NewScanner(reader)
 
@@ -38,7 +38,7 @@ func parseSetup(_ string, reader io.Reader) ([]pkg.Package, []artifact.Relations
 			name = strings.TrimSpace(name)
 
 			version := strings.TrimSpace(parts[len(parts)-1])
-			packages = append(packages, pkg.Package{
+			packages = append(packages, &pkg.Package{
 				Name:     strings.Trim(name, "'\""),
 				Version:  strings.Trim(version, "'\""),
 				Language: pkg.Python,

syft/pkg/cataloger/python/poetry_metadata.go

@@ -7,8 +7,8 @@ type PoetryMetadata struct {
 }
 
 // Pkgs returns all of the packages referenced within the poetry.lock metadata.
-func (m PoetryMetadata) Pkgs() []pkg.Package {
-	pkgs := make([]pkg.Package, 0)
+func (m PoetryMetadata) Pkgs() []*pkg.Package {
+	pkgs := make([]*pkg.Package, 0)
 
 	for _, p := range m.Packages {
 		pkgs = append(pkgs, p.Pkg())

syft/pkg/cataloger/python/poetry_metadata_package.go

@@ -11,8 +11,8 @@ type PoetryMetadataPackage struct {
 }
 
 // Pkg returns the standard `pkg.Package` representation of the package referenced within the poetry.lock metadata.
-func (p PoetryMetadataPackage) Pkg() pkg.Package {
-	return pkg.Package{
+func (p PoetryMetadataPackage) Pkg() *pkg.Package {
+	return &pkg.Package{
 		Name:     p.Name,
 		Version:  p.Version,
 		Language: pkg.Python,

syft/pkg/cataloger/python/test-fixtures/dist-info/direct_url.json

@@ -0,0 +1 @@
+{"url": "https://github.com/python-test/test.git", "vcs_info": {"commit_id": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", "vcs": "git"}}

syft/pkg/cataloger/rpmdb/parse_rpmdb.go

@@ -70,6 +70,8 @@ func parseRpmDB(resolver source.FilePathResolver, dbLocation source.Location, re
 			Metadata:     metadata,
 		}
 
+		p.SetID()
+
 		allPkgs = append(allPkgs, p)
 	}
 

syft/pkg/cataloger/ruby/catalogers.go

@@ -1,5 +1,5 @@
 /*
-Package bundler provides a concrete Cataloger implementation for Ruby Gemfile.lock bundler files.
+Package ruby bundler provides a concrete Cataloger implementation for Ruby Gemfile.lock bundler files.
 */
 package ruby
 

syft/pkg/cataloger/ruby/parse_gemfile_lock.go

@@ -17,8 +17,8 @@ var _ common.ParserFn = parseGemFileLockEntries
 var sectionsOfInterest = internal.NewStringSetFromSlice([]string{"GEM"})
 
 // parseGemFileLockEntries is a parser function for Gemfile.lock contents, returning all Gems discovered.
-func parseGemFileLockEntries(_ string, reader io.Reader) ([]pkg.Package, []artifact.Relationship, error) {
-	pkgs := make([]pkg.Package, 0)
+func parseGemFileLockEntries(_ string, reader io.Reader) ([]*pkg.Package, []artifact.Relationship, error) {
+	pkgs := make([]*pkg.Package, 0)
 	scanner := bufio.NewScanner(reader)
 
 	var currentSection string
@@ -41,7 +41,7 @@ func parseGemFileLockEntries(_ string, reader io.Reader) ([]pkg.Package, []artif
 			if len(candidate) != 2 {
 				continue
 			}
-			pkgs = append(pkgs, pkg.Package{
+			pkgs = append(pkgs, &pkg.Package{
 				Name:     candidate[0],
 				Version:  strings.Trim(candidate[1], "()"),
 				Language: pkg.Ruby,

syft/pkg/cataloger/ruby/parse_gemspec.go

@@ -61,8 +61,8 @@ func processList(s string) []string {
 	return results
 }
 
-func parseGemSpecEntries(_ string, reader io.Reader) ([]pkg.Package, []artifact.Relationship, error) {
-	var pkgs []pkg.Package
+func parseGemSpecEntries(_ string, reader io.Reader) ([]*pkg.Package, []artifact.Relationship, error) {
+	var pkgs []*pkg.Package
 	var fields = make(map[string]interface{})
 	scanner := bufio.NewScanner(reader)
 
@@ -97,7 +97,7 @@ func parseGemSpecEntries(_ string, reader io.Reader) ([]pkg.Package, []artifact.
 			return nil, nil, fmt.Errorf("unable to decode gem metadata: %w", err)
 		}
 
-		pkgs = append(pkgs, pkg.Package{
+		pkgs = append(pkgs, &pkg.Package{
 			Name:         metadata.Name,
 			Version:      metadata.Version,
 			Licenses:     metadata.Licenses,

syft/pkg/cataloger/ruby/parse_gemspec_test.go

@@ -44,7 +44,7 @@ func TestParseGemspec(t *testing.T) {
 		t.Fatalf("unexpected package count: %d!=1", len(actual))
 	}
 
-	for _, d := range deep.Equal(actual[0], expectedPkg) {
+	for _, d := range deep.Equal(actual[0], &expectedPkg) {
 		t.Errorf("diff: %+v", d)
 	}
 }

syft/pkg/cataloger/rust/cargo_metadata.go

@@ -7,8 +7,8 @@ type CargoMetadata struct {
 }
 
 // Pkgs returns all of the packages referenced within the Cargo.lock metadata.
-func (m CargoMetadata) Pkgs() []pkg.Package {
-	pkgs := make([]pkg.Package, 0)
+func (m CargoMetadata) Pkgs() []*pkg.Package {
+	pkgs := make([]*pkg.Package, 0)
 
 	for _, p := range m.Packages {
 		if p.Dependencies == nil {

syft/pkg/cataloger/rust/parse_cargo_lock.go

@@ -14,7 +14,7 @@ import (
 var _ common.ParserFn = parseCargoLock
 
 // parseCargoLock is a parser function for Cargo.lock contents, returning all rust cargo crates discovered.
-func parseCargoLock(_ string, reader io.Reader) ([]pkg.Package, []artifact.Relationship, error) {
+func parseCargoLock(_ string, reader io.Reader) ([]*pkg.Package, []artifact.Relationship, error) {
 	tree, err := toml.LoadReader(reader)
 	if err != nil {
 		return nil, nil, fmt.Errorf("unable to load Cargo.lock for parsing: %v", err)

syft/pkg/cataloger/rust/parse_cargo_lock_test.go

@@ -9,7 +9,7 @@ import (
 )
 
 func TestParseCargoLock(t *testing.T) {
-	expected := []pkg.Package{
+	expected := []*pkg.Package{
 		{
 			Name:         "ansi_term",
 			Version:      "0.12.1",

syft/pkg/package.go

@@ -14,6 +14,7 @@ import (
 // Package represents an application or library that has been bundled into a distributable format.
 // TODO: if we ignore FoundBy for ID generation should we merge the field to show it was found in two places?
 type Package struct {
+	id           artifact.ID       `hash:"ignore"`
 	Name         string            // the package name
 	Version      string            // the version of the package
 	FoundBy      string            // the specific cataloger that discovered this package
@@ -21,24 +22,27 @@ type Package struct {
 	Licenses     []string          // licenses discovered with the package metadata
 	Language     Language          // the language ecosystem this package belongs to (e.g. JavaScript, Python, etc)
 	Type         Type              // the package type (e.g. Npm, Yarn, Python, Rpm, Deb, etc)
-	CPEs         []CPE             // all possible Common Platform Enumerators
-	PURL         string            // the Package URL (see https://github.com/package-url/purl-spec)
+	CPEs         []CPE             `hash:"ignore"` // all possible Common Platform Enumerators (note: this is NOT included in the definition of the ID since all fields on a CPE are derived from other fields)
+	PURL         string            `hash:"ignore"` // the Package URL (see https://github.com/package-url/purl-spec) (note: this is NOT included in the definition of the ID since all fields on a pURL are derived from other fields)
 	MetadataType MetadataType      // the shape of the additional data in the "metadata" field
 	Metadata     interface{}       // additional data found while parsing the package source
 }
 
-func (p Package) ID() artifact.ID {
-	f, err := artifact.IDFromHash(p)
+func (p *Package) SetID() {
+	id, err := artifact.IDByHash(p)
 	if err != nil {
 		// TODO: what to do in this case?
 		log.Warnf("unable to get fingerprint of package=%s@%s: %+v", p.Name, p.Version, err)
-		return ""
+		return
 	}
+	p.id = id
+}
 
-	return f
+func (p Package) ID() artifact.ID {
+	return p.id
 }
 
 // Stringer to represent a package.
 func (p Package) String() string {
-	return fmt.Sprintf("Pkg(type=%s, name=%s, version=%s)", p.Type, p.Name, p.Version)
+	return fmt.Sprintf("Pkg(name=%q version=%q type=%q id=%q)", p.Name, p.Version, p.Type, p.id)
 }

syft/pkg/package_test.go

@@ -127,20 +127,20 @@ func TestFingerprint(t *testing.T) {
 			expectIdentical: false,
 		},
 		{
-			name: "CPEs is reflected",
+			name: "CPEs is ignored",
 			transform: func(pkg Package) Package {
 				pkg.CPEs = []CPE{}
 				return pkg
 			},
-			expectIdentical: false,
+			expectIdentical: true,
 		},
 		{
-			name: "pURL is reflected",
+			name: "pURL is ignored",
 			transform: func(pkg Package) Package {
 				pkg.PURL = "new!"
 				return pkg
 			},
-			expectIdentical: false,
+			expectIdentical: true,
 		},
 		{
 			name: "language is reflected",
@@ -190,7 +190,10 @@ func TestFingerprint(t *testing.T) {
 
 	for _, test := range tests {
 		t.Run(test.name, func(t *testing.T) {
+			originalPkg.SetID()
 			transformedPkg := test.transform(originalPkg)
+			transformedPkg.SetID()
+
 			originalFingerprint := originalPkg.ID()
 			assert.NotEmpty(t, originalFingerprint)
 			transformedFingerprint := transformedPkg.ID()

syft/pkg/python_package_metadata.go

@@ -1,8 +1,10 @@
 package pkg
 
 import (
+	"fmt"
 	"sort"
 
+	"github.com/anchore/packageurl-go"
 	"github.com/scylladb/go-set/strset"
 )
 
@@ -21,17 +23,45 @@ type PythonFileRecord struct {
 	Size   string            `json:"size,omitempty"`
 }
 
+type PythonDirectURLOriginInfo struct {
+	URL      string `json:"url"`
+	CommitID string `json:"commitId,omitempty"`
+	VCS      string `json:"vcs,omitempty"`
+}
+
 // PythonPackageMetadata represents all captured data for a python egg or wheel package.
 type PythonPackageMetadata struct {
-	Name                 string             `json:"name" mapstruct:"Name"`
-	Version              string             `json:"version" mapstruct:"Version"`
-	License              string             `json:"license" mapstruct:"License"`
-	Author               string             `json:"author" mapstruct:"Author"`
-	AuthorEmail          string             `json:"authorEmail" mapstruct:"Authoremail"`
-	Platform             string             `json:"platform" mapstruct:"Platform"`
-	Files                []PythonFileRecord `json:"files,omitempty"`
-	SitePackagesRootPath string             `json:"sitePackagesRootPath"`
-	TopLevelPackages     []string           `json:"topLevelPackages,omitempty"`
+	Name                 string                     `json:"name" mapstruct:"Name"`
+	Version              string                     `json:"version" mapstruct:"Version"`
+	License              string                     `json:"license" mapstruct:"License"`
+	Author               string                     `json:"author" mapstruct:"Author"`
+	AuthorEmail          string                     `json:"authorEmail" mapstruct:"Authoremail"`
+	Platform             string                     `json:"platform" mapstruct:"Platform"`
+	Files                []PythonFileRecord         `json:"files,omitempty"`
+	SitePackagesRootPath string                     `json:"sitePackagesRootPath"`
+	TopLevelPackages     []string                   `json:"topLevelPackages,omitempty"`
+	DirectURLOrigin      *PythonDirectURLOriginInfo `json:"directUrlOrigin,omitempty"`
+}
+
+type DirectURLOrigin struct {
+	URL         string      `json:"url"`
+	VCSInfo     VCSInfo     `json:"vcs_info"`
+	ArchiveInfo ArchiveInfo `json:"archive_info"`
+	DirInfo     DirInfo     `json:"dir_info"`
+}
+
+type DirInfo struct {
+	Editable bool `json:"editable"`
+}
+
+type ArchiveInfo struct {
+	Hash string `json:"hash"`
+}
+
+type VCSInfo struct {
+	CommitID          string `json:"commit_id"`
+	VCS               string `json:"vcs"`
+	RequestedRevision string `json:"requested_revision"`
 }
 
 func (m PythonPackageMetadata) OwnedFiles() (result []string) {
@@ -45,3 +75,33 @@ func (m PythonPackageMetadata) OwnedFiles() (result []string) {
 	sort.Strings(result)
 	return result
 }
+
+func (m PythonPackageMetadata) PackageURL() string {
+	// generate a purl from the package data
+	pURL := packageurl.NewPackageURL(
+		packageurl.TypePyPi,
+		"",
+		m.Name,
+		m.Version,
+		m.purlQualifiers(),
+		"")
+
+	return pURL.ToString()
+}
+
+func (m PythonPackageMetadata) purlQualifiers() packageurl.Qualifiers {
+	q := packageurl.Qualifiers{}
+	if m.DirectURLOrigin != nil {
+		q = append(q, m.DirectURLOrigin.vcsURLQualifier()...)
+	}
+	return q
+}
+
+func (p PythonDirectURLOriginInfo) vcsURLQualifier() packageurl.Qualifiers {
+	if p.VCS != "" {
+		// Taken from https://github.com/package-url/purl-spec/blob/master/PURL-SPECIFICATION.rst#known-qualifiers-keyvalue-pairs
+		// packageurl-go still doesn't support all qualifier names
+		return packageurl.Qualifiers{{Key: "vcs_url", Value: fmt.Sprintf("%s+%s@%s", p.VCS, p.URL, p.CommitID)}}
+	}
+	return packageurl.Qualifiers{}
+}

syft/pkg/relationships_by_file_ownership.go

@@ -27,11 +27,11 @@ func RelationshipsByFileOwnership(catalog *Catalog) []artifact.Relationship {
 	var relationships = findOwnershipByFilesRelationships(catalog)
 
 	var edges []artifact.Relationship
-	for parent, children := range relationships {
-		for child, files := range children {
+	for parentID, children := range relationships {
+		for childID, files := range children {
 			edges = append(edges, artifact.Relationship{
-				From: catalog.byID[parent],
-				To:   catalog.byID[child],
+				From: catalog.byID[parentID],
+				To:   catalog.byID[childID],
 				Type: artifact.OwnershipByFileOverlapRelationship,
 				Data: ownershipByFilesMetadata{
 					Files: files.List(),

syft/pkg/relationships_by_file_ownership_test.go

@@ -32,6 +32,7 @@ func TestOwnershipByFilesRelationship(t *testing.T) {
 						},
 					},
 				}
+				parent.SetID()
 
 				child := Package{
 					Locations: []source.Location{
@@ -40,6 +41,7 @@ func TestOwnershipByFilesRelationship(t *testing.T) {
 					},
 					Type: NpmPkg,
 				}
+				child.SetID()
 
 				relationship := artifact.Relationship{
 					From: parent,
@@ -73,6 +75,7 @@ func TestOwnershipByFilesRelationship(t *testing.T) {
 						},
 					},
 				}
+				parent.SetID()
 
 				child := Package{
 					Locations: []source.Location{
@@ -81,6 +84,7 @@ func TestOwnershipByFilesRelationship(t *testing.T) {
 					},
 					Type: NpmPkg,
 				}
+				child.SetID()
 
 				relationship := artifact.Relationship{
 					From: parent,
@@ -114,6 +118,8 @@ func TestOwnershipByFilesRelationship(t *testing.T) {
 					},
 				}
 
+				parent.SetID()
+
 				child := Package{
 					Locations: []source.Location{
 						source.NewVirtualLocation("/c/path", "/another/path"),
@@ -122,6 +128,8 @@ func TestOwnershipByFilesRelationship(t *testing.T) {
 					Type: NpmPkg,
 				}
 
+				child.SetID()
+
 				return []Package{parent, child}, nil
 			},
 		},

syft/source/coordinates.go

@@ -27,7 +27,7 @@ func NewCoordinateSet(start ...Coordinates) CoordinateSet {
 }
 
 func (c Coordinates) ID() artifact.ID {
-	f, err := artifact.IDFromHash(c)
+	f, err := artifact.IDByHash(c)
 	if err != nil {
 		// TODO: what to do in this case?
 		log.Warnf("unable to get fingerprint of location coordinate=%+v: %+v", c, err)

syft/source/directory_resolver.go

@@ -4,14 +4,16 @@ import (
 	"errors"
 	"fmt"
 	"io"
+	"io/fs"
 	"os"
 	"path"
 	"path/filepath"
 	"strings"
 
+	"github.com/anchore/syft/internal"
+
 	"github.com/anchore/stereoscope/pkg/file"
 	"github.com/anchore/stereoscope/pkg/filetree"
-	"github.com/anchore/syft/internal"
 	"github.com/anchore/syft/internal/bus"
 	"github.com/anchore/syft/internal/log"
 	"github.com/anchore/syft/syft/event"
@@ -21,23 +23,21 @@ import (
 
 var unixSystemRuntimePrefixes = []string{
 	"/proc",
-	"/sys",
 	"/dev",
+	"/sys",
 }
 
-var ErrIgnoreIrregularFile = errors.New("ignoring irregular file type")
-
 var _ FileResolver = (*directoryResolver)(nil)
 
-type pathFilterFn func(string) bool
+type pathFilterFn func(string, os.FileInfo) bool
 
 // directoryResolver implements path and content access for the directory data source.
 type directoryResolver struct {
-	path              string
-	cwdRelativeToRoot string
-	cwd               string
-	fileTree          *filetree.FileTree
-	metadata          map[file.ID]FileMetadata
+	path                    string
+	currentWdRelativeToRoot string
+	currentWd               string
+	fileTree                *filetree.FileTree
+	metadata                map[file.ID]FileMetadata
 	// TODO: wire up to report these paths in the json report
 	pathFilterFns  []pathFilterFn
 	refsByMIMEType map[string][]file.Reference
@@ -45,39 +45,37 @@ type directoryResolver struct {
 }
 
 func newDirectoryResolver(root string, pathFilters ...pathFilterFn) (*directoryResolver, error) {
-	cwd, err := os.Getwd()
+	currentWd, err := os.Getwd()
 	if err != nil {
 		return nil, fmt.Errorf("could not create directory resolver: %w", err)
 	}
 
-	var cwdRelRoot string
+	var currentWdRelRoot string
 	if path.IsAbs(root) {
-		cwdRelRoot, err = filepath.Rel(cwd, root)
+		currentWdRelRoot, err = filepath.Rel(currentWd, root)
 		if err != nil {
 			return nil, fmt.Errorf("could not create directory resolver: %w", err)
 		}
-	}
-
-	if pathFilters == nil {
-		pathFilters = []pathFilterFn{isUnixSystemRuntimePath}
+	} else {
+		currentWdRelRoot = filepath.Clean(root)
 	}
 
 	resolver := directoryResolver{
-		path:              root,
-		cwd:               cwd,
-		cwdRelativeToRoot: cwdRelRoot,
-		fileTree:          filetree.NewFileTree(),
-		metadata:          make(map[file.ID]FileMetadata),
-		pathFilterFns:     pathFilters,
-		refsByMIMEType:    make(map[string][]file.Reference),
-		errPaths:          make(map[string]error),
+		path:                    root,
+		currentWd:               currentWd,
+		currentWdRelativeToRoot: currentWdRelRoot,
+		fileTree:                filetree.NewFileTree(),
+		metadata:                make(map[file.ID]FileMetadata),
+		pathFilterFns:           append([]pathFilterFn{isUnallowableFileType, isUnixSystemRuntimePath}, pathFilters...),
+		refsByMIMEType:          make(map[string][]file.Reference),
+		errPaths:                make(map[string]error),
 	}
 
 	return &resolver, indexAllRoots(root, resolver.indexTree)
 }
 
 func (r *directoryResolver) indexTree(root string, stager *progress.Stage) ([]string, error) {
-	log.Infof("indexing filesystem path=%q", root)
+	log.Debugf("indexing filesystem path=%q", root)
 
 	var roots []string
 	var err error
@@ -93,11 +91,8 @@ func (r *directoryResolver) indexTree(root string, stager *progress.Stage) ([]st
 	// but continue forth with index regardless if the given root path exists or not.
 	fi, err := os.Stat(root)
 	if err != nil && fi != nil && !fi.IsDir() {
-		newRoot, err := r.addPathToIndex(root, fi)
-		if err = r.handleFileAccessErr(root, err); err != nil {
-			return nil, fmt.Errorf("unable to index path: %w", err)
-		}
-
+		// note: we want to index the path regardless of an error stat-ing the path
+		newRoot, _ := r.indexPath(root, fi, nil)
 		if newRoot != "" {
 			roots = append(roots, newRoot)
 		}
@@ -108,25 +103,33 @@ func (r *directoryResolver) indexTree(root string, stager *progress.Stage) ([]st
 		func(path string, info os.FileInfo, err error) error {
 			stager.Current = path
 
-			newRoot, indexErr := r.indexPath(path, info, err)
+			newRoot, err := r.indexPath(path, info, err)
+
+			if err != nil {
+				return err
+			}
+
 			if newRoot != "" {
 				roots = append(roots, newRoot)
 			}
 
-			return indexErr
+			return nil
 		})
 }
 
 func (r *directoryResolver) indexPath(path string, info os.FileInfo, err error) (string, error) {
 	// ignore any path which a filter function returns true
 	for _, filterFn := range r.pathFilterFns {
-		if filterFn(path) {
+		if filterFn != nil && filterFn(path, info) {
+			if info.IsDir() {
+				return "", fs.SkipDir
+			}
 			return "", nil
 		}
 	}
 
-	if err = r.handleFileAccessErr(path, err); err != nil {
-		return "", err
+	if r.isFileAccessErr(path, err) {
+		return "", nil
 	}
 
 	// link cycles could cause a revisit --we should not allow this
@@ -141,69 +144,100 @@ func (r *directoryResolver) indexPath(path string, info os.FileInfo, err error)
 	}
 
 	newRoot, err := r.addPathToIndex(path, info)
-	if err = r.handleFileAccessErr(path, err); err != nil {
-		return "", fmt.Errorf("unable to index path: %w", err)
+	if r.isFileAccessErr(path, err) {
+		return "", nil
 	}
 
 	return newRoot, nil
 }
 
-func (r *directoryResolver) handleFileAccessErr(path string, err error) error {
-	if errors.Is(err, os.ErrPermission) || errors.Is(err, os.ErrNotExist) || errors.Is(err, ErrIgnoreIrregularFile) {
-		// don't allow for permission errors to stop indexing, keep track of the paths and continue.
+func (r *directoryResolver) isFileAccessErr(path string, err error) bool {
+	// don't allow for errors to stop indexing, keep track of the paths and continue.
+	if err != nil {
 		log.Warnf("unable to access path=%q: %+v", path, err)
 		r.errPaths[path] = err
-		return nil
-	} else if err != nil {
-		return fmt.Errorf("unable to access path=%q: %w", path, err)
+		return true
 	}
-	return nil
+	return false
 }
 
 func (r directoryResolver) addPathToIndex(p string, info os.FileInfo) (string, error) {
-	var ref *file.Reference
-	var err error
-	var newRoot string
-
-	switch newFileTypeFromMode(info.Mode()) {
+	switch t := newFileTypeFromMode(info.Mode()); t {
 	case SymbolicLink:
-		linkTarget, err := os.Readlink(p)
-		if err != nil {
-			return "", fmt.Errorf("unable to readlink for path=%q: %w", p, err)
-		}
-		ref, err = r.fileTree.AddSymLink(file.Path(p), file.Path(linkTarget))
-		if err != nil {
-			return "", err
-		}
-
-		targetAbsPath := linkTarget
-		if !filepath.IsAbs(targetAbsPath) {
-			targetAbsPath = filepath.Clean(filepath.Join(path.Dir(p), linkTarget))
-		}
-
-		newRoot = targetAbsPath
-
+		return r.addSymlinkToIndex(p, info)
 	case Directory:
-		ref, err = r.fileTree.AddDir(file.Path(p))
-		if err != nil {
-			return "", err
-		}
-	case IrregularFile:
-		return "", ErrIgnoreIrregularFile
+		return "", r.addDirectoryToIndex(p, info)
+	case RegularFile:
+		return "", r.addFileToIndex(p, info)
 	default:
-		ref, err = r.fileTree.AddFile(file.Path(p))
-		if err != nil {
-			return "", err
-		}
+		return "", fmt.Errorf("unsupported file type: %s", t)
+	}
+}
+
+func (r directoryResolver) addDirectoryToIndex(p string, info os.FileInfo) error {
+	ref, err := r.fileTree.AddDir(file.Path(p))
+	if err != nil {
+		return err
 	}
 
-	metadata := fileMetadataFromPath(p, info)
+	location := NewLocationFromDirectory(p, *ref)
+	metadata := fileMetadataFromPath(p, info, r.isInIndex(location))
+	r.addFileMetadataToIndex(ref, metadata)
+
+	return nil
+}
+
+func (r directoryResolver) addFileToIndex(p string, info os.FileInfo) error {
+	ref, err := r.fileTree.AddFile(file.Path(p))
+	if err != nil {
+		return err
+	}
+
+	location := NewLocationFromDirectory(p, *ref)
+	metadata := fileMetadataFromPath(p, info, r.isInIndex(location))
+	r.addFileMetadataToIndex(ref, metadata)
+
+	return nil
+}
+
+func (r directoryResolver) addSymlinkToIndex(p string, info os.FileInfo) (string, error) {
+	var usedInfo = info
+
+	linkTarget, err := os.Readlink(p)
+	if err != nil {
+		return "", fmt.Errorf("unable to readlink for path=%q: %w", p, err)
+	}
+
+	// note: if the link is not absolute (e.g, /dev/stderr -> fd/2 ) we need to resolve it relative to the directory
+	// in question (e.g. resolve to /dev/fd/2)
+	if !filepath.IsAbs(linkTarget) {
+		linkTarget = filepath.Join(filepath.Dir(p), linkTarget)
+	}
+
+	ref, err := r.fileTree.AddSymLink(file.Path(p), file.Path(linkTarget))
+	if err != nil {
+		return "", err
+	}
+
+	targetAbsPath := linkTarget
+	if !filepath.IsAbs(targetAbsPath) {
+		targetAbsPath = filepath.Clean(filepath.Join(path.Dir(p), linkTarget))
+	}
+
+	location := NewLocationFromDirectory(p, *ref)
+	metadata := fileMetadataFromPath(p, usedInfo, r.isInIndex(location))
+	r.addFileMetadataToIndex(ref, metadata)
+
+	return targetAbsPath, nil
+}
+
+func (r directoryResolver) addFileMetadataToIndex(ref *file.Reference, metadata FileMetadata) {
 	if ref != nil {
-		r.refsByMIMEType[metadata.MIMEType] = append(r.refsByMIMEType[metadata.MIMEType], *ref)
+		if metadata.MIMEType != "" {
+			r.refsByMIMEType[metadata.MIMEType] = append(r.refsByMIMEType[metadata.MIMEType], *ref)
+		}
+		r.metadata[ref.ID()] = metadata
 	}
-
-	r.metadata[ref.ID()] = metadata
-	return newRoot, nil
 }
 
 func (r directoryResolver) requestPath(userPath string) (string, error) {
@@ -212,7 +246,7 @@ func (r directoryResolver) requestPath(userPath string) (string, error) {
 		userPath = path.Join(r.path, userPath)
 	} else {
 		// ensure we take into account any relative difference between the root path and the CWD for relative requests
-		userPath = path.Join(r.cwdRelativeToRoot, userPath)
+		userPath = path.Join(r.currentWdRelativeToRoot, userPath)
 	}
 
 	var err error
@@ -227,7 +261,7 @@ func (r directoryResolver) responsePath(path string) string {
 	// always return references relative to the request path (not absolute path)
 	if filepath.IsAbs(path) {
 		// we need to account for the cwd relative to the running process and the given root for the directory resolver
-		prefix := filepath.Clean(filepath.Join(r.cwd, r.cwdRelativeToRoot))
+		prefix := filepath.Clean(filepath.Join(r.currentWd, r.currentWdRelativeToRoot))
 		return strings.TrimPrefix(path, prefix+string(filepath.Separator))
 	}
 	return path
@@ -283,8 +317,6 @@ func (r directoryResolver) FilesByPath(userPaths ...string) ([]Location, error)
 		exists, ref, err := r.fileTree.File(file.Path(userStrPath))
 		if err == nil && exists {
 			references = append(references, NewLocationFromDirectory(r.responsePath(userStrPath), *ref))
-		} else {
-			log.Warnf("path (%s) not found in file tree: Exists: %t Err:%+v", userStrPath, exists, err)
 		}
 	}
 
@@ -329,9 +361,22 @@ func (r directoryResolver) FileContentsByLocation(location Location) (io.ReadClo
 	if location.ref.RealPath == "" {
 		return nil, errors.New("empty path given")
 	}
+	if !r.isInIndex(location) {
+		// this is in cases where paths have been explicitly excluded from the tree index. In which case
+		// we should DENY all content requests. Why? These paths have been indicated to be inaccessible (either
+		// by preference or these files are not readable by the current user).
+		return nil, fmt.Errorf("file content is inaccessible path=%q", location.ref.RealPath)
+	}
 	return file.NewLazyReadCloser(string(location.ref.RealPath)), nil
 }
 
+func (r directoryResolver) isInIndex(location Location) bool {
+	if location.ref.RealPath == "" {
+		return false
+	}
+	return r.fileTree.HasPath(location.ref.RealPath, filetree.FollowBasenameLinks)
+}
+
 func (r *directoryResolver) AllLocations() <-chan Location {
 	results := make(chan Location)
 	go func() {
@@ -364,10 +409,25 @@ func (r *directoryResolver) FilesByMIMEType(types ...string) ([]Location, error)
 	return locations, nil
 }
 
-func isUnixSystemRuntimePath(path string) bool {
+func isUnixSystemRuntimePath(path string, _ os.FileInfo) bool {
 	return internal.HasAnyOfPrefixes(path, unixSystemRuntimePrefixes...)
 }
 
+func isUnallowableFileType(_ string, info os.FileInfo) bool {
+	if info == nil {
+		// we can't filter out by filetype for non-existent files
+		return false
+	}
+	switch newFileTypeFromMode(info.Mode()) {
+	case CharacterDevice, Socket, BlockDevice, FIFONode, IrregularFile:
+		return true
+		// note: symlinks that point to these files may still get by. We handle this later in processing to help prevent
+		// against infinite links traversal.
+	}
+
+	return false
+}
+
 func indexAllRoots(root string, indexer func(string, *progress.Stage) ([]string, error)) error {
 	// why account for multiple roots? To cover cases when there is a symlink that references above the root path,
 	// in which case we need to additionally index where the link resolves to. it's for this reason why the filetree

syft/source/directory_resolver_test.go

@@ -1,14 +1,15 @@
 package source
 
 import (
+	"io/fs"
 	"io/ioutil"
 	"os"
 	"path"
 	"path/filepath"
 	"reflect"
 	"strings"
-	"syscall"
 	"testing"
+	"time"
 
 	"github.com/stretchr/testify/require"
 
@@ -19,6 +20,55 @@ import (
 	"github.com/wagoodman/go-progress"
 )
 
+func TestDirectoryResolver_FilesByPath_relativeRoot(t *testing.T) {
+	cases := []struct {
+		name         string
+		relativeRoot string
+		input        string
+		expected     []string
+	}{
+		{
+			name:         "should find a file from an absolute input",
+			relativeRoot: "./test-fixtures/",
+			input:        "/image-symlinks/file-1.txt",
+			expected: []string{
+				"image-symlinks/file-1.txt",
+			},
+		},
+		{
+			name:         "should find a file from a relative path",
+			relativeRoot: "./test-fixtures/",
+			input:        "image-symlinks/file-1.txt",
+			expected: []string{
+				"image-symlinks/file-1.txt",
+			},
+		},
+		{
+			name:         "should find a file from a relative path (root above cwd)",
+			relativeRoot: "../",
+			input:        "sbom/sbom.go",
+			expected: []string{
+				"sbom/sbom.go",
+			},
+		},
+	}
+	for _, c := range cases {
+		t.Run(c.name, func(t *testing.T) {
+			resolver, err := newDirectoryResolver(c.relativeRoot)
+			assert.NoError(t, err)
+
+			refs, err := resolver.FilesByPath(c.input)
+			require.NoError(t, err)
+			assert.Len(t, refs, len(c.expected))
+			s := strset.New()
+			for _, actual := range refs {
+				s.Add(actual.RealPath)
+			}
+			assert.ElementsMatch(t, c.expected, s.List())
+		})
+	}
+}
+
 func TestDirectoryResolver_FilesByPath_absoluteRoot(t *testing.T) {
 	cases := []struct {
 		name         string
@@ -85,23 +135,16 @@ func TestDirectoryResolver_FilesByPath(t *testing.T) {
 		{
 			name:     "finds a file (relative)",
 			root:     "./test-fixtures/",
-			input:    "test-fixtures/image-symlinks/file-1.txt",
-			expected: "test-fixtures/image-symlinks/file-1.txt",
+			input:    "image-symlinks/file-1.txt",
+			expected: "image-symlinks/file-1.txt",
 			refCount: 1,
 		},
 		{
 			name:     "finds a file with relative indirection",
 			root:     "./test-fixtures/../test-fixtures",
-			input:    "test-fixtures/image-symlinks/file-1.txt",
-			expected: "test-fixtures/image-symlinks/file-1.txt",
-			refCount: 1,
-		},
-		{
-			// note: this is asserting the old behavior is not supported
-			name:     "relative lookup with wrong path fails",
-			root:     "./test-fixtures/",
 			input:    "image-symlinks/file-1.txt",
-			refCount: 0,
+			expected: "image-symlinks/file-1.txt",
+			refCount: 1,
 		},
 		{
 			name:     "managed non-existing files (relative)",
@@ -113,7 +156,7 @@ func TestDirectoryResolver_FilesByPath(t *testing.T) {
 			name:     "finds a file (absolute)",
 			root:     "./test-fixtures/",
 			input:    "/image-symlinks/file-1.txt",
-			expected: "test-fixtures/image-symlinks/file-1.txt",
+			expected: "image-symlinks/file-1.txt",
 			refCount: 1,
 		},
 		{
@@ -158,17 +201,17 @@ func TestDirectoryResolver_MultipleFilesByPath(t *testing.T) {
 	}{
 		{
 			name:     "finds multiple files",
-			input:    []string{"test-fixtures/image-symlinks/file-1.txt", "test-fixtures/image-symlinks/file-2.txt"},
+			input:    []string{"image-symlinks/file-1.txt", "image-symlinks/file-2.txt"},
 			refCount: 2,
 		},
 		{
 			name:     "skips non-existing files",
-			input:    []string{"test-fixtures/image-symlinks/bogus.txt", "test-fixtures/image-symlinks/file-1.txt"},
+			input:    []string{"image-symlinks/bogus.txt", "image-symlinks/file-1.txt"},
 			refCount: 1,
 		},
 		{
 			name:     "does not return anything for non-existing directories",
-			input:    []string{"test-fixtures/non-existing/bogus.txt", "test-fixtures/non-existing/file-1.txt"},
+			input:    []string{"non-existing/bogus.txt", "non-existing/file-1.txt"},
 			refCount: 0,
 		},
 	}
@@ -210,7 +253,7 @@ func TestDirectoryResolver_FilesByGlobSingle(t *testing.T) {
 	assert.NoError(t, err)
 
 	assert.Len(t, refs, 1)
-	assert.Equal(t, "test-fixtures/image-symlinks/file-1.txt", refs[0].RealPath)
+	assert.Equal(t, "image-symlinks/file-1.txt", refs[0].RealPath)
 }
 
 func TestDirectoryResolverDoesNotIgnoreRelativeSystemPaths(t *testing.T) {
@@ -218,95 +261,128 @@ func TestDirectoryResolverDoesNotIgnoreRelativeSystemPaths(t *testing.T) {
 	resolver, err := newDirectoryResolver("test-fixtures/system_paths/target")
 	assert.NoError(t, err)
 	// ensure the correct filter function is wired up by default
-	expectedFn := reflect.ValueOf(isUnixSystemRuntimePath)
+	expectedFn := reflect.ValueOf(isUnallowableFileType)
 	actualFn := reflect.ValueOf(resolver.pathFilterFns[0])
 	assert.Equal(t, expectedFn.Pointer(), actualFn.Pointer())
 
 	// all paths should be found (non filtering matches a path)
-	refs, err := resolver.FilesByGlob("**/place")
+	locations, err := resolver.FilesByGlob("**/place")
 	assert.NoError(t, err)
 	// 4: within target/
 	// 1: target/link --> relative path to "place"
 	// 1: outside_root/link_target/place
-	assert.Len(t, refs, 6)
+	assert.Len(t, locations, 6)
 
 	// ensure that symlink indexing outside of root worked
+	testLocation := "test-fixtures/system_paths/outside_root/link_target/place"
 	ok := false
-	test_location := "test-fixtures/system_paths/outside_root/link_target/place"
-	for _, actual_loc := range refs {
-		if test_location == actual_loc.RealPath {
+	for _, location := range locations {
+		if strings.HasSuffix(location.RealPath, testLocation) {
 			ok = true
 		}
 	}
 
 	if !ok {
-		t.Fatalf("could not find test location=%q", test_location)
+		t.Fatalf("could not find test location=%q", testLocation)
 	}
 }
 
-func TestDirectoryResolverUsesPathFilterFunction(t *testing.T) {
-	// let's make certain that the index honors the filter function
-	filter := func(s string) bool {
-		// a dummy function that works for testing purposes
-		return strings.Contains(s, "dev/place") || strings.Contains(s, "proc/place") || strings.Contains(s, "sys/place")
-	}
+var _ fs.FileInfo = (*testFileInfo)(nil)
 
-	resolver, err := newDirectoryResolver("test-fixtures/system_paths/target", filter)
-	assert.NoError(t, err)
-
-	// ensure the correct filter function is wired up by default
-	expectedFn := reflect.ValueOf(filter)
-	actualFn := reflect.ValueOf(resolver.pathFilterFns[0])
-	assert.Equal(t, expectedFn.Pointer(), actualFn.Pointer())
-	assert.Len(t, resolver.pathFilterFns, 1)
-
-	refs, err := resolver.FilesByGlob("**/place")
-	assert.NoError(t, err)
-	// target/home/place + target/link/.../place + outside_root/.../place
-	assert.Len(t, refs, 3)
+type testFileInfo struct {
+	mode os.FileMode
 }
 
-func Test_isUnixSystemRuntimePath(t *testing.T) {
+func (t testFileInfo) Name() string {
+	panic("implement me")
+}
+
+func (t testFileInfo) Size() int64 {
+	panic("implement me")
+}
+
+func (t testFileInfo) Mode() fs.FileMode {
+	return t.mode
+}
+
+func (t testFileInfo) ModTime() time.Time {
+	panic("implement me")
+}
+
+func (t testFileInfo) IsDir() bool {
+	panic("implement me")
+}
+
+func (t testFileInfo) Sys() interface{} {
+	panic("implement me")
+}
+
+func Test_isUnallowableFileType(t *testing.T) {
 	tests := []struct {
-		path     string
+		name     string
+		info     os.FileInfo
 		expected bool
 	}{
 		{
-			path:     "proc/place",
+			name: "regular file",
+			info: testFileInfo{
+				mode: 0,
+			},
 			expected: false,
 		},
 		{
-			path:     "/proc/place",
-			expected: true,
-		},
-		{
-			path:     "/proc",
-			expected: true,
-		},
-		{
-			path:     "/pro/c",
+			name: "dir",
+			info: testFileInfo{
+				mode: os.ModeDir,
+			},
 			expected: false,
 		},
 		{
-			path:     "/pro",
+			name: "symlink",
+			info: testFileInfo{
+				mode: os.ModeSymlink,
+			},
 			expected: false,
 		},
 		{
-			path:     "/dev",
+			name: "socket",
+			info: testFileInfo{
+				mode: os.ModeSocket,
+			},
 			expected: true,
 		},
 		{
-			path:     "/sys",
+			name: "named pipe",
+			info: testFileInfo{
+				mode: os.ModeNamedPipe,
+			},
 			expected: true,
 		},
 		{
-			path:     "/something/sys",
-			expected: false,
+			name: "char device",
+			info: testFileInfo{
+				mode: os.ModeCharDevice,
+			},
+			expected: true,
+		},
+		{
+			name: "block device",
+			info: testFileInfo{
+				mode: os.ModeDevice,
+			},
+			expected: true,
+		},
+		{
+			name: "irregular",
+			info: testFileInfo{
+				mode: os.ModeIrregular,
+			},
+			expected: true,
 		},
 	}
 	for _, test := range tests {
-		t.Run(test.path, func(t *testing.T) {
-			assert.Equal(t, test.expected, isUnixSystemRuntimePath(test.path))
+		t.Run(test.name, func(t *testing.T) {
+			assert.Equal(t, test.expected, isUnallowableFileType("dont/care", test.info))
 		})
 	}
 }
@@ -345,9 +421,7 @@ func Test_directoryResolver_index(t *testing.T) {
 
 			// note: the index uses absolute paths, so assertions MUST keep this in mind
 			cwd, err := os.Getwd()
-			if err != nil {
-				t.Fatalf("could not get working dir: %+v", err)
-			}
+			require.NoError(t, err)
 
 			p := file.Path(path.Join(cwd, test.path))
 			assert.Equal(t, true, r.fileTree.HasPath(p))
@@ -365,32 +439,27 @@ func Test_handleFileAccessErr(t *testing.T) {
 	tests := []struct {
 		name                string
 		input               error
-		expectedErr         error
 		expectedPathTracked bool
 	}{
 		{
 			name:                "permission error does not propagate",
 			input:               os.ErrPermission,
 			expectedPathTracked: true,
-			expectedErr:         nil,
 		},
 		{
 			name:                "file does not exist error does not propagate",
 			input:               os.ErrNotExist,
 			expectedPathTracked: true,
-			expectedErr:         nil,
 		},
 		{
-			name:                "non-permission errors propagate",
+			name:                "non-permission errors are tracked",
 			input:               os.ErrInvalid,
-			expectedPathTracked: false,
-			expectedErr:         os.ErrInvalid,
+			expectedPathTracked: true,
 		},
 		{
 			name:                "non-errors ignored",
 			input:               nil,
 			expectedPathTracked: false,
-			expectedErr:         nil,
 		},
 	}
 
@@ -400,7 +469,7 @@ func Test_handleFileAccessErr(t *testing.T) {
 				errPaths: make(map[string]error),
 			}
 			p := "a/place"
-			assert.ErrorIs(t, r.handleFileAccessErr(p, test.input), test.expectedErr)
+			assert.Equal(t, r.isFileAccessErr(p, test.input), test.expectedPathTracked)
 			_, exists := r.errPaths[p]
 			assert.Equal(t, test.expectedPathTracked, exists)
 		})
@@ -493,7 +562,7 @@ func Test_directoryResolver_FilesByMIMEType(t *testing.T) {
 		{
 			fixturePath:   "./test-fixtures/image-simple",
 			mimeType:      "text/plain",
-			expectedPaths: strset.New("test-fixtures/image-simple/file-1.txt", "test-fixtures/image-simple/file-2.txt", "test-fixtures/image-simple/target/really/nested/file-3.txt", "test-fixtures/image-simple/Dockerfile"),
+			expectedPaths: strset.New("file-1.txt", "file-2.txt", "target/really/nested/file-3.txt", "Dockerfile"),
 		},
 	}
 	for _, test := range tests {
@@ -511,33 +580,81 @@ func Test_directoryResolver_FilesByMIMEType(t *testing.T) {
 	}
 }
 
-func Test_ignoreIrregularFiles(t *testing.T) {
-	// NOTE: craeting a pipe/fifo file on demand since git doesn't let me
-	// commit one. The directory resolver should ignore it (and any other
-	// file that is neither regular, a symlink or directory) when indexing a directory
-	dir := "./test-fixtures/irregular-files"
-	f := "f.fifo"
-
-	err := syscall.Mknod(filepath.Join(dir, f), syscall.S_IFIFO|0666, 0)
-	assert.NoError(t, err)
-	defer func() {
-		err := os.Remove(filepath.Join(dir, f))
-		assert.NoError(t, err)
-	}()
-
-	fileRefs, err := ioutil.ReadDir(dir)
-	assert.NoError(t, err)
-	assert.Len(t, fileRefs, 2) // two files (f.fifo and readme)
-
-	resolver, err := newDirectoryResolver(dir)
+func Test_IndexingNestedSymLinks(t *testing.T) {
+	resolver, err := newDirectoryResolver("./test-fixtures/symlinks-simple")
 	assert.NoError(t, err)
 
-	assert.Len(t, resolver.fileTree.AllFiles(), 1)
-	rp := resolver.fileTree.AllFiles()[0].RealPath
-	assert.True(t, strings.Contains(string(rp), filepath.Join(dir, "readme")))
+	// check that we can get the real path
+	locations, err := resolver.FilesByPath("./readme")
+	require.NoError(t, err)
+	assert.Len(t, locations, 1)
+
+	// check that we can access the same file via 1 symlink
+	locations, err = resolver.FilesByPath("./link_to_new_readme")
+	require.NoError(t, err)
+	assert.Len(t, locations, 1)
+
+	// check that we can access the same file via 2 symlinks
+	locations, err = resolver.FilesByPath("./link_to_link_to_new_readme")
+	require.NoError(t, err)
+	assert.Len(t, locations, 1)
+}
+
+func Test_IndexingNestedSymLinks_ignoredIndexes(t *testing.T) {
+	filterFn := func(path string, _ os.FileInfo) bool {
+		return strings.HasSuffix(path, string(filepath.Separator)+"readme")
+	}
+
+	resolver, err := newDirectoryResolver("./test-fixtures/symlinks-simple", filterFn)
+	assert.NoError(t, err)
+
+	var testingLocations []Location
+
+	// the path to the real file is PRUNED from the index, so we should NOT expect a location returned
+	locations, err := resolver.FilesByPath("./readme")
+	require.NoError(t, err)
+	assert.Empty(t, locations)
+
+	// check that we can access the same file via 1 symlink
+	locations, err = resolver.FilesByPath("./link_to_new_readme")
+	require.NoError(t, err)
+	assert.Len(t, locations, 1)
+	testingLocations = append(testingLocations, locations...)
+
+	// check that we can access the same file via 2 symlinks
+	locations, err = resolver.FilesByPath("./link_to_link_to_new_readme")
+	require.NoError(t, err)
+	assert.Len(t, locations, 1)
+	testingLocations = append(testingLocations, locations...)
+
+	// check that we CANNOT get contents from any of the link locations
+	for _, location := range testingLocations {
+		contentReader, err := resolver.FileContentsByLocation(location)
+		assert.Errorf(t, err, "expected an error for getting content from a location not in the index")
+		assert.Nil(t, contentReader)
+	}
+}
+
+func Test_IndexingNestedSymLinksOutsideOfRoot(t *testing.T) {
+	resolver, err := newDirectoryResolver("./test-fixtures/symlinks-roots/root")
+	assert.NoError(t, err)
+
+	// check that we can get the real path
+	locations, err := resolver.FilesByPath("./readme")
+	require.NoError(t, err)
+	assert.Len(t, locations, 1)
+
+	// check that we can access the same file via 2 symlinks (link_to_link_to_readme -> link_to_readme -> readme)
+	locations, err = resolver.FilesByPath("./link_to_link_to_readme")
+	require.NoError(t, err)
+	assert.Len(t, locations, 1)
+
 }
 
 func Test_directoryResolver_FileContentsByLocation(t *testing.T) {
+	cwd, err := os.Getwd()
+	require.NoError(t, err)
+
 	tests := []struct {
 		name     string
 		location Location
@@ -547,7 +664,7 @@ func Test_directoryResolver_FileContentsByLocation(t *testing.T) {
 		{
 			name: "use file reference for content requests",
 			location: NewLocationFromDirectory("some/place", file.Reference{
-				RealPath: "test-fixtures/image-simple/file-1.txt",
+				RealPath: file.Path(filepath.Join(cwd, "test-fixtures/image-simple/file-1.txt")),
 			}),
 			expects: "this file has contents",
 		},
@@ -578,3 +695,79 @@ func Test_directoryResolver_FileContentsByLocation(t *testing.T) {
 		})
 	}
 }
+
+func Test_isUnixSystemRuntimePath(t *testing.T) {
+	tests := []struct {
+		path     string
+		expected bool
+	}{
+		{
+			path:     "proc/place",
+			expected: false,
+		},
+		{
+			path:     "/proc/place",
+			expected: true,
+		},
+		{
+			path:     "/proc",
+			expected: true,
+		},
+		{
+			path:     "/pro/c",
+			expected: false,
+		},
+		{
+			path:     "/pro",
+			expected: false,
+		},
+		{
+			path:     "/dev",
+			expected: true,
+		},
+		{
+			path:     "/sys",
+			expected: true,
+		},
+		{
+			path:     "/something/sys",
+			expected: false,
+		},
+	}
+	for _, test := range tests {
+		t.Run(test.path, func(t *testing.T) {
+			assert.Equal(t, test.expected, isUnixSystemRuntimePath(test.path, nil))
+		})
+	}
+}
+
+func Test_SymlinkLoopWithGlobsShouldResolve(t *testing.T) {
+	test := func(t *testing.T) {
+		resolver, err := newDirectoryResolver("./test-fixtures/symlinks-loop")
+		require.NoError(t, err)
+
+		locations, err := resolver.FilesByGlob("**/file.target")
+		require.NoError(t, err)
+		// Note: I'm not certain that this behavior is correct, but it is not an infinite loop (which is the point of the test)
+		// - block/loop0/file.target
+		// - devices/loop0/file.target
+		// - devices/loop0/subsystem/loop0/file.target
+		assert.Len(t, locations, 3)
+	}
+
+	testWithTimeout(t, 5*time.Second, test)
+}
+
+func testWithTimeout(t *testing.T, timeout time.Duration, test func(*testing.T)) {
+	done := make(chan bool)
+	go func() {
+		test(t)
+		done <- true
+	}()
+
+	select {
+	case <-time.After(timeout):
+		t.Fatal("test timed out")
+	case <-done:
+	}
+}

syft/source/excluding_file_resolver.go

@@ -0,0 +1,101 @@
+package source
+
+import (
+	"fmt"
+	"io"
+)
+
+type excludeFn func(string) bool
+
+// excludingResolver decorates a resolver with an exclusion function that is used to
+// filter out entries in the delegate resolver
+type excludingResolver struct {
+	delegate  FileResolver
+	excludeFn excludeFn
+}
+
+// NewExcludingResolver create a new resolver which wraps the provided delegate and excludes
+// entries based on a provided path exclusion function
+func NewExcludingResolver(delegate FileResolver, excludeFn excludeFn) FileResolver {
+	return &excludingResolver{
+		delegate,
+		excludeFn,
+	}
+}
+
+func (r *excludingResolver) FileContentsByLocation(location Location) (io.ReadCloser, error) {
+	if locationMatches(&location, r.excludeFn) {
+		return nil, fmt.Errorf("no such location: %+v", location.RealPath)
+	}
+	return r.delegate.FileContentsByLocation(location)
+}
+
+func (r *excludingResolver) FileMetadataByLocation(location Location) (FileMetadata, error) {
+	if locationMatches(&location, r.excludeFn) {
+		return FileMetadata{}, fmt.Errorf("no such location: %+v", location.RealPath)
+	}
+	return r.delegate.FileMetadataByLocation(location)
+}
+
+func (r *excludingResolver) HasPath(path string) bool {
+	if r.excludeFn(path) {
+		return false
+	}
+	return r.delegate.HasPath(path)
+}
+
+func (r *excludingResolver) FilesByPath(paths ...string) ([]Location, error) {
+	locations, err := r.delegate.FilesByPath(paths...)
+	return filterLocations(locations, err, r.excludeFn)
+}
+
+func (r *excludingResolver) FilesByGlob(patterns ...string) ([]Location, error) {
+	locations, err := r.delegate.FilesByGlob(patterns...)
+	return filterLocations(locations, err, r.excludeFn)
+}
+
+func (r *excludingResolver) FilesByMIMEType(types ...string) ([]Location, error) {
+	locations, err := r.delegate.FilesByMIMEType(types...)
+	return filterLocations(locations, err, r.excludeFn)
+}
+
+func (r *excludingResolver) RelativeFileByPath(location Location, path string) *Location {
+	l := r.delegate.RelativeFileByPath(location, path)
+	if l != nil && locationMatches(l, r.excludeFn) {
+		return nil
+	}
+	return l
+}
+
+func (r *excludingResolver) AllLocations() <-chan Location {
+	c := make(chan Location)
+	go func() {
+		defer close(c)
+		for location := range r.delegate.AllLocations() {
+			if !locationMatches(&location, r.excludeFn) {
+				c <- location
+			}
+		}
+	}()
+	return c
+}
+
+func locationMatches(location *Location, exclusionFn excludeFn) bool {
+	return exclusionFn(location.RealPath) || exclusionFn(location.VirtualPath)
+}
+
+func filterLocations(locations []Location, err error, exclusionFn excludeFn) ([]Location, error) {
+	if err != nil {
+		return nil, err
+	}
+	if exclusionFn != nil {
+		for i := 0; i < len(locations); i++ {
+			location := &locations[i]
+			if locationMatches(location, exclusionFn) {
+				locations = append(locations[:i], locations[i+1:]...)
+				i--
+			}
+		}
+	}
+	return locations, nil
+}

syft/source/excluding_file_resolver_test.go

@@ -0,0 +1,197 @@
+package source
+
+import (
+	"io"
+	"strings"
+	"testing"
+
+	"github.com/anchore/stereoscope/pkg/file"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestExcludingResolver(t *testing.T) {
+
+	tests := []struct {
+		name      string
+		locations []string
+		excludeFn excludeFn
+		expected  []string
+	}{
+		{
+			name:      "keeps locations",
+			locations: []string{"a", "b", "c"},
+			excludeFn: func(s string) bool {
+				return false
+			},
+			expected: []string{"a", "b", "c"},
+		},
+		{
+			name:      "removes locations",
+			locations: []string{"d", "e", "f"},
+			excludeFn: func(s string) bool {
+				return true
+			},
+			expected: []string{},
+		},
+		{
+			name:      "removes first match",
+			locations: []string{"g", "h", "i"},
+			excludeFn: func(s string) bool {
+				return s == "g"
+			},
+			expected: []string{"h", "i"},
+		},
+		{
+			name:      "removes last match",
+			locations: []string{"j", "k", "l"},
+			excludeFn: func(s string) bool {
+				return s == "l"
+			},
+			expected: []string{"j", "k"},
+		},
+	}
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			resolver := &mockResolver{
+				locations: test.locations,
+			}
+			excludingResolver := NewExcludingResolver(resolver, test.excludeFn)
+
+			locations, _ := excludingResolver.FilesByPath()
+			assert.ElementsMatch(t, locationPaths(locations), test.expected)
+
+			locations, _ = excludingResolver.FilesByGlob()
+			assert.ElementsMatch(t, locationPaths(locations), test.expected)
+
+			locations, _ = excludingResolver.FilesByMIMEType()
+			assert.ElementsMatch(t, locationPaths(locations), test.expected)
+
+			locations = []Location{}
+
+			channel := excludingResolver.AllLocations()
+			for location := range channel {
+				locations = append(locations, location)
+			}
+			assert.ElementsMatch(t, locationPaths(locations), test.expected)
+
+			diff := difference(test.locations, test.expected)
+
+			for _, path := range diff {
+				assert.False(t, excludingResolver.HasPath(path))
+				c, err := excludingResolver.FileContentsByLocation(makeLocation(path))
+				assert.Nil(t, c)
+				assert.Error(t, err)
+				m, err := excludingResolver.FileMetadataByLocation(makeLocation(path))
+				assert.Empty(t, m.LinkDestination)
+				assert.Error(t, err)
+				l := excludingResolver.RelativeFileByPath(makeLocation(""), path)
+				assert.Nil(t, l)
+			}
+
+			for _, path := range test.expected {
+				assert.True(t, excludingResolver.HasPath(path))
+				c, err := excludingResolver.FileContentsByLocation(makeLocation(path))
+				assert.NotNil(t, c)
+				assert.Nil(t, err)
+				m, err := excludingResolver.FileMetadataByLocation(makeLocation(path))
+				assert.NotEmpty(t, m.LinkDestination)
+				assert.Nil(t, err)
+				l := excludingResolver.RelativeFileByPath(makeLocation(""), path)
+				assert.NotNil(t, l)
+			}
+		})
+	}
+}
+
+// difference returns the elements in `a` that aren't in `b`.
+func difference(a, b []string) []string {
+	mb := make(map[string]struct{}, len(b))
+	for _, x := range b {
+		mb[x] = struct{}{}
+	}
+	var diff []string
+	for _, x := range a {
+		if _, found := mb[x]; !found {
+			diff = append(diff, x)
+		}
+	}
+	return diff
+}
+
+func makeLocation(path string) Location {
+	return Location{
+		Coordinates: Coordinates{
+			RealPath:     path,
+			FileSystemID: "",
+		},
+		VirtualPath: "",
+		ref:         file.Reference{},
+	}
+}
+
+func locationPaths(locations []Location) []string {
+	paths := []string{}
+	for _, l := range locations {
+		paths = append(paths, l.RealPath)
+	}
+	return paths
+}
+
+type mockResolver struct {
+	locations []string
+}
+
+func (r *mockResolver) getLocations() ([]Location, error) {
+	out := []Location{}
+	for _, path := range r.locations {
+		out = append(out, makeLocation(path))
+	}
+	return out, nil
+}
+
+func (r *mockResolver) FileContentsByLocation(_ Location) (io.ReadCloser, error) {
+	return io.NopCloser(strings.NewReader("Hello, world!")), nil
+}
+
+func (r *mockResolver) FileMetadataByLocation(_ Location) (FileMetadata, error) {
+	return FileMetadata{
+		LinkDestination: "MOCK",
+	}, nil
+}
+
+func (r *mockResolver) HasPath(_ string) bool {
+	return true
+}
+
+func (r *mockResolver) FilesByPath(_ ...string) ([]Location, error) {
+	return r.getLocations()
+}
+
+func (r *mockResolver) FilesByGlob(_ ...string) ([]Location, error) {
+	return r.getLocations()
+}
+
+func (r *mockResolver) FilesByMIMEType(_ ...string) ([]Location, error) {
+	return r.getLocations()
+}
+
+func (r *mockResolver) RelativeFileByPath(_ Location, path string) *Location {
+	return &Location{
+		Coordinates: Coordinates{
+			RealPath: path,
+		},
+	}
+}
+
+func (r *mockResolver) AllLocations() <-chan Location {
+	c := make(chan Location)
+	go func() {
+		defer close(c)
+		locations, _ := r.getLocations()
+		for _, location := range locations {
+			c <- location
+		}
+	}()
+	return c
+}

syft/source/file_metadata.go

@@ -3,10 +3,10 @@ package source
 import (
 	"os"
 
-	"github.com/anchore/syft/internal/log"
-
 	"github.com/anchore/stereoscope/pkg/file"
 
+	"github.com/anchore/syft/internal/log"
+
 	"github.com/anchore/stereoscope/pkg/image"
 )
 
@@ -37,19 +37,24 @@ func fileMetadataByLocation(img *image.Image, location Location) (FileMetadata,
 	}, nil
 }
 
-func fileMetadataFromPath(path string, info os.FileInfo) FileMetadata {
+func fileMetadataFromPath(path string, info os.FileInfo, withMIMEType bool) FileMetadata {
+	var mimeType string
 	uid, gid := GetXid(info)
 
-	f, err := os.Open(path)
-	if err != nil {
-		// TODO: it may be that the file is inaccessible, however, this is not an error or a warning. In the future we need to track these as known-unknowns
-		f = nil
-	} else {
-		defer func() {
-			if err := f.Close(); err != nil {
-				log.Warnf("unable to close file while obtaining metadata: %s", path)
-			}
-		}()
+	if withMIMEType {
+		f, err := os.Open(path)
+		if err != nil {
+			// TODO: it may be that the file is inaccessible, however, this is not an error or a warning. In the future we need to track these as known-unknowns
+			f = nil
+		} else {
+			defer func() {
+				if err := f.Close(); err != nil {
+					log.Warnf("unable to close file while obtaining metadata: %s", path)
+				}
+			}()
+		}
+
+		mimeType = file.MIMEType(f)
 	}
 
 	return FileMetadata{
@@ -58,6 +63,6 @@ func fileMetadataFromPath(path string, info os.FileInfo) FileMetadata {
 		// unsupported across platforms
 		UserID:   uid,
 		GroupID:  gid,
-		MIMEType: file.MIMEType(f),
+		MIMEType: mimeType,
 	}
 }

syft/source/file_metadata_test.go

@@ -0,0 +1,54 @@
+package source
+
+import (
+	"os"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func Test_fileMetadataFromPath(t *testing.T) {
+
+	tests := []struct {
+		path             string
+		withMIMEType     bool
+		expectedType     string
+		expectedMIMEType string
+	}{
+		{
+			path:             "test-fixtures/symlinks-simple/readme",
+			withMIMEType:     true,
+			expectedType:     "RegularFile",
+			expectedMIMEType: "text/plain",
+		},
+		{
+			path:             "test-fixtures/symlinks-simple/link_to_new_readme",
+			withMIMEType:     true,
+			expectedType:     "SymbolicLink",
+			expectedMIMEType: "text/plain",
+		},
+		{
+			path:             "test-fixtures/symlinks-simple/readme",
+			withMIMEType:     false,
+			expectedType:     "RegularFile",
+			expectedMIMEType: "",
+		},
+		{
+			path:             "test-fixtures/symlinks-simple/link_to_new_readme",
+			withMIMEType:     false,
+			expectedType:     "SymbolicLink",
+			expectedMIMEType: "",
+		},
+	}
+	for _, test := range tests {
+		t.Run(test.path, func(t *testing.T) {
+			info, err := os.Lstat(test.path)
+			require.NoError(t, err)
+
+			actual := fileMetadataFromPath(test.path, info, test.withMIMEType)
+			assert.Equal(t, test.expectedMIMEType, actual.MIMEType)
+			assert.Equal(t, test.expectedType, string(actual.Type))
+		})
+	}
+}

syft/source/file_type.go

@@ -6,8 +6,7 @@ import (
 )
 
 const (
-	UnknownFileType FileType = "UnknownFileType"
-	RegularFile     FileType = "RegularFile"
+	RegularFile FileType = "RegularFile"
 	// IrregularFile is how syft defines files that are neither regular, symbolic or directory.
 	// For ref: the seven standard Unix file types are regular, directory, symbolic link,
 	// FIFO special, block special, character special, and socket as defined by POSIX.
@@ -18,6 +17,7 @@ const (
 	BlockDevice     FileType = "BlockDevice"
 	Directory       FileType = "Directory"
 	FIFONode        FileType = "FIFONode"
+	Socket          FileType = "Socket"
 )
 
 type FileType string
@@ -39,19 +39,32 @@ func newFileTypeFromTarHeaderTypeFlag(flag byte) FileType {
 	case tar.TypeFifo:
 		return FIFONode
 	}
-	return UnknownFileType
+	return IrregularFile
 }
 
-// TODO: fill in more types from mod...
 func newFileTypeFromMode(mode os.FileMode) FileType {
 	switch {
-	case mode&os.ModeSymlink == os.ModeSymlink:
+	case isSet(mode, os.ModeSymlink):
 		return SymbolicLink
+	case isSet(mode, os.ModeIrregular):
+		return IrregularFile
+	case isSet(mode, os.ModeCharDevice):
+		return CharacterDevice
+	case isSet(mode, os.ModeDevice):
+		return BlockDevice
+	case isSet(mode, os.ModeNamedPipe):
+		return FIFONode
+	case isSet(mode, os.ModeSocket):
+		return Socket
 	case mode.IsDir():
 		return Directory
-	case !mode.IsRegular():
-		return IrregularFile
-	default:
+	case mode.IsRegular():
 		return RegularFile
+	default:
+		return IrregularFile
 	}
 }
+
+func isSet(mode, field os.FileMode) bool {
+	return mode&field != 0
+}

syft/source/location.go

@@ -3,21 +3,19 @@ package source
 import (
 	"fmt"
 
-	"github.com/mitchellh/hashstructure/v2"
-
 	"github.com/anchore/stereoscope/pkg/file"
 	"github.com/anchore/stereoscope/pkg/image"
 	"github.com/anchore/syft/internal/log"
 )
 
-var _ hashstructure.Hashable = (*Location)(nil)
-
 // Location represents a path relative to a particular filesystem resolved to a specific file.Reference. This struct is used as a key
 // in content fetching to uniquely identify a file relative to a request (the VirtualPath).
 type Location struct {
 	Coordinates
-	VirtualPath string         // The path to the file which may or may not have hardlinks / symlinks
-	ref         file.Reference // The file reference relative to the stereoscope.FileCatalog that has more information about this location.
+	// note: it is IMPORTANT to ignore anything but the coordinates for a Location when considering the ID (hash value)
+	// since the coordinates are the minimally correct ID for a location (symlinks should not come into play)
+	VirtualPath string         `hash:"ignore"` // The path to the file which may or may not have hardlinks / symlinks
+	ref         file.Reference `hash:"ignore"` // The file reference relative to the stereoscope.FileCatalog that has more information about this location.
 }
 
 // NewLocation creates a new Location representing a path without denoting a filesystem or FileCatalog reference.
@@ -97,9 +95,3 @@ func (l Location) String() string {
 	}
 	return fmt.Sprintf("Location<%s>", str)
 }
-
-func (l Location) Hash() (uint64, error) {
-	// since location is part of the package definition it is important that only coordinates are used during object
-	// hashing. (Location hash should be a pass-through for the coordinates and not include ref or VirtualPath.)
-	return hashstructure.Hash(l.ID(), hashstructure.FormatV2, nil)
-}

syft/source/location_test.go

@@ -0,0 +1,47 @@
+package source
+
+import (
+	"testing"
+
+	"github.com/anchore/stereoscope/pkg/file"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestLocation_ID(t *testing.T) {
+	tests := []struct {
+		name        string
+		coordinates Coordinates
+		virtualPath string
+		ref         file.Reference
+	}{
+		{
+			name: "coordinates should match location hash",
+			coordinates: Coordinates{
+				RealPath:     "path!",
+				FileSystemID: "filesystem!",
+			},
+		},
+		{
+			name: "coordinates should match location hash (with extra fields)",
+			coordinates: Coordinates{
+				RealPath:     "path!",
+				FileSystemID: "filesystem!",
+			},
+			virtualPath: "virtualPath!",
+			ref: file.Reference{
+				RealPath: "other-real-path!",
+			},
+		},
+	}
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			l := Location{
+				Coordinates: test.coordinates,
+				VirtualPath: test.virtualPath,
+				ref:         test.ref,
+			}
+			assert.Equal(t, l.ID(), test.coordinates.ID())
+		})
+	}
+
+}

syft/source/source.go

@@ -9,11 +9,14 @@ import (
 	"fmt"
 	"io/ioutil"
 	"os"
+	"path/filepath"
+	"strings"
 	"sync"
 
 	"github.com/anchore/stereoscope"
 	"github.com/anchore/stereoscope/pkg/image"
 	"github.com/anchore/syft/internal/log"
+	"github.com/bmatcuk/doublestar/v2"
 	"github.com/mholt/archiver/v3"
 	"github.com/spf13/afero"
 )
@@ -26,41 +29,42 @@ type Source struct {
 	directoryResolver *directoryResolver
 	path              string
 	mutex             *sync.Mutex
+	Exclusions        []string
 }
 
 type sourceDetector func(string) (image.Source, string, error)
 
 // New produces a Source based on userInput like dir: or image:tag
-func New(userInput string, registryOptions *image.RegistryOptions) (*Source, func(), error) {
+func New(userInput string, registryOptions *image.RegistryOptions, exclusions []string) (*Source, func(), error) {
 	fs := afero.NewOsFs()
 	parsedScheme, imageSource, location, err := detectScheme(fs, image.DetectSource, userInput)
 	if err != nil {
 		return &Source{}, func() {}, fmt.Errorf("unable to parse input=%q: %w", userInput, err)
 	}
 
+	source := &Source{}
+	cleanupFn := func() {}
+
 	switch parsedScheme {
 	case FileScheme:
-		return generateFileSource(fs, location)
+		source, cleanupFn, err = generateFileSource(fs, location)
 	case DirectoryScheme:
-		return generateDirectorySource(fs, location)
+		source, cleanupFn, err = generateDirectorySource(fs, location)
 	case ImageScheme:
-		return generateImageSource(location, userInput, imageSource, registryOptions)
+		source, cleanupFn, err = generateImageSource(userInput, location, imageSource, registryOptions)
+	default:
+		err = fmt.Errorf("unable to process input for scanning: '%s'", userInput)
 	}
 
-	return &Source{}, func() {}, fmt.Errorf("unable to process input for scanning: '%s'", userInput)
+	if err == nil {
+		source.Exclusions = exclusions
+	}
+
+	return source, cleanupFn, err
 }
 
-func generateImageSource(location, userInput string, imageSource image.Source, registryOptions *image.RegistryOptions) (*Source, func(), error) {
-	img, err := stereoscope.GetImageFromSource(location, imageSource, registryOptions)
-	if err != nil {
-		log.Debugf("error parsing location: %s after detecting scheme; pulling image: %s", location, userInput)
-		// we may have been to aggressive reading the source hint
-		// try the input as supplied by the user if our initial parse failed
-		img, err = stereoscope.GetImageFromSource(userInput, imageSource, registryOptions)
-	}
-
-	cleanup := stereoscope.Cleanup
-
+func generateImageSource(userInput, location string, imageSource image.Source, registryOptions *image.RegistryOptions) (*Source, func(), error) {
+	img, cleanup, err := getImageWithRetryStrategy(userInput, location, imageSource, registryOptions)
 	if err != nil || img == nil {
 		return &Source{}, cleanup, fmt.Errorf("could not fetch image '%s': %w", location, err)
 	}
@@ -73,6 +77,54 @@ func generateImageSource(location, userInput string, imageSource image.Source, r
 	return &s, cleanup, nil
 }
 
+func parseScheme(userInput string) string {
+	parts := strings.SplitN(userInput, ":", 2)
+	if len(parts) < 2 {
+		return ""
+	}
+
+	return parts[0]
+}
+
+func getImageWithRetryStrategy(userInput, location string, imageSource image.Source, registryOptions *image.RegistryOptions) (*image.Image, func(), error) {
+	img, err := stereoscope.GetImageFromSource(location, imageSource, registryOptions)
+	if err == nil {
+		// Success on the first try!
+		return img, stereoscope.Cleanup, nil
+	}
+
+	scheme := parseScheme(userInput)
+	if !(scheme == "docker" || scheme == "registry") {
+		// Image retrieval failed, and we shouldn't retry it. It's most likely that the
+		// user _did_ intend the parsed scheme, but there was a legitimate failure with
+		// using the scheme to load the image. Alert the user to this failure, so they
+		// can fix the problem.
+		return nil, nil, err
+	}
+
+	// Maybe the user wanted "docker" or "registry" to refer to an _image name_
+	// (e.g. "docker:latest"), not a scheme. We'll retry image retrieval with this
+	// alternative interpretation, in an attempt to avoid unnecessary user friction.
+
+	log.Warnf(
+		"scheme %q specified, but it coincides with a common image name; re-examining user input %q"+
+			" without scheme parsing because image retrieval using scheme parsing was unsuccessful: %v",
+		scheme,
+		userInput,
+		err,
+	)
+
+	// We need to determine the image source again, such that this determination
+	// doesn't take scheme parsing into account.
+	imageSource = image.DetermineImagePullSource(userInput)
+	img, err = stereoscope.GetImageFromSource(userInput, imageSource, registryOptions)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	return img, stereoscope.Cleanup, nil
+}
+
 func generateDirectorySource(fs afero.Fs, location string) (*Source, func(), error) {
 	fileMeta, err := fs.Stat(location)
 	if err != nil {
@@ -180,7 +232,11 @@ func (s *Source) FileResolver(scope Scope) (FileResolver, error) {
 		s.mutex.Lock()
 		defer s.mutex.Unlock()
 		if s.directoryResolver == nil {
-			resolver, err := newDirectoryResolver(s.path)
+			exclusionFunctions, err := getDirectoryExclusionFunctions(s.path, s.Exclusions)
+			if err != nil {
+				return nil, err
+			}
+			resolver, err := newDirectoryResolver(s.path, exclusionFunctions...)
 			if err != nil {
 				return nil, err
 			}
@@ -188,14 +244,24 @@ func (s *Source) FileResolver(scope Scope) (FileResolver, error) {
 		}
 		return s.directoryResolver, nil
 	case ImageScheme:
+		var resolver FileResolver
+		var err error
 		switch scope {
 		case SquashedScope:
-			return newImageSquashResolver(s.Image)
+			resolver, err = newImageSquashResolver(s.Image)
 		case AllLayersScope:
-			return newAllLayersResolver(s.Image)
+			resolver, err = newAllLayersResolver(s.Image)
 		default:
 			return nil, fmt.Errorf("bad image scope provided: %+v", scope)
 		}
+		if err != nil {
+			return nil, err
+		}
+		// image tree contains all paths, so we filter out the excluded entries afterwards
+		if len(s.Exclusions) > 0 {
+			resolver = NewExcludingResolver(resolver, getImageExclusionFunction(s.Exclusions))
+		}
+		return resolver, nil
 	}
 	return nil, fmt.Errorf("unable to determine FilePathResolver with current scheme=%q", s.Metadata.Scheme)
 }
@@ -214,3 +280,71 @@ func unarchiveToTmp(path string, unarchiver archiver.Unarchiver) (string, func()
 
 	return tempDir, cleanupFn, unarchiver.Unarchive(path, tempDir)
 }
+
+func getImageExclusionFunction(exclusions []string) func(string) bool {
+	if len(exclusions) == 0 {
+		return nil
+	}
+	// add subpath exclusions
+	for _, exclusion := range exclusions {
+		exclusions = append(exclusions, exclusion+"/**")
+	}
+	return func(path string) bool {
+		for _, exclusion := range exclusions {
+			matches, err := doublestar.Match(exclusion, path)
+			if err != nil {
+				return false
+			}
+			if matches {
+				return true
+			}
+		}
+		return false
+	}
+}
+
+func getDirectoryExclusionFunctions(root string, exclusions []string) ([]pathFilterFn, error) {
+	if len(exclusions) == 0 {
+		return nil, nil
+	}
+
+	// this is what directoryResolver.indexTree is doing to get the absolute path:
+	root, err := filepath.Abs(root)
+	if err != nil {
+		return nil, err
+	}
+
+	if !strings.HasSuffix(root, "/") {
+		root += "/"
+	}
+
+	var errors []string
+	for idx, exclusion := range exclusions {
+		// check exclusions for supported paths, these are all relative to the "scan root"
+		if strings.HasPrefix(exclusion, "./") || strings.HasPrefix(exclusion, "*/") || strings.HasPrefix(exclusion, "**/") {
+			exclusion = strings.TrimPrefix(exclusion, "./")
+			exclusions[idx] = root + exclusion
+		} else {
+			errors = append(errors, exclusion)
+		}
+	}
+
+	if errors != nil {
+		return nil, fmt.Errorf("invalid exclusion pattern(s): '%s' (must start with one of: './', '*/', or '**/')", strings.Join(errors, "', '"))
+	}
+
+	return []pathFilterFn{
+		func(path string, _ os.FileInfo) bool {
+			for _, exclusion := range exclusions {
+				matches, err := doublestar.Match(exclusion, path)
+				if err != nil {
+					return false
+				}
+				if matches {
+					return true
+				}
+			}
+			return false
+		},
+	}, nil
+}

syft/source/source_test.go

@@ -6,9 +6,12 @@ import (
 	"os/exec"
 	"path"
 	"path/filepath"
+	"strings"
 	"syscall"
 	"testing"
 
+	"github.com/anchore/stereoscope/pkg/imagetest"
+
 	"github.com/stretchr/testify/require"
 
 	"github.com/stretchr/testify/assert"
@@ -55,19 +58,19 @@ func TestNewFromDirectory(t *testing.T) {
 		{
 			desc:       "path detected",
 			input:      "test-fixtures",
-			inputPaths: []string{"test-fixtures/path-detected/.vimrc"},
+			inputPaths: []string{"path-detected/.vimrc"},
 			expRefs:    1,
 		},
 		{
 			desc:       "directory ignored",
 			input:      "test-fixtures",
-			inputPaths: []string{"test-fixtures/path-detected"},
+			inputPaths: []string{"path-detected"},
 			expRefs:    0,
 		},
 		{
 			desc:       "no files-by-path detected",
 			input:      "test-fixtures",
-			inputPaths: []string{"test-fixtures/no-path-detected"},
+			inputPaths: []string{"no-path-detected"},
 			expRefs:    0,
 		},
 	}
@@ -184,21 +187,21 @@ func TestNewFromDirectoryShared(t *testing.T) {
 			desc:       "path detected",
 			input:      "test-fixtures",
 			notExist:   "foobar/",
-			inputPaths: []string{"test-fixtures/path-detected/.vimrc"},
+			inputPaths: []string{"path-detected/.vimrc"},
 			expRefs:    1,
 		},
 		{
 			desc:       "directory ignored",
 			input:      "test-fixtures",
 			notExist:   "foobar/",
-			inputPaths: []string{"test-fixtures/path-detected"},
+			inputPaths: []string{"path-detected"},
 			expRefs:    0,
 		},
 		{
 			desc:       "no files-by-path detected",
 			input:      "test-fixtures",
 			notExist:   "foobar/",
-			inputPaths: []string{"test-fixtures/no-path-detected"},
+			inputPaths: []string{"no-path-detected"},
 			expRefs:    0,
 		},
 	}
@@ -306,7 +309,9 @@ func TestFilesByGlob(t *testing.T) {
 				t.Errorf("could not get resolver error: %+v", err)
 			}
 			contents, err := resolver.FilesByGlob(test.glob)
-
+			if err != nil {
+				t.Errorf("could not get files by glob: %s+v", err)
+			}
 			if len(contents) != test.expected {
 				t.Errorf("unexpected number of files found by glob (%s): %d != %d", test.glob, len(contents), test.expected)
 			}
@@ -315,6 +320,219 @@ func TestFilesByGlob(t *testing.T) {
 	}
 }
 
+func TestDirectoryExclusions(t *testing.T) {
+	testCases := []struct {
+		desc       string
+		input      string
+		glob       string
+		expected   int
+		exclusions []string
+		err        bool
+	}{
+		{
+			input:      "test-fixtures/system_paths",
+			desc:       "exclude everything",
+			glob:       "**",
+			expected:   0,
+			exclusions: []string{"**/*"},
+		},
+		{
+			input:      "test-fixtures/image-simple",
+			desc:       "a single path excluded",
+			glob:       "**",
+			expected:   3,
+			exclusions: []string{"**/target/**"},
+		},
+		{
+			input:      "test-fixtures/image-simple",
+			desc:       "exclude explicit directory relative to the root",
+			glob:       "**",
+			expected:   3,
+			exclusions: []string{"./target"},
+		},
+		{
+			input:      "test-fixtures/image-simple",
+			desc:       "exclude explicit file relative to the root",
+			glob:       "**",
+			expected:   3,
+			exclusions: []string{"./file-1.txt"},
+		},
+		{
+			input:      "test-fixtures/image-simple",
+			desc:       "exclude wildcard relative to the root",
+			glob:       "**",
+			expected:   2,
+			exclusions: []string{"./*.txt"},
+		},
+		{
+			input:      "test-fixtures/image-simple",
+			desc:       "exclude files deeper",
+			glob:       "**",
+			expected:   3,
+			exclusions: []string{"**/really/**"},
+		},
+		{
+			input:      "test-fixtures/image-simple",
+			desc:       "files excluded with extension",
+			glob:       "**",
+			expected:   1,
+			exclusions: []string{"**/*.txt"},
+		},
+		{
+			input:      "test-fixtures/image-simple",
+			desc:       "keep files with different extensions",
+			glob:       "**",
+			expected:   4,
+			exclusions: []string{"**/target/**/*.jar"},
+		},
+		{
+			input:      "test-fixtures/path-detected",
+			desc:       "file directly excluded",
+			glob:       "**",
+			expected:   1,
+			exclusions: []string{"**/empty"},
+		},
+		{
+			input:      "test-fixtures/path-detected",
+			desc:       "pattern error containing **/",
+			glob:       "**",
+			expected:   1,
+			exclusions: []string{"/**/empty"},
+			err:        true,
+		},
+		{
+			input:      "test-fixtures/path-detected",
+			desc:       "pattern error incorrect start",
+			glob:       "**",
+			expected:   1,
+			exclusions: []string{"empty"},
+			err:        true,
+		},
+		{
+			input:      "test-fixtures/path-detected",
+			desc:       "pattern error starting with /",
+			glob:       "**",
+			expected:   1,
+			exclusions: []string{"/empty"},
+			err:        true,
+		},
+	}
+	registryOpts := &image.RegistryOptions{}
+	for _, test := range testCases {
+		t.Run(test.desc, func(t *testing.T) {
+			src, fn, err := New("dir:"+test.input, registryOpts, test.exclusions)
+			defer fn()
+
+			if test.err {
+				_, err = src.FileResolver(SquashedScope)
+				if err == nil {
+					t.Errorf("expected an error for patterns: %s", strings.Join(test.exclusions, " or "))
+				}
+				return
+			}
+
+			if err != nil {
+				t.Errorf("could not create NewDirScope: %+v", err)
+			}
+			resolver, err := src.FileResolver(SquashedScope)
+			if err != nil {
+				t.Errorf("could not get resolver error: %+v", err)
+			}
+			contents, err := resolver.FilesByGlob(test.glob)
+			if err != nil {
+				t.Errorf("could not get files by glob: %s+v", err)
+			}
+			if len(contents) != test.expected {
+				t.Errorf("wrong number of files after exclusions (%s): %d != %d", test.glob, len(contents), test.expected)
+			}
+		})
+	}
+}
+
+func TestImageExclusions(t *testing.T) {
+	testCases := []struct {
+		desc       string
+		input      string
+		glob       string
+		expected   int
+		exclusions []string
+	}{
+		// NOTE: in the Dockerfile, /target is moved to /, which makes /really a top-level dir
+		{
+			input:      "image-simple",
+			desc:       "a single path excluded",
+			glob:       "**",
+			expected:   2,
+			exclusions: []string{"/really/**"},
+		},
+		{
+			input:      "image-simple",
+			desc:       "a directly referenced directory is excluded",
+			glob:       "**",
+			expected:   2,
+			exclusions: []string{"/really"},
+		},
+		{
+			input:      "image-simple",
+			desc:       "a partial directory is not excluded",
+			glob:       "**",
+			expected:   3,
+			exclusions: []string{"/reall"},
+		},
+		{
+			input:      "image-simple",
+			desc:       "exclude files deeper",
+			glob:       "**",
+			expected:   2,
+			exclusions: []string{"**/nested/**"},
+		},
+		{
+			input:      "image-simple",
+			desc:       "files excluded with extension",
+			glob:       "**",
+			expected:   2,
+			exclusions: []string{"**/*1.txt"},
+		},
+		{
+			input:      "image-simple",
+			desc:       "keep files with different extensions",
+			glob:       "**",
+			expected:   3,
+			exclusions: []string{"**/target/**/*.jar"},
+		},
+		{
+			input:      "image-simple",
+			desc:       "file directly excluded",
+			glob:       "**",
+			expected:   2,
+			exclusions: []string{"**/somefile-1.txt"}, // file-1 renamed to somefile-1 in Dockerfile
+		},
+	}
+	registryOpts := &image.RegistryOptions{}
+	for _, test := range testCases {
+		t.Run(test.desc, func(t *testing.T) {
+			archiveLocation := imagetest.PrepareFixtureImage(t, "docker-archive", test.input)
+			src, fn, err := New(archiveLocation, registryOpts, test.exclusions)
+			defer fn()
+
+			if err != nil {
+				t.Errorf("could not create NewDirScope: %+v", err)
+			}
+			resolver, err := src.FileResolver(SquashedScope)
+			if err != nil {
+				t.Errorf("could not get resolver error: %+v", err)
+			}
+			contents, err := resolver.FilesByGlob(test.glob)
+			if err != nil {
+				t.Errorf("could not get files by glob: %s+v", err)
+			}
+			if len(contents) != test.expected {
+				t.Errorf("wrong number of files after exclusions (%s): %d != %d", test.glob, len(contents), test.expected)
+			}
+		})
+	}
+}
+
 // createArchive creates a new archive file at destinationArchivePath based on the directory found at sourceDirPath.
 func createArchive(t testing.TB, sourceDirPath, destinationArchivePath string) {
 	t.Helper()

syft/source/test-fixtures/symlinks-loop/README.md

@@ -0,0 +1 @@
+this mimics a partial layout on a linux system within /sys/devices/virtual to help with ensuring globbing for files does not end up in a infinite loop

syft/source/test-fixtures/symlinks-loop/block/loop0

@@ -0,0 +1 @@
+../devices/loop0

syft/source/test-fixtures/symlinks-loop/devices/loop0/file.target

@@ -0,0 +1 @@
+contents!

syft/source/test-fixtures/symlinks-loop/devices/loop0/subsystem

@@ -0,0 +1 @@
+../../block

syft/source/test-fixtures/symlinks-roots/outside/link_to_readme

@@ -0,0 +1 @@
+../root/readme

syft/source/test-fixtures/symlinks-roots/root/link_to_link_to_readme

@@ -0,0 +1 @@
+../outside/link_to_readme

syft/source/test-fixtures/symlinks-roots/root/readme

@@ -0,0 +1 @@
+contents!