remove unstable CLI test (bad binaries) (#721)

Signed-off-by: Alex Goodman <alex.goodman@anchore.com>

.github/scripts/goreleaser-install.sh

@@ -338,7 +338,7 @@ hash_sha256_verify() {
 		return 1
 	fi
 	BASENAME=${TARGET##*/}
-	want=$(grep "${BASENAME}" "${checksums}" 2>/dev/null | tr '\t' ' ' | cut -d ' ' -f 1)
+	want=$(grep "${BASENAME}\$" "${checksums}" 2>/dev/null | tr '\t' ' ' | cut -d ' ' -f 1)
 	if [ -z "$want" ]; then
 		log_err "hash_sha256_verify unable to find checksum for '${TARGET}' in '${checksums}'"
 		return 1

Makefile

@@ -100,7 +100,7 @@ bootstrap-tools: $(TEMPDIR)
 	curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(TEMPDIR)/ v1.42.1
 	curl -sSfL https://raw.githubusercontent.com/wagoodman/go-bouncer/master/bouncer.sh | sh -s -- -b $(TEMPDIR)/ v0.2.0
 	curl -sSfL https://raw.githubusercontent.com/anchore/chronicle/main/install.sh | sh -s -- -b $(TEMPDIR)/ v0.3.0
-	.github/scripts/goreleaser-install.sh -b $(TEMPDIR)/ v0.177.0
+	.github/scripts/goreleaser-install.sh -b $(TEMPDIR)/ v1.2.2
 	GOBIN="$(shell realpath $(TEMPDIR))" go install github.com/neilpa/yajsv@v1.4.0
 
 .PHONY: bootstrap-go

README.md

@@ -93,6 +93,23 @@ file:path/to/yourproject/file          read directly from a path on disk (any si
 registry:yourrepo/yourimage:tag        pull image directly from a registry (no container runtime required)
 ```
 
+### Excluding file paths
+
+Syft can exclude files and paths from being scanned within a source by using glob expressions
+with one or more `--exclude` parameters:
+```
+syft <source> --exclude './out/**/*.json' --exclude /etc
+```
+**Note:** in the case of _image scanning_, since the entire filesystem is scanned it is
+possible to use absolute paths like `/etc` or `/usr/**/*.txt` whereas _directory scans_
+exclude files _relative to the specified directory_. For example: scanning `/usr/foo` with
+`--exclude ./package.json` would exclude `/usr/foo/package.json` and `--exclude '**/package.json'`
+would exclude all `package.json` files under `/usr/foo`. For _directory scans_,
+it is required to begin path expressions with `./`, `*/`, or `**/`, all of which
+will be resolved _relative to the specified scan directory_. Keep in mind, your shell
+may attempt to expand wildcards, so put those parameters in single quotes, like:
+`'**/*.json'`.
+
 ### Output formats
 
 The output format for Syft is configurable as well:
@@ -219,6 +236,12 @@ file: ""
 # same as SYFT_CHECK_FOR_APP_UPDATE env var
 check-for-app-update: true
 
+# a list of globs to exclude from scanning. same as --exclude ; for example:
+# exclude:
+#   - '/etc/**'
+#   - './out/**/*.json'
+exclude:
+
 # cataloging packages is exposed through the packages and power-user subcommands
 package:
   cataloger:

cmd/check_for_application_update.go

@@ -11,6 +11,7 @@ import (
 
 func checkForApplicationUpdate() {
 	if appConfig.CheckForAppUpdate {
+		log.Debugf("checking if new vesion of %s is available", internal.ApplicationName)
 		isAvailable, newVersion, err := version.IsUpdateAvailable()
 		if err != nil {
 			// this should never stop the application

cmd/cmd.go

@@ -1,17 +1,19 @@
 package cmd
 
 import (
+	"encoding/json"
 	"fmt"
 	"os"
-
-	"github.com/spf13/cobra"
+	"sort"
 
 	"github.com/anchore/stereoscope"
 	"github.com/anchore/syft/internal/config"
 	"github.com/anchore/syft/internal/log"
 	"github.com/anchore/syft/internal/logger"
+	"github.com/anchore/syft/internal/version"
 	"github.com/anchore/syft/syft"
 	"github.com/gookit/color"
+	"github.com/spf13/cobra"
 	"github.com/spf13/viper"
 	"github.com/wagoodman/go-partybus"
 )
@@ -28,6 +30,8 @@ func init() {
 		initAppConfig,
 		initLogging,
 		logAppConfig,
+		checkForApplicationUpdate,
+		logAppVersion,
 		initEventBus,
 	)
 }
@@ -110,3 +114,33 @@ func initEventBus() {
 	stereoscope.SetBus(eventBus)
 	syft.SetBus(eventBus)
 }
+
+func logAppVersion() {
+	versionInfo := version.FromBuild()
+	log.Infof("syft version: %s", versionInfo.Version)
+
+	var fields map[string]interface{}
+	bytes, err := json.Marshal(versionInfo)
+	if err != nil {
+		return
+	}
+	err = json.Unmarshal(bytes, &fields)
+	if err != nil {
+		return
+	}
+
+	keys := make([]string, 0, len(fields))
+	for k := range fields {
+		keys = append(keys, k)
+	}
+	sort.Strings(keys)
+
+	for idx, field := range keys {
+		value := fields[field]
+		branch := "├──"
+		if idx == len(fields)-1 {
+			branch = "└──"
+		}
+		log.Debugf("  %s %s: %s", branch, field, value)
+	}
+}

cmd/packages.go

@@ -132,6 +132,11 @@ func setPackageFlags(flags *pflag.FlagSet) {
 		"include dockerfile for upload to Anchore Enterprise",
 	)
 
+	flags.StringArrayP(
+		"exclude", "", nil,
+		"exclude paths from being scanned using a glob expression",
+	)
+
 	flags.Bool(
 		"overwrite-existing-image", false,
 		"overwrite an existing image during the upload to Anchore Enterprise",
@@ -158,6 +163,10 @@ func bindPackagesConfigOptions(flags *pflag.FlagSet) error {
 		return err
 	}
 
+	if err := viper.BindPFlag("exclude", flags.Lookup("exclude")); err != nil {
+		return err
+	}
+
 	// Upload options //////////////////////////////////////////////////////////
 
 	if err := viper.BindPFlag("anchore.host", flags.Lookup("host")); err != nil {
@@ -251,11 +260,9 @@ func packagesExecWorker(userInput string) <-chan error {
 			return
 		}
 
-		checkForApplicationUpdate()
-
-		src, cleanup, err := source.New(userInput, appConfig.Registry.ToOptions())
+		src, cleanup, err := source.New(userInput, appConfig.Registry.ToOptions(), appConfig.Exclusions)
 		if err != nil {
-			errs <- fmt.Errorf("failed to determine image source: %w", err)
+			errs <- fmt.Errorf("failed to construct source from user input %q: %w", userInput, err)
 			return
 		}
 		if cleanup != nil {

cmd/power_user.go

@@ -111,9 +111,7 @@ func powerUserExecWorker(userInput string) <-chan error {
 			return
 		}
 
-		checkForApplicationUpdate()
-
-		src, cleanup, err := source.New(userInput, appConfig.Registry.ToOptions())
+		src, cleanup, err := source.New(userInput, appConfig.Registry.ToOptions(), appConfig.Exclusions)
 		if err != nil {
 			errs <- err
 			return

go.mod

@@ -13,7 +13,7 @@ require (
 	github.com/anchore/go-testutils v0.0.0-20200925183923-d5f45b0d3c04
 	github.com/anchore/go-version v1.2.2-0.20200701162849-18adb9c92b9b
 	github.com/anchore/packageurl-go v0.0.0-20210922164639-b3fa992ebd29
-	github.com/anchore/stereoscope v0.0.0-20211203160213-5a5e323a5c89
+	github.com/anchore/stereoscope v0.0.0-20211222141827-6e663afeef5d
 	// we are hinting brotli to latest due to warning when installing archiver v3:
 	// go: warning: github.com/andybalholm/brotli@v1.0.1: retracted by module author: occasional panics and data corruption
 	github.com/andybalholm/brotli v1.0.4 // indirect

go.sum

@@ -111,8 +111,8 @@ github.com/anchore/go-version v1.2.2-0.20200701162849-18adb9c92b9b h1:e1bmaoJfZV
 github.com/anchore/go-version v1.2.2-0.20200701162849-18adb9c92b9b/go.mod h1:Bkc+JYWjMCF8OyZ340IMSIi2Ebf3uwByOk6ho4wne1E=
 github.com/anchore/packageurl-go v0.0.0-20210922164639-b3fa992ebd29 h1:K9LfnxwhqvihqU0+MF325FNy7fsKV9EGaUxdfR4gnWk=
 github.com/anchore/packageurl-go v0.0.0-20210922164639-b3fa992ebd29/go.mod h1:Oc1UkGaJwY6ND6vtAqPSlYrptKRJngHwkwB6W7l1uP0=
-github.com/anchore/stereoscope v0.0.0-20211203160213-5a5e323a5c89 h1:pcMFN7wjIQIxdZm8NA0R3JiRRFn5Eyx9mhagHOVS4Bw=
-github.com/anchore/stereoscope v0.0.0-20211203160213-5a5e323a5c89/go.mod h1:FNm1rtauEjkC3elA3jr3bJkU+/4QiovApwJdPnHQ9x0=
+github.com/anchore/stereoscope v0.0.0-20211222141827-6e663afeef5d h1:ggdC5SRb8rAXmBkwxah8gfSxwpoAVcpp12tQ+XbDi20=
+github.com/anchore/stereoscope v0.0.0-20211222141827-6e663afeef5d/go.mod h1:FNm1rtauEjkC3elA3jr3bJkU+/4QiovApwJdPnHQ9x0=
 github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883/go.mod h1:rCTlJbsFo29Kk6CurOXKm700vrz8f0KW0JNfpkRJY/8=
 github.com/andybalholm/brotli v1.0.1/go.mod h1:loMXtMfwqflxFJPmdbJO0a3KNoPuLBgiu3qAvBg8x/Y=
 github.com/andybalholm/brotli v1.0.4 h1:V7DdXeJtZscaqfNuAdSRuRFzuiKlHSC/Zh3zl9qY3JY=

install.sh

@@ -246,7 +246,7 @@ extract_from_dmg() {
   dmg_file=$1
   mount_point="/Volumes/tmp-dmg"
   hdiutil attach -quiet -nobrowse -mountpoint "${mount_point}" "${dmg_file}"
-  cp -fR "${mount_point}/" ./
+  cp -fR "${mount_point}/." ./
   hdiutil detach -quiet -force "${mount_point}"
 }
 http_download_curl() {

internal/config/application.go

@@ -44,6 +44,7 @@ type Application struct {
 	FileContents       fileContents       `yaml:"file-contents" json:"file-contents" mapstructure:"file-contents"`
 	Secrets            secrets            `yaml:"secrets" json:"secrets" mapstructure:"secrets"`
 	Registry           registry           `yaml:"registry" json:"registry" mapstructure:"registry"`
+	Exclusions         []string           `yaml:"exclude" json:"exclude" mapstructure:"exclude"`
 }
 
 // PowerUserCatalogerEnabledDefault switches all catalogers to be enabled when running power-user command

internal/constants.go

@@ -6,5 +6,5 @@ const (
 
 	// JSONSchemaVersion is the current schema version output by the JSON presenter
 	// This is roughly following the "SchemaVer" guidelines for versioning the JSON schema. Please see schema/json/README.md for details on how to increment.
-	JSONSchemaVersion = "2.0.1"
+	JSONSchemaVersion = "2.0.2"
 )

internal/formats/spdx22json/test-fixtures/snapshot/TestSPDXJSONDirectoryPresenter.golden

@@ -3,7 +3,7 @@
  "name": "/some/path",
  "spdxVersion": "SPDX-2.2",
  "creationInfo": {
-  "created": "2021-12-01T15:08:29.469369Z",
+  "created": "2021-12-20T19:12:47.869816Z",
   "creators": [
    "Organization: Anchore, Inc",
    "Tool: syft-[not provided]"
@@ -11,10 +11,10 @@
   "licenseListVersion": "3.15"
  },
  "dataLicense": "CC0-1.0",
- "documentNamespace": "https://anchore.com/syft/dir/some/path-f4586501-2da6-4541-a8e9-232b32f25e9a",
+ "documentNamespace": "https://anchore.com/syft/dir/some/path-4b896ded-7852-4e31-b764-136b53bdf346",
  "packages": [
   {
-   "SPDXID": "SPDXRef-2a115ac97d018a0e",
+   "SPDXID": "SPDXRef-1d97af55efe9512f",
    "name": "package-1",
    "licenseConcluded": "MIT",
    "downloadLocation": "NOASSERTION",
@@ -36,7 +36,7 @@
    "versionInfo": "1.0.1"
   },
   {
-   "SPDXID": "SPDXRef-5e920b2bece2c3ae",
+   "SPDXID": "SPDXRef-ad3d1c4abd84bf75",
    "name": "package-2",
    "licenseConcluded": "NONE",
    "downloadLocation": "NOASSERTION",

internal/formats/spdx22json/test-fixtures/snapshot/TestSPDXJSONImagePresenter.golden

@@ -3,7 +3,7 @@
  "name": "user-image-input",
  "spdxVersion": "SPDX-2.2",
  "creationInfo": {
-  "created": "2021-12-01T15:08:29.476498Z",
+  "created": "2021-12-20T19:13:07.647486Z",
   "creators": [
    "Organization: Anchore, Inc",
    "Tool: syft-[not provided]"
@@ -11,10 +11,10 @@
   "licenseListVersion": "3.15"
  },
  "dataLicense": "CC0-1.0",
- "documentNamespace": "https://anchore.com/syft/image/user-image-input-e3b7637c-9b2f-4005-a683-58e60f979082",
+ "documentNamespace": "https://anchore.com/syft/image/user-image-input-174da656-1824-4bd3-8604-28919f8a65bc",
  "packages": [
   {
-   "SPDXID": "SPDXRef-888661d4f0362f02",
+   "SPDXID": "SPDXRef-d16127444133b5c1",
    "name": "package-1",
    "licenseConcluded": "MIT",
    "downloadLocation": "NOASSERTION",
@@ -36,7 +36,7 @@
    "versionInfo": "1.0.1"
   },
   {
-   "SPDXID": "SPDXRef-4068ff5e8926b305",
+   "SPDXID": "SPDXRef-24907357f3705420",
    "name": "package-2",
    "licenseConcluded": "NONE",
    "downloadLocation": "NOASSERTION",

internal/formats/syftjson/encoder.go

@@ -8,7 +8,7 @@ import (
 )
 
 func encoder(output io.Writer, s sbom.SBOM) error {
-	doc := ToFormatModel(s)
+	doc := toFormatModel(s)
 
 	enc := json.NewEncoder(output)
 	// prevent > and < from being escaped in the payload

internal/formats/syftjson/model/package.go

@@ -4,9 +4,10 @@ import (
 	"encoding/json"
 	"fmt"
 
+	"github.com/anchore/syft/syft/source"
+
 	"github.com/anchore/syft/internal/log"
 	"github.com/anchore/syft/syft/pkg"
-	"github.com/anchore/syft/syft/source"
 )
 
 // Package represents a pkg.Package object specialized for JSON marshaling and unmarshalling.

internal/formats/syftjson/test-fixtures/snapshot/TestDirectoryPresenter.golden

@@ -1,7 +1,7 @@
 {
  "artifacts": [
   {
-   "id": "2a115ac97d018a0e",
+   "id": "1d97af55efe9512f",
    "name": "package-1",
    "version": "1.0.1",
    "type": "python",
@@ -36,7 +36,7 @@
    }
   },
   {
-   "id": "5e920b2bece2c3ae",
+   "id": "ad3d1c4abd84bf75",
    "name": "package-2",
    "version": "2.0.1",
    "type": "deb",
@@ -83,7 +83,7 @@
   }
  },
  "schema": {
-  "version": "2.0.1",
-  "url": "https://raw.githubusercontent.com/anchore/syft/main/schema/json/schema-2.0.1.json"
+  "version": "2.0.2",
+  "url": "https://raw.githubusercontent.com/anchore/syft/main/schema/json/schema-2.0.2.json"
  }
 }

internal/formats/syftjson/test-fixtures/snapshot/TestEncodeFullJSONDocument.golden

@@ -1,7 +1,7 @@
 {
  "artifacts": [
   {
-   "id": "962403cfb7be50d7",
+   "id": "d9a7c58726ab4bef",
    "name": "package-1",
    "version": "1.0.1",
    "type": "python",
@@ -31,7 +31,7 @@
    }
   },
   {
-   "id": "b11f44847bba0ed1",
+   "id": "ac462e450060da2c",
    "name": "package-2",
    "version": "2.0.1",
    "type": "deb",
@@ -62,8 +62,8 @@
  ],
  "artifactRelationships": [
   {
-   "parent": "962403cfb7be50d7",
-   "child": "b11f44847bba0ed1",
+   "parent": "",
+   "child": "",
    "type": "ownership-by-file-overlap",
    "metadata": {
     "file": "path"
@@ -179,7 +179,7 @@
   }
  },
  "schema": {
-  "version": "2.0.1",
-  "url": "https://raw.githubusercontent.com/anchore/syft/main/schema/json/schema-2.0.1.json"
+  "version": "2.0.2",
+  "url": "https://raw.githubusercontent.com/anchore/syft/main/schema/json/schema-2.0.2.json"
  }
 }

internal/formats/syftjson/test-fixtures/snapshot/TestImagePresenter.golden

@@ -1,7 +1,7 @@
 {
  "artifacts": [
   {
-   "id": "888661d4f0362f02",
+   "id": "d16127444133b5c1",
    "name": "package-1",
    "version": "1.0.1",
    "type": "python",
@@ -9,7 +9,7 @@
    "locations": [
     {
      "path": "/somefile-1.txt",
-     "layerID": "sha256:fb6beecb75b39f4bb813dbf177e501edd5ddb3e69bb45cedeb78c676ee1b7a59"
+     "layerID": "sha256:16e64541f2ddf59a90391ce7bb8af90313f7d373f2105d88f3d3267b72e0ebab"
     }
    ],
    "licenses": [
@@ -32,7 +32,7 @@
    }
   },
   {
-   "id": "4068ff5e8926b305",
+   "id": "24907357f3705420",
    "name": "package-2",
    "version": "2.0.1",
    "type": "deb",
@@ -40,7 +40,7 @@
    "locations": [
     {
      "path": "/somefile-2.txt",
-     "layerID": "sha256:319b588ce64253a87b533c8ed01cf0025e0eac98e7b516e12532957e1244fdec"
+     "layerID": "sha256:de6c235f76ea24c8503ec08891445b5d6a8bdf8249117ed8d8b0b6fb3ebe4f67"
     }
    ],
    "licenses": [],
@@ -67,7 +67,7 @@
   "type": "image",
   "target": {
    "userInput": "user-image-input",
-   "imageID": "sha256:2480160b55bec40c44d3b145c7b2c1c47160db8575c3dcae086d76b9370ae7ca",
+   "imageID": "sha256:9624b89704d23fa5f61b427379d172dac91dc7a508c4d7dea7aed0e04a4cf39e",
    "manifestDigest": "sha256:2731251dc34951c0e50fcc643b4c5f74922dad1a5d98f302b504cf46cd5d9368",
    "mediaType": "application/vnd.docker.distribution.manifest.v2+json",
    "tags": [
@@ -77,17 +77,17 @@
    "layers": [
     {
      "mediaType": "application/vnd.docker.image.rootfs.diff.tar.gzip",
-     "digest": "sha256:fb6beecb75b39f4bb813dbf177e501edd5ddb3e69bb45cedeb78c676ee1b7a59",
+     "digest": "sha256:16e64541f2ddf59a90391ce7bb8af90313f7d373f2105d88f3d3267b72e0ebab",
      "size": 22
     },
     {
      "mediaType": "application/vnd.docker.image.rootfs.diff.tar.gzip",
-     "digest": "sha256:319b588ce64253a87b533c8ed01cf0025e0eac98e7b516e12532957e1244fdec",
+     "digest": "sha256:de6c235f76ea24c8503ec08891445b5d6a8bdf8249117ed8d8b0b6fb3ebe4f67",
      "size": 16
     }
    ],
-   "manifest": "eyJzY2hlbWFWZXJzaW9uIjoyLCJtZWRpYVR5cGUiOiJhcHBsaWNhdGlvbi92bmQuZG9ja2VyLmRpc3RyaWJ1dGlvbi5tYW5pZmVzdC52Mitqc29uIiwiY29uZmlnIjp7Im1lZGlhVHlwZSI6ImFwcGxpY2F0aW9uL3ZuZC5kb2NrZXIuY29udGFpbmVyLmltYWdlLnYxK2pzb24iLCJzaXplIjo2NjcsImRpZ2VzdCI6InNoYTI1NjoyNDgwMTYwYjU1YmVjNDBjNDRkM2IxNDVjN2IyYzFjNDcxNjBkYjg1NzVjM2RjYWUwODZkNzZiOTM3MGFlN2NhIn0sImxheWVycyI6W3sibWVkaWFUeXBlIjoiYXBwbGljYXRpb24vdm5kLmRvY2tlci5pbWFnZS5yb290ZnMuZGlmZi50YXIuZ3ppcCIsInNpemUiOjIwNDgsImRpZ2VzdCI6InNoYTI1NjpmYjZiZWVjYjc1YjM5ZjRiYjgxM2RiZjE3N2U1MDFlZGQ1ZGRiM2U2OWJiNDVjZWRlYjc4YzY3NmVlMWI3YTU5In0seyJtZWRpYVR5cGUiOiJhcHBsaWNhdGlvbi92bmQuZG9ja2VyLmltYWdlLnJvb3Rmcy5kaWZmLnRhci5nemlwIiwic2l6ZSI6MjA0OCwiZGlnZXN0Ijoic2hhMjU2OjMxOWI1ODhjZTY0MjUzYTg3YjUzM2M4ZWQwMWNmMDAyNWUwZWFjOThlN2I1MTZlMTI1MzI5NTdlMTI0NGZkZWMifV19",
-   "config": "eyJhcmNoaXRlY3R1cmUiOiJhbWQ2NCIsImNvbmZpZyI6eyJFbnYiOlsiUEFUSD0vdXNyL2xvY2FsL3NiaW46L3Vzci9sb2NhbC9iaW46L3Vzci9zYmluOi91c3IvYmluOi9zYmluOi9iaW4iXSwiV29ya2luZ0RpciI6Ii8iLCJPbkJ1aWxkIjpudWxsfSwiY3JlYXRlZCI6IjIwMjEtMTAtMDRUMTE6NDA6MDAuNjM4Mzk0NVoiLCJoaXN0b3J5IjpbeyJjcmVhdGVkIjoiMjAyMS0xMC0wNFQxMTo0MDowMC41OTA3MzE2WiIsImNyZWF0ZWRfYnkiOiJBREQgZmlsZS0xLnR4dCAvc29tZWZpbGUtMS50eHQgIyBidWlsZGtpdCIsImNvbW1lbnQiOiJidWlsZGtpdC5kb2NrZXJmaWxlLnYwIn0seyJjcmVhdGVkIjoiMjAyMS0xMC0wNFQxMTo0MDowMC42MzgzOTQ1WiIsImNyZWF0ZWRfYnkiOiJBREQgZmlsZS0yLnR4dCAvc29tZWZpbGUtMi50eHQgIyBidWlsZGtpdCIsImNvbW1lbnQiOiJidWlsZGtpdC5kb2NrZXJmaWxlLnYwIn1dLCJvcyI6ImxpbnV4Iiwicm9vdGZzIjp7InR5cGUiOiJsYXllcnMiLCJkaWZmX2lkcyI6WyJzaGEyNTY6ZmI2YmVlY2I3NWIzOWY0YmI4MTNkYmYxNzdlNTAxZWRkNWRkYjNlNjliYjQ1Y2VkZWI3OGM2NzZlZTFiN2E1OSIsInNoYTI1NjozMTliNTg4Y2U2NDI1M2E4N2I1MzNjOGVkMDFjZjAwMjVlMGVhYzk4ZTdiNTE2ZTEyNTMyOTU3ZTEyNDRmZGVjIl19fQ==",
+   "manifest": "eyJzY2hlbWFWZXJzaW9uIjoyLCJtZWRpYVR5cGUiOiJhcHBsaWNhdGlvbi92bmQuZG9ja2VyLmRpc3RyaWJ1dGlvbi5tYW5pZmVzdC52Mitqc29uIiwiY29uZmlnIjp7Im1lZGlhVHlwZSI6ImFwcGxpY2F0aW9uL3ZuZC5kb2NrZXIuY29udGFpbmVyLmltYWdlLnYxK2pzb24iLCJzaXplIjo2NjcsImRpZ2VzdCI6InNoYTI1Njo5NjI0Yjg5NzA0ZDIzZmE1ZjYxYjQyNzM3OWQxNzJkYWM5MWRjN2E1MDhjNGQ3ZGVhN2FlZDBlMDRhNGNmMzllIn0sImxheWVycyI6W3sibWVkaWFUeXBlIjoiYXBwbGljYXRpb24vdm5kLmRvY2tlci5pbWFnZS5yb290ZnMuZGlmZi50YXIuZ3ppcCIsInNpemUiOjIwNDgsImRpZ2VzdCI6InNoYTI1NjoxNmU2NDU0MWYyZGRmNTlhOTAzOTFjZTdiYjhhZjkwMzEzZjdkMzczZjIxMDVkODhmM2QzMjY3YjcyZTBlYmFiIn0seyJtZWRpYVR5cGUiOiJhcHBsaWNhdGlvbi92bmQuZG9ja2VyLmltYWdlLnJvb3Rmcy5kaWZmLnRhci5nemlwIiwic2l6ZSI6MjA0OCwiZGlnZXN0Ijoic2hhMjU2OmRlNmMyMzVmNzZlYTI0Yzg1MDNlYzA4ODkxNDQ1YjVkNmE4YmRmODI0OTExN2VkOGQ4YjBiNmZiM2ViZTRmNjcifV19",
+   "config": "eyJhcmNoaXRlY3R1cmUiOiJhbWQ2NCIsImNvbmZpZyI6eyJFbnYiOlsiUEFUSD0vdXNyL2xvY2FsL3NiaW46L3Vzci9sb2NhbC9iaW46L3Vzci9zYmluOi91c3IvYmluOi9zYmluOi9iaW4iXSwiV29ya2luZ0RpciI6Ii8iLCJPbkJ1aWxkIjpudWxsfSwiY3JlYXRlZCI6IjIwMjEtMTItMDFUMTI6MTM6NDMuNDAxMzkxNFoiLCJoaXN0b3J5IjpbeyJjcmVhdGVkIjoiMjAyMS0xMi0wMVQxMjoxMzo0My4zNDM2MzQyWiIsImNyZWF0ZWRfYnkiOiJBREQgZmlsZS0xLnR4dCAvc29tZWZpbGUtMS50eHQgIyBidWlsZGtpdCIsImNvbW1lbnQiOiJidWlsZGtpdC5kb2NrZXJmaWxlLnYwIn0seyJjcmVhdGVkIjoiMjAyMS0xMi0wMVQxMjoxMzo0My40MDEzOTE0WiIsImNyZWF0ZWRfYnkiOiJBREQgZmlsZS0yLnR4dCAvc29tZWZpbGUtMi50eHQgIyBidWlsZGtpdCIsImNvbW1lbnQiOiJidWlsZGtpdC5kb2NrZXJmaWxlLnYwIn1dLCJvcyI6ImxpbnV4Iiwicm9vdGZzIjp7InR5cGUiOiJsYXllcnMiLCJkaWZmX2lkcyI6WyJzaGEyNTY6MTZlNjQ1NDFmMmRkZjU5YTkwMzkxY2U3YmI4YWY5MDMxM2Y3ZDM3M2YyMTA1ZDg4ZjNkMzI2N2I3MmUwZWJhYiIsInNoYTI1NjpkZTZjMjM1Zjc2ZWEyNGM4NTAzZWMwODg5MTQ0NWI1ZDZhOGJkZjgyNDkxMTdlZDhkOGIwYjZmYjNlYmU0ZjY3Il19fQ==",
    "repoDigests": []
   }
  },
@@ -104,7 +104,7 @@
   }
  },
  "schema": {
-  "version": "2.0.1",
-  "url": "https://raw.githubusercontent.com/anchore/syft/main/schema/json/schema-2.0.1.json"
+  "version": "2.0.2",
+  "url": "https://raw.githubusercontent.com/anchore/syft/main/schema/json/schema-2.0.2.json"
  }
 }

internal/formats/syftjson/to_format_model.go

@@ -19,8 +19,7 @@ import (
 	"github.com/anchore/syft/syft/source"
 )
 
-// TODO: this is exported for the use of the power-user command (temp)
-func ToFormatModel(s sbom.SBOM) model.Document {
+func toFormatModel(s sbom.SBOM) model.Document {
 	src, err := toSourceModel(s.Source)
 	if err != nil {
 		log.Warnf("unable to create syft-json source object: %+v", err)

syft/artifact/id.go

@@ -13,7 +13,7 @@ type Identifiable interface {
 	ID() ID
 }
 
-func IDFromHash(obj interface{}) (ID, error) {
+func IDByHash(obj interface{}) (ID, error) {
 	f, err := hashstructure.Hash(obj, hashstructure.FormatV2, &hashstructure.HashOptions{
 		ZeroNil:      true,
 		SlicesAsSets: true,

syft/pkg/cargo_package_metadata.go

@@ -9,8 +9,8 @@ type CargoPackageMetadata struct {
 }
 
 // Pkg returns the standard `pkg.Package` representation of the package referenced within the Cargo.lock metadata.
-func (p CargoPackageMetadata) Pkg() Package {
-	return Package{
+func (p CargoPackageMetadata) Pkg() *Package {
+	return &Package{
 		Name:         p.Name,
 		Version:      p.Version,
 		Language:     Rust,

syft/pkg/catalog.go

@@ -51,6 +51,7 @@ func (c *Catalog) Package(id artifact.ID) *Package {
 		log.Warnf("unable to copy package id=%q name=%q: %+v", id, v.Name, err)
 		return nil
 	}
+	p.id = v.id
 	return &p
 }
 
@@ -75,8 +76,12 @@ func (c *Catalog) Add(p Package) {
 	c.lock.Lock()
 	defer c.lock.Unlock()
 
-	// note: since we are capturing the ID, we cannot modify the package being added from this point forward
 	id := p.ID()
+	if id == "" {
+		log.Warnf("found package with empty ID while adding to the catalog: %+v", p)
+		p.SetID()
+		id = p.ID()
+	}
 
 	// store by package ID
 	c.byID[id] = p
@@ -142,7 +147,7 @@ func (c *Catalog) Sorted(types ...Type) (pkgs []Package) {
 	sort.SliceStable(pkgs, func(i, j int) bool {
 		if pkgs[i].Name == pkgs[j].Name {
 			if pkgs[i].Version == pkgs[j].Version {
-				if pkgs[i].Type == pkgs[j].Type {
+				if pkgs[i].Type == pkgs[j].Type && len(pkgs[i].Locations) > 0 && len(pkgs[j].Locations) > 0 {
 					return pkgs[i].Locations[0].String() < pkgs[j].Locations[0].String()
 				}
 				return pkgs[i].Type < pkgs[j].Type

syft/pkg/catalog_test.go

@@ -10,23 +10,6 @@ import (
 	"github.com/anchore/syft/syft/source"
 )
 
-var catalogAddAndRemoveTestPkgs = []Package{
-	{
-		Locations: []source.Location{
-			source.NewVirtualLocation("/a/path", "/another/path"),
-			source.NewVirtualLocation("/b/path", "/bee/path"),
-		},
-		Type: RpmPkg,
-	},
-	{
-		Locations: []source.Location{
-			source.NewVirtualLocation("/c/path", "/another/path"),
-			source.NewVirtualLocation("/d/path", "/another/path"),
-		},
-		Type: NpmPkg,
-	},
-}
-
 type expectedIndexes struct {
 	byType map[Type]*strset.Set
 	byPath map[string]*strset.Set
@@ -34,18 +17,38 @@ type expectedIndexes struct {
 
 func TestCatalogAddPopulatesIndex(t *testing.T) {
 
+	var pkgs = []Package{
+		{
+			Locations: []source.Location{
+				source.NewVirtualLocation("/a/path", "/another/path"),
+				source.NewVirtualLocation("/b/path", "/bee/path"),
+			},
+			Type: RpmPkg,
+		},
+		{
+			Locations: []source.Location{
+				source.NewVirtualLocation("/c/path", "/another/path"),
+				source.NewVirtualLocation("/d/path", "/another/path"),
+			},
+			Type: NpmPkg,
+		},
+	}
+
+	for i := range pkgs {
+		p := &pkgs[i]
+		p.SetID()
+	}
+
 	fixtureID := func(i int) string {
-		return string(catalogAddAndRemoveTestPkgs[i].ID())
+		return string(pkgs[i].ID())
 	}
 
 	tests := []struct {
 		name            string
-		pkgs            []Package
 		expectedIndexes expectedIndexes
 	}{
 		{
 			name: "vanilla-add",
-			pkgs: catalogAddAndRemoveTestPkgs,
 			expectedIndexes: expectedIndexes{
 				byType: map[Type]*strset.Set{
 					RpmPkg: strset.New(fixtureID(0)),
@@ -65,7 +68,7 @@ func TestCatalogAddPopulatesIndex(t *testing.T) {
 
 	for _, test := range tests {
 		t.Run(test.name, func(t *testing.T) {
-			c := NewCatalog(test.pkgs...)
+			c := NewCatalog(pkgs...)
 
 			assertIndexes(t, c, test.expectedIndexes)
 
@@ -75,9 +78,7 @@ func TestCatalogAddPopulatesIndex(t *testing.T) {
 
 func assertIndexes(t *testing.T, c *Catalog, expectedIndexes expectedIndexes) {
 	// assert path index
-	if len(c.idsByPath) != len(expectedIndexes.byPath) {
-		t.Errorf("unexpected path index length: %d != %d", len(c.idsByPath), len(expectedIndexes.byPath))
-	}
+	assert.Len(t, c.idsByPath, len(expectedIndexes.byPath), "unexpected path index length")
 	for path, expectedIds := range expectedIndexes.byPath {
 		actualIds := strset.New()
 		for _, p := range c.PackagesByPath(path) {
@@ -90,9 +91,7 @@ func assertIndexes(t *testing.T, c *Catalog, expectedIndexes expectedIndexes) {
 	}
 
 	// assert type index
-	if len(c.idsByType) != len(expectedIndexes.byType) {
-		t.Errorf("unexpected type index length: %d != %d", len(c.idsByType), len(expectedIndexes.byType))
-	}
+	assert.Len(t, c.idsByType, len(expectedIndexes.byType), "unexpected type index length")
 	for ty, expectedIds := range expectedIndexes.byType {
 		actualIds := strset.New()
 		for p := range c.Enumerate(ty) {

syft/pkg/cataloger/apkdb/parse_apk_db.go

@@ -21,14 +21,25 @@ import (
 // integrity check
 var _ common.ParserFn = parseApkDB
 
+func newApkDBPackage(d *pkg.ApkMetadata) *pkg.Package {
+	return &pkg.Package{
+		Name:         d.Package,
+		Version:      d.Version,
+		Licenses:     strings.Split(d.License, " "),
+		Type:         pkg.ApkPkg,
+		MetadataType: pkg.ApkMetadataType,
+		Metadata:     *d,
+	}
+}
+
 // parseApkDb parses individual packages from a given Alpine DB file. For more information on specific fields
 // see https://wiki.alpinelinux.org/wiki/Apk_spec .
-func parseApkDB(_ string, reader io.Reader) ([]pkg.Package, []artifact.Relationship, error) {
+func parseApkDB(_ string, reader io.Reader) ([]*pkg.Package, []artifact.Relationship, error) {
 	// larger capacity for the scanner.
 	const maxScannerCapacity = 1024 * 1024
 	// a new larger buffer for the scanner
 	bufScan := make([]byte, maxScannerCapacity)
-	packages := make([]pkg.Package, 0)
+	packages := make([]*pkg.Package, 0)
 
 	scanner := bufio.NewScanner(reader)
 	scanner.Buffer(bufScan, maxScannerCapacity)
@@ -52,14 +63,7 @@ func parseApkDB(_ string, reader io.Reader) ([]pkg.Package, []artifact.Relations
 			return nil, nil, err
 		}
 		if metadata != nil {
-			packages = append(packages, pkg.Package{
-				Name:         metadata.Package,
-				Version:      metadata.Version,
-				Licenses:     strings.Split(metadata.License, " "),
-				Type:         pkg.ApkPkg,
-				MetadataType: pkg.ApkMetadataType,
-				Metadata:     *metadata,
-			})
+			packages = append(packages, newApkDBPackage(metadata))
 		}
 	}
 

syft/pkg/cataloger/apkdb/parse_apk_db_test.go

@@ -647,11 +647,11 @@ func TestSinglePackageDetails(t *testing.T) {
 func TestMultiplePackages(t *testing.T) {
 	tests := []struct {
 		fixture  string
-		expected []pkg.Package
+		expected []*pkg.Package
 	}{
 		{
 			fixture: "test-fixtures/multiple",
-			expected: []pkg.Package{
+			expected: []*pkg.Package{
 				{
 					Name:         "libc-utils",
 					Version:      "0.7.2-r0",

syft/pkg/cataloger/catalog.go

@@ -49,10 +49,10 @@ func Catalog(resolver source.FileResolver, theDistro *distro.Distro, catalogers
 
 	// perform analysis, accumulating errors for each failed analysis
 	var errs error
-	for _, theCataloger := range catalogers {
+	for _, c := range catalogers {
 		// find packages from the underlying raw data
-		log.Debugf("cataloging with %q", theCataloger.Name())
-		packages, relationships, err := theCataloger.Catalog(resolver)
+		log.Debugf("cataloging with %q", c.Name())
+		packages, relationships, err := c.Catalog(resolver)
 		if err != nil {
 			errs = multierror.Append(errs, err)
 			continue
@@ -64,10 +64,10 @@ func Catalog(resolver source.FileResolver, theDistro *distro.Distro, catalogers
 		packagesDiscovered.N += int64(catalogedPackages)
 
 		for _, p := range packages {
-			// generate CPEs
+			// generate CPEs (note: this is excluded from package ID, so is safe to mutate)
 			p.CPEs = cpe.Generate(p)
 
-			// generate PURL
+			// generate PURL (note: this is excluded from package ID, so is safe to mutate)
 			p.PURL = generatePackageURL(p, theDistro)
 
 			// create file-to-package relationships for files owned by the package

syft/pkg/cataloger/common/generic_cataloger.go

@@ -45,22 +45,23 @@ func (c *GenericCataloger) Catalog(resolver source.FileResolver) ([]pkg.Package,
 		contentReader, err := resolver.FileContentsByLocation(location)
 		if err != nil {
 			// TODO: fail or log?
-			return nil, nil, fmt.Errorf("unable to fetch contents for location=%v : %w", location, err)
+			return nil, nil, fmt.Errorf("unable to fetch contents at location=%v: %w", location, err)
 		}
 
 		discoveredPackages, discoveredRelationships, err := parser(location.RealPath, contentReader)
 		internal.CloseAndLogError(contentReader, location.VirtualPath)
 		if err != nil {
 			// TODO: should we fail? or only log?
-			log.Warnf("cataloger '%s' failed to parse entries (location=%+v): %+v", c.upstreamCataloger, location, err)
+			log.Warnf("cataloger '%s' failed to parse entries at location=%+v: %+v", c.upstreamCataloger, location, err)
 			continue
 		}
 
 		for _, p := range discoveredPackages {
 			p.FoundBy = c.upstreamCataloger
 			p.Locations = append(p.Locations, location)
+			p.SetID()
 
-			packages = append(packages, p)
+			packages = append(packages, *p)
 		}
 
 		relationships = append(relationships, discoveredRelationships...)

syft/pkg/cataloger/common/generic_cataloger_test.go

@@ -13,12 +13,12 @@ import (
 	"github.com/anchore/syft/syft/source"
 )
 
-func parser(_ string, reader io.Reader) ([]pkg.Package, []artifact.Relationship, error) {
+func parser(_ string, reader io.Reader) ([]*pkg.Package, []artifact.Relationship, error) {
 	contents, err := ioutil.ReadAll(reader)
 	if err != nil {
 		panic(err)
 	}
-	return []pkg.Package{
+	return []*pkg.Package{
 		{
 			Name: string(contents),
 		},

syft/pkg/cataloger/common/parser.go

@@ -8,4 +8,4 @@ import (
 )
 
 // ParserFn standardizes a function signature for parser functions that accept the virtual file path (not usable for file reads) and contents and return any discovered packages from that file
-type ParserFn func(string, io.Reader) ([]pkg.Package, []artifact.Relationship, error)
+type ParserFn func(string, io.Reader) ([]*pkg.Package, []artifact.Relationship, error)

syft/pkg/cataloger/deb/cataloger.go

@@ -68,6 +68,8 @@ func (c *Cataloger) Catalog(resolver source.FileResolver) ([]pkg.Package, []arti
 
 			// fetch additional data from the copyright file to derive the license information
 			addLicenses(resolver, dbLocation, p)
+
+			p.SetID()
 		}
 
 		allPackages = append(allPackages, pkgs...)

syft/pkg/cataloger/deb/parse_dpkg_status.go

@@ -20,6 +20,16 @@ var (
 	sourceRegexp     = regexp.MustCompile(`(?P<name>\S+)( \((?P<version>.*)\))?`)
 )
 
+func newDpkgPackage(d pkg.DpkgMetadata) pkg.Package {
+	return pkg.Package{
+		Name:         d.Package,
+		Version:      d.Version,
+		Type:         pkg.DebPkg,
+		MetadataType: pkg.DpkgMetadataType,
+		Metadata:     d,
+	}
+}
+
 // parseDpkgStatus is a parser function for Debian DB status contents, returning all Debian packages listed.
 func parseDpkgStatus(reader io.Reader) ([]pkg.Package, error) {
 	buffedReader := bufio.NewReader(reader)
@@ -37,13 +47,7 @@ func parseDpkgStatus(reader io.Reader) ([]pkg.Package, error) {
 		}
 
 		if entry.Package != "" {
-			packages = append(packages, pkg.Package{
-				Name:         entry.Package,
-				Version:      entry.Version,
-				Type:         pkg.DebPkg,
-				MetadataType: pkg.DpkgMetadataType,
-				Metadata:     entry,
-			})
+			packages = append(packages, newDpkgPackage(entry))
 		}
 	}
 

syft/pkg/cataloger/golang/parse_go_bin.go

@@ -17,6 +17,28 @@ const (
 
 type exeOpener func(file io.ReadCloser) ([]exe, error)
 
+func newGoBinaryPackage(name, version, h1Digest, goVersion, architecture string, location source.Location) pkg.Package {
+	p := pkg.Package{
+		Name:     name,
+		Version:  version,
+		Language: pkg.Go,
+		Type:     pkg.GoModulePkg,
+		Locations: []source.Location{
+			location,
+		},
+		MetadataType: pkg.GolangBinMetadataType,
+		Metadata: pkg.GolangBinMetadata{
+			GoCompiledVersion: goVersion,
+			H1Digest:          h1Digest,
+			Architecture:      architecture,
+		},
+	}
+
+	p.SetID()
+
+	return p
+}
+
 func parseGoBin(location source.Location, reader io.ReadCloser, opener exeOpener) (pkgs []pkg.Package, err error) {
 	var exes []exe
 	// it has been found that there are stdlib paths within openExe that can panic. We want to prevent this behavior
@@ -54,21 +76,10 @@ func buildGoPkgInfo(location source.Location, mod, goVersion, arch string) []pkg
 		}
 
 		if fields[0] == packageIdentifier || fields[0] == replaceIdentifier {
-			pkgsSlice = append(pkgsSlice, pkg.Package{
-				Name:     fields[1],
-				Version:  fields[2],
-				Language: pkg.Go,
-				Type:     pkg.GoModulePkg,
-				Locations: []source.Location{
-					location,
-				},
-				MetadataType: pkg.GolangBinMetadataType,
-				Metadata: pkg.GolangBinMetadata{
-					GoCompiledVersion: goVersion,
-					H1Digest:          fields[3],
-					Architecture:      arch,
-				},
-			})
+			name := fields[1]
+			version := fields[2]
+			h1Digest := fields[3]
+			pkgsSlice = append(pkgsSlice, newGoBinaryPackage(name, version, h1Digest, goVersion, arch, location))
 		}
 	}
 

syft/pkg/cataloger/golang/parse_go_bin_test.go

@@ -147,6 +147,10 @@ func TestBuildGoPkgInfo(t *testing.T) {
 
 	for _, test := range tests {
 		t.Run(test.name, func(t *testing.T) {
+			for i := range test.expected {
+				p := &test.expected[i]
+				p.SetID()
+			}
 			location := source.Location{
 				Coordinates: source.Coordinates{
 					RealPath:     "/a-path",

syft/pkg/cataloger/golang/parse_go_mod.go

@@ -12,8 +12,8 @@ import (
 )
 
 // parseGoMod takes a go.mod and lists all packages discovered.
-func parseGoMod(path string, reader io.Reader) ([]pkg.Package, []artifact.Relationship, error) {
-	packages := make(map[string]pkg.Package)
+func parseGoMod(path string, reader io.Reader) ([]*pkg.Package, []artifact.Relationship, error) {
+	packages := make(map[string]*pkg.Package)
 
 	contents, err := ioutil.ReadAll(reader)
 	if err != nil {
@@ -26,7 +26,7 @@ func parseGoMod(path string, reader io.Reader) ([]pkg.Package, []artifact.Relati
 	}
 
 	for _, m := range file.Require {
-		packages[m.Mod.Path] = pkg.Package{
+		packages[m.Mod.Path] = &pkg.Package{
 			Name:     m.Mod.Path,
 			Version:  m.Mod.Version,
 			Language: pkg.Go,
@@ -36,7 +36,7 @@ func parseGoMod(path string, reader io.Reader) ([]pkg.Package, []artifact.Relati
 
 	// remove any old packages and replace with new ones...
 	for _, m := range file.Replace {
-		packages[m.New.Path] = pkg.Package{
+		packages[m.New.Path] = &pkg.Package{
 			Name:     m.New.Path,
 			Version:  m.New.Version,
 			Language: pkg.Go,
@@ -49,7 +49,7 @@ func parseGoMod(path string, reader io.Reader) ([]pkg.Package, []artifact.Relati
 		delete(packages, m.Mod.Path)
 	}
 
-	pkgsSlice := make([]pkg.Package, len(packages))
+	pkgsSlice := make([]*pkg.Package, len(packages))
 	idx := 0
 	for _, p := range packages {
 		pkgsSlice[idx] = p

syft/pkg/cataloger/golang/parse_go_mod_test.go

@@ -87,7 +87,7 @@ func TestParseGoMod(t *testing.T) {
 					continue
 				}
 
-				diffs := deep.Equal(a, e)
+				diffs := deep.Equal(a, &e)
 				if len(diffs) > 0 {
 					t.Errorf("diffs found for %q", a.Name)
 					for _, d := range diffs {

syft/pkg/cataloger/java/archive_parser.go

@@ -35,7 +35,7 @@ type archiveParser struct {
 }
 
 // parseJavaArchive is a parser function for java archive contents, returning all Java libraries and nested archives.
-func parseJavaArchive(virtualPath string, reader io.Reader) ([]pkg.Package, []artifact.Relationship, error) {
+func parseJavaArchive(virtualPath string, reader io.Reader) ([]*pkg.Package, []artifact.Relationship, error) {
 	parser, cleanupFn, err := newJavaArchiveParser(virtualPath, reader, true)
 	// note: even on error, we should always run cleanup functions
 	defer cleanupFn()
@@ -81,8 +81,8 @@ func newJavaArchiveParser(virtualPath string, reader io.Reader, detectNested boo
 }
 
 // parse the loaded archive and return all packages found.
-func (j *archiveParser) parse() ([]pkg.Package, []artifact.Relationship, error) {
-	var pkgs []pkg.Package
+func (j *archiveParser) parse() ([]*pkg.Package, []artifact.Relationship, error) {
+	var pkgs []*pkg.Package
 	var relationships []artifact.Relationship
 
 	// find the parent package from the java manifest
@@ -110,7 +110,7 @@ func (j *archiveParser) parse() ([]pkg.Package, []artifact.Relationship, error)
 
 	// lastly, add the parent package to the list (assuming the parent exists)
 	if parentPkg != nil {
-		pkgs = append([]pkg.Package{*parentPkg}, pkgs...)
+		pkgs = append([]*pkg.Package{parentPkg}, pkgs...)
 	}
 
 	return pkgs, relationships, nil
@@ -158,12 +158,12 @@ func (j *archiveParser) discoverMainPackage() (*pkg.Package, error) {
 // parent package, returning all listed Java packages found for each pom
 // properties discovered and potentially updating the given parentPkg with new
 // data.
-func (j *archiveParser) discoverPkgsFromAllMavenFiles(parentPkg *pkg.Package) ([]pkg.Package, error) {
+func (j *archiveParser) discoverPkgsFromAllMavenFiles(parentPkg *pkg.Package) ([]*pkg.Package, error) {
 	if parentPkg == nil {
 		return nil, nil
 	}
 
-	var pkgs []pkg.Package
+	var pkgs []*pkg.Package
 
 	properties, err := pomPropertiesByParentPath(j.archivePath, j.fileManifest.GlobMatch(pomPropertiesGlob), j.virtualPath)
 	if err != nil {
@@ -183,7 +183,7 @@ func (j *archiveParser) discoverPkgsFromAllMavenFiles(parentPkg *pkg.Package) ([
 
 		pkgFromPom := newPackageFromMavenData(propertiesObj, pomProject, parentPkg, j.virtualPath)
 		if pkgFromPom != nil {
-			pkgs = append(pkgs, *pkgFromPom)
+			pkgs = append(pkgs, pkgFromPom)
 		}
 	}
 
@@ -192,8 +192,8 @@ func (j *archiveParser) discoverPkgsFromAllMavenFiles(parentPkg *pkg.Package) ([
 
 // discoverPkgsFromNestedArchives finds Java archives within Java archives, returning all listed Java packages found and
 // associating each discovered package to the given parent package.
-func (j *archiveParser) discoverPkgsFromNestedArchives(parentPkg *pkg.Package) ([]pkg.Package, []artifact.Relationship, error) {
-	var pkgs []pkg.Package
+func (j *archiveParser) discoverPkgsFromNestedArchives(parentPkg *pkg.Package) ([]*pkg.Package, []artifact.Relationship, error) {
+	var pkgs []*pkg.Package
 	var relationships []artifact.Relationship
 
 	// search and parse pom.properties files & fetch the contents

syft/pkg/cataloger/java/archive_parser_test.go

@@ -257,7 +257,7 @@ func TestParseJar(t *testing.T) {
 			var parent *pkg.Package
 			for _, a := range actual {
 				if strings.Contains(a.Name, "example-") {
-					parent = &a
+					parent = a
 				}
 			}
 
@@ -292,7 +292,7 @@ func TestParseJar(t *testing.T) {
 				// write censored data back
 				a.Metadata = metadata
 
-				diffs := deep.Equal(e, a)
+				diffs := deep.Equal(&e, a)
 				if len(diffs) > 0 {
 					t.Errorf("diffs found for %q", a.Name)
 					for _, d := range diffs {
@@ -527,7 +527,7 @@ func TestParseNestedJar(t *testing.T) {
 
 			actualNameVersionPairSet := internal.NewStringSet()
 			for _, a := range actual {
-				key := makeKey(&a)
+				key := makeKey(a)
 				actualNameVersionPairSet.Add(key)
 				if !expectedNameVersionPairSet.Contains(key) {
 					t.Errorf("extra package: %s", a)
@@ -545,7 +545,7 @@ func TestParseNestedJar(t *testing.T) {
 			}
 
 			for _, a := range actual {
-				actualKey := makeKey(&a)
+				actualKey := makeKey(a)
 
 				metadata := a.Metadata.(pkg.JavaMetadata)
 				if actualKey == "spring-boot|0.0.1-SNAPSHOT" {

syft/pkg/cataloger/javascript/parse_package_json.go

@@ -50,7 +50,6 @@ type Repository struct {
 // ---> name: "Isaac Z. Schlueter" email: "i@izs.me" url: "http://blog.izs.me"
 var authorPattern = regexp.MustCompile(`^\s*(?P<name>[^<(]*)(\s+<(?P<email>.*)>)?(\s\((?P<url>.*)\))?\s*$`)
 
-// Exports Author.UnmarshalJSON interface to help normalize the json structure.
 func (a *Author) UnmarshalJSON(b []byte) error {
 	var authorStr string
 	var fields map[string]string
@@ -135,7 +134,7 @@ func licenseFromJSON(b []byte) (string, error) {
 	return "", errors.New("unable to unmarshal license field as either string or object")
 }
 
-func licensesFromJSON(p PackageJSON) ([]string, error) {
+func (p PackageJSON) licensesFromJSON() ([]string, error) {
 	if p.License == nil && p.Licenses == nil {
 		// This package.json doesn't specify any licenses whatsoever
 		return []string{}, nil
@@ -163,8 +162,8 @@ func licensesFromJSON(p PackageJSON) ([]string, error) {
 }
 
 // parsePackageJSON parses a package.json and returns the discovered JavaScript packages.
-func parsePackageJSON(_ string, reader io.Reader) ([]pkg.Package, []artifact.Relationship, error) {
-	var packages []pkg.Package
+func parsePackageJSON(_ string, reader io.Reader) ([]*pkg.Package, []artifact.Relationship, error) {
+	var packages []*pkg.Package
 	dec := json.NewDecoder(reader)
 
 	for {
@@ -180,30 +179,34 @@ func parsePackageJSON(_ string, reader io.Reader) ([]pkg.Package, []artifact.Rel
 			return nil, nil, nil
 		}
 
-		licenses, err := licensesFromJSON(p)
-		if err != nil {
-			return nil, nil, fmt.Errorf("failed to parse package.json file: %w", err)
-		}
-
-		packages = append(packages, pkg.Package{
-			Name:         p.Name,
-			Version:      p.Version,
-			Licenses:     licenses,
-			Language:     pkg.JavaScript,
-			Type:         pkg.NpmPkg,
-			MetadataType: pkg.NpmPackageJSONMetadataType,
-			Metadata: pkg.NpmPackageJSONMetadata{
-				Author:   p.Author.AuthorString(),
-				Homepage: p.Homepage,
-				URL:      p.Repository.URL,
-				Licenses: licenses,
-			},
-		})
+		packages = append(packages, newPackageJSONPackage(p))
 	}
 
 	return packages, nil, nil
 }
 
+func newPackageJSONPackage(p PackageJSON) *pkg.Package {
+	licenses, err := p.licensesFromJSON()
+	if err != nil {
+		log.Warnf("unable to extract licenses from javascript package.json: %+v", err)
+	}
+
+	return &pkg.Package{
+		Name:         p.Name,
+		Version:      p.Version,
+		Licenses:     licenses,
+		Language:     pkg.JavaScript,
+		Type:         pkg.NpmPkg,
+		MetadataType: pkg.NpmPackageJSONMetadataType,
+		Metadata: pkg.NpmPackageJSONMetadata{
+			Author:   p.Author.AuthorString(),
+			Homepage: p.Homepage,
+			URL:      p.Repository.URL,
+			Licenses: licenses,
+		},
+	}
+}
+
 func (p PackageJSON) hasNameAndVersionValues() bool {
 	return p.Name != "" && p.Version != ""
 }

syft/pkg/cataloger/javascript/parse_package_json_test.go

@@ -136,7 +136,7 @@ func TestParsePackageJSON(t *testing.T) {
 				t.Fatalf("unexpected package count: %d!=1", len(actual))
 			}
 
-			for _, d := range deep.Equal(actual[0], test.ExpectedPkg) {
+			for _, d := range deep.Equal(actual[0], &test.ExpectedPkg) {
 
 				t.Errorf("diff: %+v", d)
 			}

syft/pkg/cataloger/javascript/parse_package_lock.go

@@ -29,14 +29,14 @@ type Dependency struct {
 }
 
 // parsePackageLock parses a package-lock.json and returns the discovered JavaScript packages.
-func parsePackageLock(path string, reader io.Reader) ([]pkg.Package, []artifact.Relationship, error) {
+func parsePackageLock(path string, reader io.Reader) ([]*pkg.Package, []artifact.Relationship, error) {
 	// in the case we find package-lock.json files in the node_modules directories, skip those
 	// as the whole purpose of the lock file is for the specific dependencies of the root project
 	if pathContainsNodeModulesDirectory(path) {
 		return nil, nil, nil
 	}
 
-	var packages []pkg.Package
+	var packages []*pkg.Package
 	dec := json.NewDecoder(reader)
 
 	for {
@@ -47,7 +47,7 @@ func parsePackageLock(path string, reader io.Reader) ([]pkg.Package, []artifact.
 			return nil, nil, fmt.Errorf("failed to parse package-lock.json file: %w", err)
 		}
 		for name, pkgMeta := range lock.Dependencies {
-			packages = append(packages, pkg.Package{
+			packages = append(packages, &pkg.Package{
 				Name:     name,
 				Version:  pkgMeta.Version,
 				Language: pkg.JavaScript,

syft/pkg/cataloger/javascript/parse_package_lock_test.go

@@ -4,10 +4,12 @@ import (
 	"os"
 	"testing"
 
+	"github.com/stretchr/testify/assert"
+
 	"github.com/anchore/syft/syft/pkg"
 )
 
-func assertPkgsEqual(t *testing.T, actual []pkg.Package, expected map[string]pkg.Package) {
+func assertPkgsEqual(t *testing.T, actual []*pkg.Package, expected map[string]pkg.Package) {
 	t.Helper()
 	if len(actual) != len(expected) {
 		for _, a := range actual {
@@ -18,26 +20,11 @@ func assertPkgsEqual(t *testing.T, actual []pkg.Package, expected map[string]pkg
 
 	for _, a := range actual {
 		expectedPkg, ok := expected[a.Name]
-		if !ok {
-			t.Errorf("unexpected package found: '%s'", a.Name)
-		}
-
-		if expectedPkg.Version != a.Version {
-			t.Errorf("%s : unexpected package version: '%s', expected: '%s'", a.Name, a.Version, expectedPkg.Version)
-		}
-
-		if a.Language != expectedPkg.Language {
-			t.Errorf("%s : bad language: '%+v', expected: '%+v'", a.Name, a.Language, expectedPkg.Language)
-		}
-
-		if a.Type != expectedPkg.Type {
-			t.Errorf("%s : bad package type: %+v, expected: %+v", a.Name, a.Type, expectedPkg.Type)
-		}
-
-		if len(a.Licenses) < len(expectedPkg.Licenses) {
-			t.Errorf("%s : bad package licenses count: '%+v'", a.Name, a.Licenses)
-		}
-
+		assert.True(t, ok)
+		assert.Equal(t, expectedPkg.Version, a.Version, "bad version")
+		assert.Equal(t, expectedPkg.Language, a.Language, "bad language")
+		assert.Equal(t, expectedPkg.Type, a.Type, "bad type")
+		assert.Equal(t, expectedPkg.Licenses, a.Licenses, "bad license count")
 	}
 }
 

syft/pkg/cataloger/javascript/parse_yarn_lock.go

@@ -35,14 +35,14 @@ const (
 	noVersion = ""
 )
 
-func parseYarnLock(path string, reader io.Reader) ([]pkg.Package, []artifact.Relationship, error) {
+func parseYarnLock(path string, reader io.Reader) ([]*pkg.Package, []artifact.Relationship, error) {
 	// in the case we find yarn.lock files in the node_modules directories, skip those
 	// as the whole purpose of the lock file is for the specific dependencies of the project
 	if pathContainsNodeModulesDirectory(path) {
 		return nil, nil, nil
 	}
 
-	var packages []pkg.Package
+	var packages []*pkg.Package
 	scanner := bufio.NewScanner(reader)
 	parsedPackages := internal.NewStringSet()
 	currentPackage := noPackage
@@ -106,8 +106,8 @@ func findPackageVersion(line string) string {
 	return noVersion
 }
 
-func newYarnLockPackage(name, version string) pkg.Package {
-	return pkg.Package{
+func newYarnLockPackage(name, version string) *pkg.Package {
+	return &pkg.Package{
 		Name:     name,
 		Version:  version,
 		Language: pkg.JavaScript,

syft/pkg/cataloger/package_url_test.go

@@ -10,11 +10,13 @@ import (
 
 func TestPackageURL(t *testing.T) {
 	tests := []struct {
+		name     string
 		pkg      pkg.Package
 		distro   *distro.Distro
 		expected string
 	}{
 		{
+			name: "golang",
 			pkg: pkg.Package{
 				Name:    "github.com/anchore/syft",
 				Version: "v0.1.0",
@@ -23,14 +25,38 @@ func TestPackageURL(t *testing.T) {
 			expected: "pkg:golang/github.com/anchore/syft@v0.1.0",
 		},
 		{
+			name: "pip with vcs url",
 			pkg: pkg.Package{
-				Name:    "name",
-				Version: "v0.1.0",
+				Name:    "bad-name",
+				Version: "bad-v0.1.0",
 				Type:    pkg.PythonPkg,
+				Metadata: pkg.PythonPackageMetadata{
+					Name:    "name",
+					Version: "v0.1.0",
+					DirectURLOrigin: &pkg.PythonDirectURLOriginInfo{
+						VCS:      "git",
+						URL:      "https://github.com/test/test.git",
+						CommitID: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
+					},
+				},
+			},
+			expected: "pkg:pypi/name@v0.1.0?vcs_url=git+https:%2F%2Fgithub.com%2Ftest%2Ftest.git@aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
+		},
+		{
+			name: "pip without vcs url",
+			pkg: pkg.Package{
+				Name:    "bad-name",
+				Version: "bad-v0.1.0",
+				Type:    pkg.PythonPkg,
+				Metadata: pkg.PythonPackageMetadata{
+					Name:    "name",
+					Version: "v0.1.0",
+				},
 			},
 			expected: "pkg:pypi/name@v0.1.0",
 		},
 		{
+			name: "gem",
 			pkg: pkg.Package{
 				Name:    "name",
 				Version: "v0.1.0",
@@ -39,6 +65,7 @@ func TestPackageURL(t *testing.T) {
 			expected: "pkg:gem/name@v0.1.0",
 		},
 		{
+			name: "npm",
 			pkg: pkg.Package{
 				Name:    "name",
 				Version: "v0.1.0",
@@ -47,6 +74,7 @@ func TestPackageURL(t *testing.T) {
 			expected: "pkg:npm/name@v0.1.0",
 		},
 		{
+			name: "deb with arch",
 			distro: &distro.Distro{
 				Type: distro.Ubuntu,
 			},
@@ -63,6 +91,7 @@ func TestPackageURL(t *testing.T) {
 			expected: "pkg:deb/ubuntu/name@v0.1.0?arch=amd64",
 		},
 		{
+			name: "deb with epoch",
 			distro: &distro.Distro{
 				Type: distro.CentOS,
 			},
@@ -81,6 +110,7 @@ func TestPackageURL(t *testing.T) {
 			expected: "pkg:rpm/centos/name@0.1.0-3?arch=amd64&epoch=2",
 		},
 		{
+			name: "deb with nil epoch",
 			distro: &distro.Distro{
 				Type: distro.CentOS,
 			},
@@ -99,6 +129,7 @@ func TestPackageURL(t *testing.T) {
 			expected: "pkg:rpm/centos/name@0.1.0-3?arch=amd64",
 		},
 		{
+			name: "deb with unknown distro",
 			distro: &distro.Distro{
 				Type: distro.UnknownDistroType,
 			},
@@ -110,6 +141,7 @@ func TestPackageURL(t *testing.T) {
 			expected: "pkg:deb/name@v0.1.0",
 		},
 		{
+			name: "cargo",
 			pkg: pkg.Package{
 				Name:    "name",
 				Version: "v0.1.0",
@@ -120,7 +152,7 @@ func TestPackageURL(t *testing.T) {
 	}
 
 	for _, test := range tests {
-		t.Run(string(test.pkg.Type)+"|"+test.expected, func(t *testing.T) {
+		t.Run(test.name, func(t *testing.T) {
 			actual := generatePackageURL(test.pkg, test.distro)
 			if actual != test.expected {
 				dmp := diffmatchpatch.New()

syft/pkg/cataloger/php/parse_composer_lock.go

@@ -8,7 +8,6 @@ import (
 	"github.com/anchore/syft/syft/artifact"
 
 	"github.com/anchore/syft/syft/pkg"
-	"github.com/anchore/syft/syft/pkg/cataloger/common"
 )
 
 type ComposerLock struct {
@@ -21,12 +20,9 @@ type Dependency struct {
 	Version string `json:"version"`
 }
 
-// integrity check
-var _ common.ParserFn = parseComposerLock
-
 // parseComposerLock is a parser function for Composer.lock contents, returning "Default" php packages discovered.
-func parseComposerLock(_ string, reader io.Reader) ([]pkg.Package, []artifact.Relationship, error) {
-	packages := make([]pkg.Package, 0)
+func parseComposerLock(_ string, reader io.Reader) ([]*pkg.Package, []artifact.Relationship, error) {
+	packages := make([]*pkg.Package, 0)
 	dec := json.NewDecoder(reader)
 
 	for {
@@ -39,7 +35,7 @@ func parseComposerLock(_ string, reader io.Reader) ([]pkg.Package, []artifact.Re
 		for _, pkgMeta := range lock.Packages {
 			version := pkgMeta.Version
 			name := pkgMeta.Name
-			packages = append(packages, pkg.Package{
+			packages = append(packages, &pkg.Package{
 				Name:     name,
 				Version:  version,
 				Language: pkg.PHP,

syft/pkg/cataloger/php/parse_composer_lock_test.go

@@ -9,7 +9,7 @@ import (
 )
 
 func TestParseComposerFileLock(t *testing.T) {
-	expected := []pkg.Package{
+	expected := []*pkg.Package{
 		{
 			Name:     "adoy/fastcgi-client",
 			Version:  "1.0.2",

syft/pkg/cataloger/php/parse_installed_json.go

@@ -40,8 +40,8 @@ func (w *installedJSONComposerV2) UnmarshalJSON(data []byte) error {
 var _ common.ParserFn = parseComposerLock
 
 // parseComposerLock is a parser function for Composer.lock contents, returning "Default" php packages discovered.
-func parseInstalledJSON(_ string, reader io.Reader) ([]pkg.Package, []artifact.Relationship, error) {
-	packages := make([]pkg.Package, 0)
+func parseInstalledJSON(_ string, reader io.Reader) ([]*pkg.Package, []artifact.Relationship, error) {
+	packages := make([]*pkg.Package, 0)
 	dec := json.NewDecoder(reader)
 
 	for {
@@ -54,7 +54,7 @@ func parseInstalledJSON(_ string, reader io.Reader) ([]pkg.Package, []artifact.R
 		for _, pkgMeta := range lock.Packages {
 			version := pkgMeta.Version
 			name := pkgMeta.Name
-			packages = append(packages, pkg.Package{
+			packages = append(packages, &pkg.Package{
 				Name:     name,
 				Version:  version,
 				Language: pkg.PHP,

syft/pkg/cataloger/php/parse_installed_json_test.go

@@ -9,7 +9,7 @@ import (
 )
 
 func TestParseInstalledJsonComposerV1(t *testing.T) {
-	expected := []pkg.Package{
+	expected := []*pkg.Package{
 		{
 			Name:     "asm89/stack-cors",
 			Version:  "1.3.0",
@@ -41,7 +41,7 @@ func TestParseInstalledJsonComposerV1(t *testing.T) {
 }
 
 func TestParseInstalledJsonComposerV2(t *testing.T) {
-	expected := []pkg.Package{
+	expected := []*pkg.Package{
 		{
 			Name:     "asm89/stack-cors",
 			Version:  "1.3.0",

syft/pkg/cataloger/python/package_cataloger.go

@@ -2,7 +2,9 @@ package python
 
 import (
 	"bufio"
+	"encoding/json"
 	"fmt"
+	"io/ioutil"
 	"path/filepath"
 
 	"github.com/anchore/syft/internal"
@@ -74,7 +76,7 @@ func (c *PackageCataloger) catalogEggOrWheel(resolver source.FileResolver, metad
 		licenses = []string{metadata.License}
 	}
 
-	return &pkg.Package{
+	p := &pkg.Package{
 		Name:         metadata.Name,
 		Version:      metadata.Version,
 		FoundBy:      c.Name(),
@@ -84,7 +86,11 @@ func (c *PackageCataloger) catalogEggOrWheel(resolver source.FileResolver, metad
 		Type:         pkg.PythonPkg,
 		MetadataType: pkg.PythonPackageMetadataType,
 		Metadata:     *metadata,
-	}, nil
+	}
+
+	p.SetID()
+
+	return p, nil
 }
 
 // fetchRecordFiles finds a corresponding RECORD file for the given python package metadata file and returns the set of file records contained.
@@ -148,6 +154,40 @@ func (c *PackageCataloger) fetchTopLevelPackages(resolver source.FileResolver, m
 	return pkgs, sources, nil
 }
 
+func (c *PackageCataloger) fetchDirectURLData(resolver source.FileResolver, metadataLocation source.Location) (d *pkg.PythonDirectURLOriginInfo, sources []source.Location, err error) {
+	parentDir := filepath.Dir(metadataLocation.RealPath)
+	directURLPath := filepath.Join(parentDir, "direct_url.json")
+	directURLLocation := resolver.RelativeFileByPath(metadataLocation, directURLPath)
+
+	if directURLLocation == nil {
+		return nil, nil, nil
+	}
+
+	sources = append(sources, *directURLLocation)
+
+	directURLContents, err := resolver.FileContentsByLocation(*directURLLocation)
+	if err != nil {
+		return nil, nil, err
+	}
+	defer internal.CloseAndLogError(directURLContents, directURLLocation.VirtualPath)
+
+	buffer, err := ioutil.ReadAll(directURLContents)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	var directURLJson pkg.DirectURLOrigin
+	if err := json.Unmarshal(buffer, &directURLJson); err != nil {
+		return nil, nil, err
+	}
+
+	return &pkg.PythonDirectURLOriginInfo{
+		URL:      directURLJson.URL,
+		CommitID: directURLJson.VCSInfo.CommitID,
+		VCS:      directURLJson.VCSInfo.VCS,
+	}, sources, nil
+}
+
 // assembleEggOrWheelMetadata discovers and accumulates python package metadata from multiple file sources and returns a single metadata object as well as a list of files where the metadata was derived from.
 func (c *PackageCataloger) assembleEggOrWheelMetadata(resolver source.FileResolver, metadataLocation source.Location) (*pkg.PythonPackageMetadata, []source.Location, error) {
 	var sources = []source.Location{metadataLocation}
@@ -179,5 +219,13 @@ func (c *PackageCataloger) assembleEggOrWheelMetadata(resolver source.FileResolv
 	sources = append(sources, s...)
 	metadata.TopLevelPackages = p
 
+	// attach any direct-url package data found for the given wheel/egg installation
+	d, s, err := c.fetchDirectURLData(resolver, metadataLocation)
+	if err != nil {
+		return nil, nil, err
+	}
+	sources = append(sources, s...)
+	metadata.DirectURLOrigin = d
+
 	return &metadata, sources, nil
 }

syft/pkg/cataloger/python/package_cataloger_test.go

@@ -55,6 +55,7 @@ func TestPythonPackageWheelCataloger(t *testing.T) {
 				"test-fixtures/dist-info/METADATA",
 				"test-fixtures/dist-info/RECORD",
 				"test-fixtures/dist-info/top_level.txt",
+				"test-fixtures/dist-info/direct_url.json",
 			},
 			expectedPackage: pkg.Package{
 				Name:         "Pygments",
@@ -82,6 +83,7 @@ func TestPythonPackageWheelCataloger(t *testing.T) {
 						{Path: "pygments/x_util.py", Digest: &pkg.PythonFileDigest{"sha256", "qpzzsOW31KT955agi-7NS--90I0iNiJCyLJQnRCHgKI="}, Size: "10778"},
 					},
 					TopLevelPackages: []string{"pygments", "something_else"},
+					DirectURLOrigin:  &pkg.PythonDirectURLOriginInfo{URL: "https://github.com/python-test/test.git", VCS: "git", CommitID: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"},
 				},
 			},
 		},

syft/pkg/cataloger/python/parse_pipfile_lock.go

@@ -38,8 +38,8 @@ type Dependency struct {
 var _ common.ParserFn = parsePipfileLock
 
 // parsePipfileLock is a parser function for Pipfile.lock contents, returning "Default" python packages discovered.
-func parsePipfileLock(_ string, reader io.Reader) ([]pkg.Package, []artifact.Relationship, error) {
-	packages := make([]pkg.Package, 0)
+func parsePipfileLock(_ string, reader io.Reader) ([]*pkg.Package, []artifact.Relationship, error) {
+	packages := make([]*pkg.Package, 0)
 	dec := json.NewDecoder(reader)
 
 	for {
@@ -51,7 +51,7 @@ func parsePipfileLock(_ string, reader io.Reader) ([]pkg.Package, []artifact.Rel
 		}
 		for name, pkgMeta := range lock.Default {
 			version := strings.TrimPrefix(pkgMeta.Version, "==")
-			packages = append(packages, pkg.Package{
+			packages = append(packages, &pkg.Package{
 				Name:     name,
 				Version:  version,
 				Language: pkg.Python,

syft/pkg/cataloger/python/parse_poetry_lock.go

@@ -14,7 +14,7 @@ import (
 var _ common.ParserFn = parsePoetryLock
 
 // parsePoetryLock is a parser function for poetry.lock contents, returning all python packages discovered.
-func parsePoetryLock(_ string, reader io.Reader) ([]pkg.Package, []artifact.Relationship, error) {
+func parsePoetryLock(_ string, reader io.Reader) ([]*pkg.Package, []artifact.Relationship, error) {
 	tree, err := toml.LoadReader(reader)
 	if err != nil {
 		return nil, nil, fmt.Errorf("unable to load poetry.lock for parsing: %v", err)

syft/pkg/cataloger/python/parse_poetry_lock_test.go

@@ -9,7 +9,7 @@ import (
 )
 
 func TestParsePoetryLock(t *testing.T) {
-	expected := []pkg.Package{
+	expected := []*pkg.Package{
 		{
 			Name:     "added-value",
 			Version:  "0.14.2",

syft/pkg/cataloger/python/parse_requirements.go

@@ -16,8 +16,8 @@ var _ common.ParserFn = parseRequirementsTxt
 
 // parseRequirementsTxt takes a Python requirements.txt file, returning all Python packages that are locked to a
 // specific version.
-func parseRequirementsTxt(_ string, reader io.Reader) ([]pkg.Package, []artifact.Relationship, error) {
-	packages := make([]pkg.Package, 0)
+func parseRequirementsTxt(_ string, reader io.Reader) ([]*pkg.Package, []artifact.Relationship, error) {
+	packages := make([]*pkg.Package, 0)
 
 	scanner := bufio.NewScanner(reader)
 	for scanner.Scan() {
@@ -44,7 +44,7 @@ func parseRequirementsTxt(_ string, reader io.Reader) ([]pkg.Package, []artifact
 			parts := strings.Split(uncommented, "==")
 			name := strings.TrimSpace(parts[0])
 			version := strings.TrimSpace(parts[1])
-			packages = append(packages, pkg.Package{
+			packages = append(packages, &pkg.Package{
 				Name:     name,
 				Version:  version,
 				Language: pkg.Python,

syft/pkg/cataloger/python/parse_requirements_test.go

@@ -4,12 +4,14 @@ import (
 	"os"
 	"testing"
 
+	"github.com/stretchr/testify/assert"
+
 	"github.com/go-test/deep"
 
 	"github.com/anchore/syft/syft/pkg"
 )
 
-func assertPackagesEqual(t *testing.T, actual []pkg.Package, expected map[string]pkg.Package) {
+func assertPackagesEqual(t *testing.T, actual []*pkg.Package, expected map[string]pkg.Package) {
 	t.Helper()
 	if len(actual) != len(expected) {
 		for _, a := range actual {
@@ -20,11 +22,9 @@ func assertPackagesEqual(t *testing.T, actual []pkg.Package, expected map[string
 
 	for _, a := range actual {
 		expectedPkg, ok := expected[a.Name]
-		if !ok {
-			t.Errorf("unexpected package found: '%s'", a.Name)
-		}
+		assert.True(t, ok)
 
-		for _, d := range deep.Equal(a, expectedPkg) {
+		for _, d := range deep.Equal(a, &expectedPkg) {
 			t.Errorf("diff: %+v", d)
 		}
 	}

syft/pkg/cataloger/python/parse_setup.go

@@ -20,8 +20,8 @@ var _ common.ParserFn = parseSetup
 //		" mypy2 == v0.770", ' mypy3== v0.770',  --> match(name=mypy2 version=v0.770), match(name=mypy3, version=v0.770)
 var pinnedDependency = regexp.MustCompile(`['"]\W?(\w+\W?==\W?[\w\.]*)`)
 
-func parseSetup(_ string, reader io.Reader) ([]pkg.Package, []artifact.Relationship, error) {
-	packages := make([]pkg.Package, 0)
+func parseSetup(_ string, reader io.Reader) ([]*pkg.Package, []artifact.Relationship, error) {
+	packages := make([]*pkg.Package, 0)
 
 	scanner := bufio.NewScanner(reader)
 
@@ -38,7 +38,7 @@ func parseSetup(_ string, reader io.Reader) ([]pkg.Package, []artifact.Relations
 			name = strings.TrimSpace(name)
 
 			version := strings.TrimSpace(parts[len(parts)-1])
-			packages = append(packages, pkg.Package{
+			packages = append(packages, &pkg.Package{
 				Name:     strings.Trim(name, "'\""),
 				Version:  strings.Trim(version, "'\""),
 				Language: pkg.Python,

syft/pkg/cataloger/python/poetry_metadata.go

@@ -7,8 +7,8 @@ type PoetryMetadata struct {
 }
 
 // Pkgs returns all of the packages referenced within the poetry.lock metadata.
-func (m PoetryMetadata) Pkgs() []pkg.Package {
-	pkgs := make([]pkg.Package, 0)
+func (m PoetryMetadata) Pkgs() []*pkg.Package {
+	pkgs := make([]*pkg.Package, 0)
 
 	for _, p := range m.Packages {
 		pkgs = append(pkgs, p.Pkg())

syft/pkg/cataloger/python/poetry_metadata_package.go

@@ -11,8 +11,8 @@ type PoetryMetadataPackage struct {
 }
 
 // Pkg returns the standard `pkg.Package` representation of the package referenced within the poetry.lock metadata.
-func (p PoetryMetadataPackage) Pkg() pkg.Package {
-	return pkg.Package{
+func (p PoetryMetadataPackage) Pkg() *pkg.Package {
+	return &pkg.Package{
 		Name:     p.Name,
 		Version:  p.Version,
 		Language: pkg.Python,

syft/pkg/cataloger/python/test-fixtures/dist-info/direct_url.json

@@ -0,0 +1 @@
+{"url": "https://github.com/python-test/test.git", "vcs_info": {"commit_id": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", "vcs": "git"}}

syft/pkg/cataloger/rpmdb/parse_rpmdb.go

@@ -70,6 +70,8 @@ func parseRpmDB(resolver source.FilePathResolver, dbLocation source.Location, re
 			Metadata:     metadata,
 		}
 
+		p.SetID()
+
 		allPkgs = append(allPkgs, p)
 	}
 

syft/pkg/cataloger/ruby/catalogers.go

@@ -1,5 +1,5 @@
 /*
-Package bundler provides a concrete Cataloger implementation for Ruby Gemfile.lock bundler files.
+Package ruby bundler provides a concrete Cataloger implementation for Ruby Gemfile.lock bundler files.
 */
 package ruby
 

syft/pkg/cataloger/ruby/parse_gemfile_lock.go

@@ -17,8 +17,8 @@ var _ common.ParserFn = parseGemFileLockEntries
 var sectionsOfInterest = internal.NewStringSetFromSlice([]string{"GEM"})
 
 // parseGemFileLockEntries is a parser function for Gemfile.lock contents, returning all Gems discovered.
-func parseGemFileLockEntries(_ string, reader io.Reader) ([]pkg.Package, []artifact.Relationship, error) {
-	pkgs := make([]pkg.Package, 0)
+func parseGemFileLockEntries(_ string, reader io.Reader) ([]*pkg.Package, []artifact.Relationship, error) {
+	pkgs := make([]*pkg.Package, 0)
 	scanner := bufio.NewScanner(reader)
 
 	var currentSection string
@@ -41,7 +41,7 @@ func parseGemFileLockEntries(_ string, reader io.Reader) ([]pkg.Package, []artif
 			if len(candidate) != 2 {
 				continue
 			}
-			pkgs = append(pkgs, pkg.Package{
+			pkgs = append(pkgs, &pkg.Package{
 				Name:     candidate[0],
 				Version:  strings.Trim(candidate[1], "()"),
 				Language: pkg.Ruby,

syft/pkg/cataloger/ruby/parse_gemspec.go

@@ -61,8 +61,8 @@ func processList(s string) []string {
 	return results
 }
 
-func parseGemSpecEntries(_ string, reader io.Reader) ([]pkg.Package, []artifact.Relationship, error) {
-	var pkgs []pkg.Package
+func parseGemSpecEntries(_ string, reader io.Reader) ([]*pkg.Package, []artifact.Relationship, error) {
+	var pkgs []*pkg.Package
 	var fields = make(map[string]interface{})
 	scanner := bufio.NewScanner(reader)
 
@@ -97,7 +97,7 @@ func parseGemSpecEntries(_ string, reader io.Reader) ([]pkg.Package, []artifact.
 			return nil, nil, fmt.Errorf("unable to decode gem metadata: %w", err)
 		}
 
-		pkgs = append(pkgs, pkg.Package{
+		pkgs = append(pkgs, &pkg.Package{
 			Name:         metadata.Name,
 			Version:      metadata.Version,
 			Licenses:     metadata.Licenses,

syft/pkg/cataloger/ruby/parse_gemspec_test.go

@@ -44,7 +44,7 @@ func TestParseGemspec(t *testing.T) {
 		t.Fatalf("unexpected package count: %d!=1", len(actual))
 	}
 
-	for _, d := range deep.Equal(actual[0], expectedPkg) {
+	for _, d := range deep.Equal(actual[0], &expectedPkg) {
 		t.Errorf("diff: %+v", d)
 	}
 }

syft/pkg/cataloger/rust/cargo_metadata.go

@@ -7,8 +7,8 @@ type CargoMetadata struct {
 }
 
 // Pkgs returns all of the packages referenced within the Cargo.lock metadata.
-func (m CargoMetadata) Pkgs() []pkg.Package {
-	pkgs := make([]pkg.Package, 0)
+func (m CargoMetadata) Pkgs() []*pkg.Package {
+	pkgs := make([]*pkg.Package, 0)
 
 	for _, p := range m.Packages {
 		if p.Dependencies == nil {

syft/pkg/cataloger/rust/parse_cargo_lock.go

@@ -14,7 +14,7 @@ import (
 var _ common.ParserFn = parseCargoLock
 
 // parseCargoLock is a parser function for Cargo.lock contents, returning all rust cargo crates discovered.
-func parseCargoLock(_ string, reader io.Reader) ([]pkg.Package, []artifact.Relationship, error) {
+func parseCargoLock(_ string, reader io.Reader) ([]*pkg.Package, []artifact.Relationship, error) {
 	tree, err := toml.LoadReader(reader)
 	if err != nil {
 		return nil, nil, fmt.Errorf("unable to load Cargo.lock for parsing: %v", err)

syft/pkg/cataloger/rust/parse_cargo_lock_test.go

@@ -9,7 +9,7 @@ import (
 )
 
 func TestParseCargoLock(t *testing.T) {
-	expected := []pkg.Package{
+	expected := []*pkg.Package{
 		{
 			Name:         "ansi_term",
 			Version:      "0.12.1",

syft/pkg/package.go

@@ -14,6 +14,7 @@ import (
 // Package represents an application or library that has been bundled into a distributable format.
 // TODO: if we ignore FoundBy for ID generation should we merge the field to show it was found in two places?
 type Package struct {
+	id           artifact.ID       `hash:"ignore"`
 	Name         string            // the package name
 	Version      string            // the version of the package
 	FoundBy      string            // the specific cataloger that discovered this package
@@ -21,24 +22,27 @@ type Package struct {
 	Licenses     []string          // licenses discovered with the package metadata
 	Language     Language          // the language ecosystem this package belongs to (e.g. JavaScript, Python, etc)
 	Type         Type              // the package type (e.g. Npm, Yarn, Python, Rpm, Deb, etc)
-	CPEs         []CPE             // all possible Common Platform Enumerators
-	PURL         string            // the Package URL (see https://github.com/package-url/purl-spec)
+	CPEs         []CPE             `hash:"ignore"` // all possible Common Platform Enumerators (note: this is NOT included in the definition of the ID since all fields on a CPE are derived from other fields)
+	PURL         string            `hash:"ignore"` // the Package URL (see https://github.com/package-url/purl-spec) (note: this is NOT included in the definition of the ID since all fields on a pURL are derived from other fields)
 	MetadataType MetadataType      // the shape of the additional data in the "metadata" field
 	Metadata     interface{}       // additional data found while parsing the package source
 }
 
-func (p Package) ID() artifact.ID {
-	f, err := artifact.IDFromHash(p)
+func (p *Package) SetID() {
+	id, err := artifact.IDByHash(p)
 	if err != nil {
 		// TODO: what to do in this case?
 		log.Warnf("unable to get fingerprint of package=%s@%s: %+v", p.Name, p.Version, err)
-		return ""
+		return
 	}
+	p.id = id
+}
 
-	return f
+func (p Package) ID() artifact.ID {
+	return p.id
 }
 
 // Stringer to represent a package.
 func (p Package) String() string {
-	return fmt.Sprintf("Pkg(type=%s, name=%s, version=%s)", p.Type, p.Name, p.Version)
+	return fmt.Sprintf("Pkg(name=%q version=%q type=%q id=%q)", p.Name, p.Version, p.Type, p.id)
 }

syft/pkg/package_test.go

@@ -127,20 +127,20 @@ func TestFingerprint(t *testing.T) {
 			expectIdentical: false,
 		},
 		{
-			name: "CPEs is reflected",
+			name: "CPEs is ignored",
 			transform: func(pkg Package) Package {
 				pkg.CPEs = []CPE{}
 				return pkg
 			},
-			expectIdentical: false,
+			expectIdentical: true,
 		},
 		{
-			name: "pURL is reflected",
+			name: "pURL is ignored",
 			transform: func(pkg Package) Package {
 				pkg.PURL = "new!"
 				return pkg
 			},
-			expectIdentical: false,
+			expectIdentical: true,
 		},
 		{
 			name: "language is reflected",
@@ -190,7 +190,10 @@ func TestFingerprint(t *testing.T) {
 
 	for _, test := range tests {
 		t.Run(test.name, func(t *testing.T) {
+			originalPkg.SetID()
 			transformedPkg := test.transform(originalPkg)
+			transformedPkg.SetID()
+
 			originalFingerprint := originalPkg.ID()
 			assert.NotEmpty(t, originalFingerprint)
 			transformedFingerprint := transformedPkg.ID()

syft/pkg/python_package_metadata.go

@@ -1,8 +1,10 @@
 package pkg
 
 import (
+	"fmt"
 	"sort"
 
+	"github.com/anchore/packageurl-go"
 	"github.com/scylladb/go-set/strset"
 )
 
@@ -21,17 +23,45 @@ type PythonFileRecord struct {
 	Size   string            `json:"size,omitempty"`
 }
 
+type PythonDirectURLOriginInfo struct {
+	URL      string `json:"url"`
+	CommitID string `json:"commitId,omitempty"`
+	VCS      string `json:"vcs,omitempty"`
+}
+
 // PythonPackageMetadata represents all captured data for a python egg or wheel package.
 type PythonPackageMetadata struct {
-	Name                 string             `json:"name" mapstruct:"Name"`
-	Version              string             `json:"version" mapstruct:"Version"`
-	License              string             `json:"license" mapstruct:"License"`
-	Author               string             `json:"author" mapstruct:"Author"`
-	AuthorEmail          string             `json:"authorEmail" mapstruct:"Authoremail"`
-	Platform             string             `json:"platform" mapstruct:"Platform"`
-	Files                []PythonFileRecord `json:"files,omitempty"`
-	SitePackagesRootPath string             `json:"sitePackagesRootPath"`
-	TopLevelPackages     []string           `json:"topLevelPackages,omitempty"`
+	Name                 string                     `json:"name" mapstruct:"Name"`
+	Version              string                     `json:"version" mapstruct:"Version"`
+	License              string                     `json:"license" mapstruct:"License"`
+	Author               string                     `json:"author" mapstruct:"Author"`
+	AuthorEmail          string                     `json:"authorEmail" mapstruct:"Authoremail"`
+	Platform             string                     `json:"platform" mapstruct:"Platform"`
+	Files                []PythonFileRecord         `json:"files,omitempty"`
+	SitePackagesRootPath string                     `json:"sitePackagesRootPath"`
+	TopLevelPackages     []string                   `json:"topLevelPackages,omitempty"`
+	DirectURLOrigin      *PythonDirectURLOriginInfo `json:"directUrlOrigin,omitempty"`
+}
+
+type DirectURLOrigin struct {
+	URL         string      `json:"url"`
+	VCSInfo     VCSInfo     `json:"vcs_info"`
+	ArchiveInfo ArchiveInfo `json:"archive_info"`
+	DirInfo     DirInfo     `json:"dir_info"`
+}
+
+type DirInfo struct {
+	Editable bool `json:"editable"`
+}
+
+type ArchiveInfo struct {
+	Hash string `json:"hash"`
+}
+
+type VCSInfo struct {
+	CommitID          string `json:"commit_id"`
+	VCS               string `json:"vcs"`
+	RequestedRevision string `json:"requested_revision"`
 }
 
 func (m PythonPackageMetadata) OwnedFiles() (result []string) {
@@ -45,3 +75,33 @@ func (m PythonPackageMetadata) OwnedFiles() (result []string) {
 	sort.Strings(result)
 	return result
 }
+
+func (m PythonPackageMetadata) PackageURL() string {
+	// generate a purl from the package data
+	pURL := packageurl.NewPackageURL(
+		packageurl.TypePyPi,
+		"",
+		m.Name,
+		m.Version,
+		m.purlQualifiers(),
+		"")
+
+	return pURL.ToString()
+}
+
+func (m PythonPackageMetadata) purlQualifiers() packageurl.Qualifiers {
+	q := packageurl.Qualifiers{}
+	if m.DirectURLOrigin != nil {
+		q = append(q, m.DirectURLOrigin.vcsURLQualifier()...)
+	}
+	return q
+}
+
+func (p PythonDirectURLOriginInfo) vcsURLQualifier() packageurl.Qualifiers {
+	if p.VCS != "" {
+		// Taken from https://github.com/package-url/purl-spec/blob/master/PURL-SPECIFICATION.rst#known-qualifiers-keyvalue-pairs
+		// packageurl-go still doesn't support all qualifier names
+		return packageurl.Qualifiers{{Key: "vcs_url", Value: fmt.Sprintf("%s+%s@%s", p.VCS, p.URL, p.CommitID)}}
+	}
+	return packageurl.Qualifiers{}
+}

syft/pkg/relationships_by_file_ownership.go

@@ -27,11 +27,11 @@ func RelationshipsByFileOwnership(catalog *Catalog) []artifact.Relationship {
 	var relationships = findOwnershipByFilesRelationships(catalog)
 
 	var edges []artifact.Relationship
-	for parent, children := range relationships {
-		for child, files := range children {
+	for parentID, children := range relationships {
+		for childID, files := range children {
 			edges = append(edges, artifact.Relationship{
-				From: catalog.byID[parent],
-				To:   catalog.byID[child],
+				From: catalog.byID[parentID],
+				To:   catalog.byID[childID],
 				Type: artifact.OwnershipByFileOverlapRelationship,
 				Data: ownershipByFilesMetadata{
 					Files: files.List(),

syft/pkg/relationships_by_file_ownership_test.go

@@ -32,6 +32,7 @@ func TestOwnershipByFilesRelationship(t *testing.T) {
 						},
 					},
 				}
+				parent.SetID()
 
 				child := Package{
 					Locations: []source.Location{
@@ -40,6 +41,7 @@ func TestOwnershipByFilesRelationship(t *testing.T) {
 					},
 					Type: NpmPkg,
 				}
+				child.SetID()
 
 				relationship := artifact.Relationship{
 					From: parent,
@@ -73,6 +75,7 @@ func TestOwnershipByFilesRelationship(t *testing.T) {
 						},
 					},
 				}
+				parent.SetID()
 
 				child := Package{
 					Locations: []source.Location{
@@ -81,6 +84,7 @@ func TestOwnershipByFilesRelationship(t *testing.T) {
 					},
 					Type: NpmPkg,
 				}
+				child.SetID()
 
 				relationship := artifact.Relationship{
 					From: parent,
@@ -114,6 +118,8 @@ func TestOwnershipByFilesRelationship(t *testing.T) {
 					},
 				}
 
+				parent.SetID()
+
 				child := Package{
 					Locations: []source.Location{
 						source.NewVirtualLocation("/c/path", "/another/path"),
@@ -122,6 +128,8 @@ func TestOwnershipByFilesRelationship(t *testing.T) {
 					Type: NpmPkg,
 				}
 
+				child.SetID()
+
 				return []Package{parent, child}, nil
 			},
 		},

syft/source/coordinates.go

@@ -27,7 +27,7 @@ func NewCoordinateSet(start ...Coordinates) CoordinateSet {
 }
 
 func (c Coordinates) ID() artifact.ID {
-	f, err := artifact.IDFromHash(c)
+	f, err := artifact.IDByHash(c)
 	if err != nil {
 		// TODO: what to do in this case?
 		log.Warnf("unable to get fingerprint of location coordinate=%+v: %+v", c, err)

syft/source/directory_resolver.go

@@ -4,6 +4,7 @@ import (
 	"errors"
 	"fmt"
 	"io"
+	"io/fs"
 	"os"
 	"path"
 	"path/filepath"
@@ -59,17 +60,13 @@ func newDirectoryResolver(root string, pathFilters ...pathFilterFn) (*directoryR
 		currentWdRelRoot = filepath.Clean(root)
 	}
 
-	if pathFilters == nil {
-		pathFilters = []pathFilterFn{isUnallowableFileType, isUnixSystemRuntimePath}
-	}
-
 	resolver := directoryResolver{
 		path:                    root,
 		currentWd:               currentWd,
 		currentWdRelativeToRoot: currentWdRelRoot,
 		fileTree:                filetree.NewFileTree(),
 		metadata:                make(map[file.ID]FileMetadata),
-		pathFilterFns:           pathFilters,
+		pathFilterFns:           append([]pathFilterFn{isUnallowableFileType, isUnixSystemRuntimePath}, pathFilters...),
 		refsByMIMEType:          make(map[string][]file.Reference),
 		errPaths:                make(map[string]error),
 	}
@@ -95,7 +92,7 @@ func (r *directoryResolver) indexTree(root string, stager *progress.Stage) ([]st
 	fi, err := os.Stat(root)
 	if err != nil && fi != nil && !fi.IsDir() {
 		// note: we want to index the path regardless of an error stat-ing the path
-		newRoot := r.indexPath(root, fi, nil)
+		newRoot, _ := r.indexPath(root, fi, nil)
 		if newRoot != "" {
 			roots = append(roots, newRoot)
 		}
@@ -106,7 +103,12 @@ func (r *directoryResolver) indexTree(root string, stager *progress.Stage) ([]st
 		func(path string, info os.FileInfo, err error) error {
 			stager.Current = path
 
-			newRoot := r.indexPath(path, info, err)
+			newRoot, err := r.indexPath(path, info, err)
+
+			if err != nil {
+				return err
+			}
+
 			if newRoot != "" {
 				roots = append(roots, newRoot)
 			}
@@ -115,35 +117,38 @@ func (r *directoryResolver) indexTree(root string, stager *progress.Stage) ([]st
 		})
 }
 
-func (r *directoryResolver) indexPath(path string, info os.FileInfo, err error) string {
+func (r *directoryResolver) indexPath(path string, info os.FileInfo, err error) (string, error) {
 	// ignore any path which a filter function returns true
 	for _, filterFn := range r.pathFilterFns {
-		if filterFn(path, info) {
-			return ""
+		if filterFn != nil && filterFn(path, info) {
+			if info.IsDir() {
+				return "", fs.SkipDir
+			}
+			return "", nil
 		}
 	}
 
 	if r.isFileAccessErr(path, err) {
-		return ""
+		return "", nil
 	}
 
 	// link cycles could cause a revisit --we should not allow this
 	if r.fileTree.HasPath(file.Path(path)) {
-		return ""
+		return "", nil
 	}
 
 	if info == nil {
 		// walk may not be able to provide a FileInfo object, don't allow for this to stop indexing; keep track of the paths and continue.
 		r.errPaths[path] = fmt.Errorf("no file info observable at path=%q", path)
-		return ""
+		return "", nil
 	}
 
 	newRoot, err := r.addPathToIndex(path, info)
 	if r.isFileAccessErr(path, err) {
-		return ""
+		return "", nil
 	}
 
-	return newRoot
+	return newRoot, nil
 }
 
 func (r *directoryResolver) isFileAccessErr(path string, err error) bool {

syft/source/excluding_file_resolver.go

@@ -0,0 +1,101 @@
+package source
+
+import (
+	"fmt"
+	"io"
+)
+
+type excludeFn func(string) bool
+
+// excludingResolver decorates a resolver with an exclusion function that is used to
+// filter out entries in the delegate resolver
+type excludingResolver struct {
+	delegate  FileResolver
+	excludeFn excludeFn
+}
+
+// NewExcludingResolver create a new resolver which wraps the provided delegate and excludes
+// entries based on a provided path exclusion function
+func NewExcludingResolver(delegate FileResolver, excludeFn excludeFn) FileResolver {
+	return &excludingResolver{
+		delegate,
+		excludeFn,
+	}
+}
+
+func (r *excludingResolver) FileContentsByLocation(location Location) (io.ReadCloser, error) {
+	if locationMatches(&location, r.excludeFn) {
+		return nil, fmt.Errorf("no such location: %+v", location.RealPath)
+	}
+	return r.delegate.FileContentsByLocation(location)
+}
+
+func (r *excludingResolver) FileMetadataByLocation(location Location) (FileMetadata, error) {
+	if locationMatches(&location, r.excludeFn) {
+		return FileMetadata{}, fmt.Errorf("no such location: %+v", location.RealPath)
+	}
+	return r.delegate.FileMetadataByLocation(location)
+}
+
+func (r *excludingResolver) HasPath(path string) bool {
+	if r.excludeFn(path) {
+		return false
+	}
+	return r.delegate.HasPath(path)
+}
+
+func (r *excludingResolver) FilesByPath(paths ...string) ([]Location, error) {
+	locations, err := r.delegate.FilesByPath(paths...)
+	return filterLocations(locations, err, r.excludeFn)
+}
+
+func (r *excludingResolver) FilesByGlob(patterns ...string) ([]Location, error) {
+	locations, err := r.delegate.FilesByGlob(patterns...)
+	return filterLocations(locations, err, r.excludeFn)
+}
+
+func (r *excludingResolver) FilesByMIMEType(types ...string) ([]Location, error) {
+	locations, err := r.delegate.FilesByMIMEType(types...)
+	return filterLocations(locations, err, r.excludeFn)
+}
+
+func (r *excludingResolver) RelativeFileByPath(location Location, path string) *Location {
+	l := r.delegate.RelativeFileByPath(location, path)
+	if l != nil && locationMatches(l, r.excludeFn) {
+		return nil
+	}
+	return l
+}
+
+func (r *excludingResolver) AllLocations() <-chan Location {
+	c := make(chan Location)
+	go func() {
+		defer close(c)
+		for location := range r.delegate.AllLocations() {
+			if !locationMatches(&location, r.excludeFn) {
+				c <- location
+			}
+		}
+	}()
+	return c
+}
+
+func locationMatches(location *Location, exclusionFn excludeFn) bool {
+	return exclusionFn(location.RealPath) || exclusionFn(location.VirtualPath)
+}
+
+func filterLocations(locations []Location, err error, exclusionFn excludeFn) ([]Location, error) {
+	if err != nil {
+		return nil, err
+	}
+	if exclusionFn != nil {
+		for i := 0; i < len(locations); i++ {
+			location := &locations[i]
+			if locationMatches(location, exclusionFn) {
+				locations = append(locations[:i], locations[i+1:]...)
+				i--
+			}
+		}
+	}
+	return locations, nil
+}

syft/source/excluding_file_resolver_test.go

@@ -0,0 +1,197 @@
+package source
+
+import (
+	"io"
+	"strings"
+	"testing"
+
+	"github.com/anchore/stereoscope/pkg/file"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestExcludingResolver(t *testing.T) {
+
+	tests := []struct {
+		name      string
+		locations []string
+		excludeFn excludeFn
+		expected  []string
+	}{
+		{
+			name:      "keeps locations",
+			locations: []string{"a", "b", "c"},
+			excludeFn: func(s string) bool {
+				return false
+			},
+			expected: []string{"a", "b", "c"},
+		},
+		{
+			name:      "removes locations",
+			locations: []string{"d", "e", "f"},
+			excludeFn: func(s string) bool {
+				return true
+			},
+			expected: []string{},
+		},
+		{
+			name:      "removes first match",
+			locations: []string{"g", "h", "i"},
+			excludeFn: func(s string) bool {
+				return s == "g"
+			},
+			expected: []string{"h", "i"},
+		},
+		{
+			name:      "removes last match",
+			locations: []string{"j", "k", "l"},
+			excludeFn: func(s string) bool {
+				return s == "l"
+			},
+			expected: []string{"j", "k"},
+		},
+	}
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			resolver := &mockResolver{
+				locations: test.locations,
+			}
+			excludingResolver := NewExcludingResolver(resolver, test.excludeFn)
+
+			locations, _ := excludingResolver.FilesByPath()
+			assert.ElementsMatch(t, locationPaths(locations), test.expected)
+
+			locations, _ = excludingResolver.FilesByGlob()
+			assert.ElementsMatch(t, locationPaths(locations), test.expected)
+
+			locations, _ = excludingResolver.FilesByMIMEType()
+			assert.ElementsMatch(t, locationPaths(locations), test.expected)
+
+			locations = []Location{}
+
+			channel := excludingResolver.AllLocations()
+			for location := range channel {
+				locations = append(locations, location)
+			}
+			assert.ElementsMatch(t, locationPaths(locations), test.expected)
+
+			diff := difference(test.locations, test.expected)
+
+			for _, path := range diff {
+				assert.False(t, excludingResolver.HasPath(path))
+				c, err := excludingResolver.FileContentsByLocation(makeLocation(path))
+				assert.Nil(t, c)
+				assert.Error(t, err)
+				m, err := excludingResolver.FileMetadataByLocation(makeLocation(path))
+				assert.Empty(t, m.LinkDestination)
+				assert.Error(t, err)
+				l := excludingResolver.RelativeFileByPath(makeLocation(""), path)
+				assert.Nil(t, l)
+			}
+
+			for _, path := range test.expected {
+				assert.True(t, excludingResolver.HasPath(path))
+				c, err := excludingResolver.FileContentsByLocation(makeLocation(path))
+				assert.NotNil(t, c)
+				assert.Nil(t, err)
+				m, err := excludingResolver.FileMetadataByLocation(makeLocation(path))
+				assert.NotEmpty(t, m.LinkDestination)
+				assert.Nil(t, err)
+				l := excludingResolver.RelativeFileByPath(makeLocation(""), path)
+				assert.NotNil(t, l)
+			}
+		})
+	}
+}
+
+// difference returns the elements in `a` that aren't in `b`.
+func difference(a, b []string) []string {
+	mb := make(map[string]struct{}, len(b))
+	for _, x := range b {
+		mb[x] = struct{}{}
+	}
+	var diff []string
+	for _, x := range a {
+		if _, found := mb[x]; !found {
+			diff = append(diff, x)
+		}
+	}
+	return diff
+}
+
+func makeLocation(path string) Location {
+	return Location{
+		Coordinates: Coordinates{
+			RealPath:     path,
+			FileSystemID: "",
+		},
+		VirtualPath: "",
+		ref:         file.Reference{},
+	}
+}
+
+func locationPaths(locations []Location) []string {
+	paths := []string{}
+	for _, l := range locations {
+		paths = append(paths, l.RealPath)
+	}
+	return paths
+}
+
+type mockResolver struct {
+	locations []string
+}
+
+func (r *mockResolver) getLocations() ([]Location, error) {
+	out := []Location{}
+	for _, path := range r.locations {
+		out = append(out, makeLocation(path))
+	}
+	return out, nil
+}
+
+func (r *mockResolver) FileContentsByLocation(_ Location) (io.ReadCloser, error) {
+	return io.NopCloser(strings.NewReader("Hello, world!")), nil
+}
+
+func (r *mockResolver) FileMetadataByLocation(_ Location) (FileMetadata, error) {
+	return FileMetadata{
+		LinkDestination: "MOCK",
+	}, nil
+}
+
+func (r *mockResolver) HasPath(_ string) bool {
+	return true
+}
+
+func (r *mockResolver) FilesByPath(_ ...string) ([]Location, error) {
+	return r.getLocations()
+}
+
+func (r *mockResolver) FilesByGlob(_ ...string) ([]Location, error) {
+	return r.getLocations()
+}
+
+func (r *mockResolver) FilesByMIMEType(_ ...string) ([]Location, error) {
+	return r.getLocations()
+}
+
+func (r *mockResolver) RelativeFileByPath(_ Location, path string) *Location {
+	return &Location{
+		Coordinates: Coordinates{
+			RealPath: path,
+		},
+	}
+}
+
+func (r *mockResolver) AllLocations() <-chan Location {
+	c := make(chan Location)
+	go func() {
+		defer close(c)
+		locations, _ := r.getLocations()
+		for _, location := range locations {
+			c <- location
+		}
+	}()
+	return c
+}

syft/source/location.go

@@ -3,21 +3,19 @@ package source
 import (
 	"fmt"
 
-	"github.com/mitchellh/hashstructure/v2"
-
 	"github.com/anchore/stereoscope/pkg/file"
 	"github.com/anchore/stereoscope/pkg/image"
 	"github.com/anchore/syft/internal/log"
 )
 
-var _ hashstructure.Hashable = (*Location)(nil)
-
 // Location represents a path relative to a particular filesystem resolved to a specific file.Reference. This struct is used as a key
 // in content fetching to uniquely identify a file relative to a request (the VirtualPath).
 type Location struct {
 	Coordinates
-	VirtualPath string         // The path to the file which may or may not have hardlinks / symlinks
-	ref         file.Reference // The file reference relative to the stereoscope.FileCatalog that has more information about this location.
+	// note: it is IMPORTANT to ignore anything but the coordinates for a Location when considering the ID (hash value)
+	// since the coordinates are the minimally correct ID for a location (symlinks should not come into play)
+	VirtualPath string         `hash:"ignore"` // The path to the file which may or may not have hardlinks / symlinks
+	ref         file.Reference `hash:"ignore"` // The file reference relative to the stereoscope.FileCatalog that has more information about this location.
 }
 
 // NewLocation creates a new Location representing a path without denoting a filesystem or FileCatalog reference.
@@ -97,9 +95,3 @@ func (l Location) String() string {
 	}
 	return fmt.Sprintf("Location<%s>", str)
 }
-
-func (l Location) Hash() (uint64, error) {
-	// since location is part of the package definition it is important that only coordinates are used during object
-	// hashing. (Location hash should be a pass-through for the coordinates and not include ref or VirtualPath.)
-	return hashstructure.Hash(l.ID(), hashstructure.FormatV2, nil)
-}

syft/source/location_test.go

@@ -0,0 +1,47 @@
+package source
+
+import (
+	"testing"
+
+	"github.com/anchore/stereoscope/pkg/file"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestLocation_ID(t *testing.T) {
+	tests := []struct {
+		name        string
+		coordinates Coordinates
+		virtualPath string
+		ref         file.Reference
+	}{
+		{
+			name: "coordinates should match location hash",
+			coordinates: Coordinates{
+				RealPath:     "path!",
+				FileSystemID: "filesystem!",
+			},
+		},
+		{
+			name: "coordinates should match location hash (with extra fields)",
+			coordinates: Coordinates{
+				RealPath:     "path!",
+				FileSystemID: "filesystem!",
+			},
+			virtualPath: "virtualPath!",
+			ref: file.Reference{
+				RealPath: "other-real-path!",
+			},
+		},
+	}
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			l := Location{
+				Coordinates: test.coordinates,
+				VirtualPath: test.virtualPath,
+				ref:         test.ref,
+			}
+			assert.Equal(t, l.ID(), test.coordinates.ID())
+		})
+	}
+
+}

syft/source/source.go

@@ -9,11 +9,14 @@ import (
 	"fmt"
 	"io/ioutil"
 	"os"
+	"path/filepath"
+	"strings"
 	"sync"
 
 	"github.com/anchore/stereoscope"
 	"github.com/anchore/stereoscope/pkg/image"
 	"github.com/anchore/syft/internal/log"
+	"github.com/bmatcuk/doublestar/v2"
 	"github.com/mholt/archiver/v3"
 	"github.com/spf13/afero"
 )
@@ -26,41 +29,42 @@ type Source struct {
 	directoryResolver *directoryResolver
 	path              string
 	mutex             *sync.Mutex
+	Exclusions        []string
 }
 
 type sourceDetector func(string) (image.Source, string, error)
 
 // New produces a Source based on userInput like dir: or image:tag
-func New(userInput string, registryOptions *image.RegistryOptions) (*Source, func(), error) {
+func New(userInput string, registryOptions *image.RegistryOptions, exclusions []string) (*Source, func(), error) {
 	fs := afero.NewOsFs()
 	parsedScheme, imageSource, location, err := detectScheme(fs, image.DetectSource, userInput)
 	if err != nil {
 		return &Source{}, func() {}, fmt.Errorf("unable to parse input=%q: %w", userInput, err)
 	}
 
+	source := &Source{}
+	cleanupFn := func() {}
+
 	switch parsedScheme {
 	case FileScheme:
-		return generateFileSource(fs, location)
+		source, cleanupFn, err = generateFileSource(fs, location)
 	case DirectoryScheme:
-		return generateDirectorySource(fs, location)
+		source, cleanupFn, err = generateDirectorySource(fs, location)
 	case ImageScheme:
-		return generateImageSource(location, userInput, imageSource, registryOptions)
+		source, cleanupFn, err = generateImageSource(userInput, location, imageSource, registryOptions)
+	default:
+		err = fmt.Errorf("unable to process input for scanning: '%s'", userInput)
 	}
 
-	return &Source{}, func() {}, fmt.Errorf("unable to process input for scanning: '%s'", userInput)
+	if err == nil {
+		source.Exclusions = exclusions
+	}
+
+	return source, cleanupFn, err
 }
 
-func generateImageSource(location, userInput string, imageSource image.Source, registryOptions *image.RegistryOptions) (*Source, func(), error) {
-	img, err := stereoscope.GetImageFromSource(location, imageSource, registryOptions)
-	if err != nil {
-		log.Debugf("error parsing location: %s after detecting scheme; pulling image: %s", location, userInput)
-		// we may have been to aggressive reading the source hint
-		// try the input as supplied by the user if our initial parse failed
-		img, err = stereoscope.GetImageFromSource(userInput, imageSource, registryOptions)
-	}
-
-	cleanup := stereoscope.Cleanup
-
+func generateImageSource(userInput, location string, imageSource image.Source, registryOptions *image.RegistryOptions) (*Source, func(), error) {
+	img, cleanup, err := getImageWithRetryStrategy(userInput, location, imageSource, registryOptions)
 	if err != nil || img == nil {
 		return &Source{}, cleanup, fmt.Errorf("could not fetch image '%s': %w", location, err)
 	}
@@ -73,6 +77,54 @@ func generateImageSource(location, userInput string, imageSource image.Source, r
 	return &s, cleanup, nil
 }
 
+func parseScheme(userInput string) string {
+	parts := strings.SplitN(userInput, ":", 2)
+	if len(parts) < 2 {
+		return ""
+	}
+
+	return parts[0]
+}
+
+func getImageWithRetryStrategy(userInput, location string, imageSource image.Source, registryOptions *image.RegistryOptions) (*image.Image, func(), error) {
+	img, err := stereoscope.GetImageFromSource(location, imageSource, registryOptions)
+	if err == nil {
+		// Success on the first try!
+		return img, stereoscope.Cleanup, nil
+	}
+
+	scheme := parseScheme(userInput)
+	if !(scheme == "docker" || scheme == "registry") {
+		// Image retrieval failed, and we shouldn't retry it. It's most likely that the
+		// user _did_ intend the parsed scheme, but there was a legitimate failure with
+		// using the scheme to load the image. Alert the user to this failure, so they
+		// can fix the problem.
+		return nil, nil, err
+	}
+
+	// Maybe the user wanted "docker" or "registry" to refer to an _image name_
+	// (e.g. "docker:latest"), not a scheme. We'll retry image retrieval with this
+	// alternative interpretation, in an attempt to avoid unnecessary user friction.
+
+	log.Warnf(
+		"scheme %q specified, but it coincides with a common image name; re-examining user input %q"+
+			" without scheme parsing because image retrieval using scheme parsing was unsuccessful: %v",
+		scheme,
+		userInput,
+		err,
+	)
+
+	// We need to determine the image source again, such that this determination
+	// doesn't take scheme parsing into account.
+	imageSource = image.DetermineImagePullSource(userInput)
+	img, err = stereoscope.GetImageFromSource(userInput, imageSource, registryOptions)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	return img, stereoscope.Cleanup, nil
+}
+
 func generateDirectorySource(fs afero.Fs, location string) (*Source, func(), error) {
 	fileMeta, err := fs.Stat(location)
 	if err != nil {
@@ -180,7 +232,11 @@ func (s *Source) FileResolver(scope Scope) (FileResolver, error) {
 		s.mutex.Lock()
 		defer s.mutex.Unlock()
 		if s.directoryResolver == nil {
-			resolver, err := newDirectoryResolver(s.path)
+			exclusionFunctions, err := getDirectoryExclusionFunctions(s.path, s.Exclusions)
+			if err != nil {
+				return nil, err
+			}
+			resolver, err := newDirectoryResolver(s.path, exclusionFunctions...)
 			if err != nil {
 				return nil, err
 			}
@@ -188,14 +244,24 @@ func (s *Source) FileResolver(scope Scope) (FileResolver, error) {
 		}
 		return s.directoryResolver, nil
 	case ImageScheme:
+		var resolver FileResolver
+		var err error
 		switch scope {
 		case SquashedScope:
-			return newImageSquashResolver(s.Image)
+			resolver, err = newImageSquashResolver(s.Image)
 		case AllLayersScope:
-			return newAllLayersResolver(s.Image)
+			resolver, err = newAllLayersResolver(s.Image)
 		default:
 			return nil, fmt.Errorf("bad image scope provided: %+v", scope)
 		}
+		if err != nil {
+			return nil, err
+		}
+		// image tree contains all paths, so we filter out the excluded entries afterwards
+		if len(s.Exclusions) > 0 {
+			resolver = NewExcludingResolver(resolver, getImageExclusionFunction(s.Exclusions))
+		}
+		return resolver, nil
 	}
 	return nil, fmt.Errorf("unable to determine FilePathResolver with current scheme=%q", s.Metadata.Scheme)
 }
@@ -214,3 +280,71 @@ func unarchiveToTmp(path string, unarchiver archiver.Unarchiver) (string, func()
 
 	return tempDir, cleanupFn, unarchiver.Unarchive(path, tempDir)
 }
+
+func getImageExclusionFunction(exclusions []string) func(string) bool {
+	if len(exclusions) == 0 {
+		return nil
+	}
+	// add subpath exclusions
+	for _, exclusion := range exclusions {
+		exclusions = append(exclusions, exclusion+"/**")
+	}
+	return func(path string) bool {
+		for _, exclusion := range exclusions {
+			matches, err := doublestar.Match(exclusion, path)
+			if err != nil {
+				return false
+			}
+			if matches {
+				return true
+			}
+		}
+		return false
+	}
+}
+
+func getDirectoryExclusionFunctions(root string, exclusions []string) ([]pathFilterFn, error) {
+	if len(exclusions) == 0 {
+		return nil, nil
+	}
+
+	// this is what directoryResolver.indexTree is doing to get the absolute path:
+	root, err := filepath.Abs(root)
+	if err != nil {
+		return nil, err
+	}
+
+	if !strings.HasSuffix(root, "/") {
+		root += "/"
+	}
+
+	var errors []string
+	for idx, exclusion := range exclusions {
+		// check exclusions for supported paths, these are all relative to the "scan root"
+		if strings.HasPrefix(exclusion, "./") || strings.HasPrefix(exclusion, "*/") || strings.HasPrefix(exclusion, "**/") {
+			exclusion = strings.TrimPrefix(exclusion, "./")
+			exclusions[idx] = root + exclusion
+		} else {
+			errors = append(errors, exclusion)
+		}
+	}
+
+	if errors != nil {
+		return nil, fmt.Errorf("invalid exclusion pattern(s): '%s' (must start with one of: './', '*/', or '**/')", strings.Join(errors, "', '"))
+	}
+
+	return []pathFilterFn{
+		func(path string, _ os.FileInfo) bool {
+			for _, exclusion := range exclusions {
+				matches, err := doublestar.Match(exclusion, path)
+				if err != nil {
+					return false
+				}
+				if matches {
+					return true
+				}
+			}
+			return false
+		},
+	}, nil
+}

syft/source/source_test.go

@@ -6,9 +6,12 @@ import (
 	"os/exec"
 	"path"
 	"path/filepath"
+	"strings"
 	"syscall"
 	"testing"
 
+	"github.com/anchore/stereoscope/pkg/imagetest"
+
 	"github.com/stretchr/testify/require"
 
 	"github.com/stretchr/testify/assert"
@@ -317,6 +320,219 @@ func TestFilesByGlob(t *testing.T) {
 	}
 }
 
+func TestDirectoryExclusions(t *testing.T) {
+	testCases := []struct {
+		desc       string
+		input      string
+		glob       string
+		expected   int
+		exclusions []string
+		err        bool
+	}{
+		{
+			input:      "test-fixtures/system_paths",
+			desc:       "exclude everything",
+			glob:       "**",
+			expected:   0,
+			exclusions: []string{"**/*"},
+		},
+		{
+			input:      "test-fixtures/image-simple",
+			desc:       "a single path excluded",
+			glob:       "**",
+			expected:   3,
+			exclusions: []string{"**/target/**"},
+		},
+		{
+			input:      "test-fixtures/image-simple",
+			desc:       "exclude explicit directory relative to the root",
+			glob:       "**",
+			expected:   3,
+			exclusions: []string{"./target"},
+		},
+		{
+			input:      "test-fixtures/image-simple",
+			desc:       "exclude explicit file relative to the root",
+			glob:       "**",
+			expected:   3,
+			exclusions: []string{"./file-1.txt"},
+		},
+		{
+			input:      "test-fixtures/image-simple",
+			desc:       "exclude wildcard relative to the root",
+			glob:       "**",
+			expected:   2,
+			exclusions: []string{"./*.txt"},
+		},
+		{
+			input:      "test-fixtures/image-simple",
+			desc:       "exclude files deeper",
+			glob:       "**",
+			expected:   3,
+			exclusions: []string{"**/really/**"},
+		},
+		{
+			input:      "test-fixtures/image-simple",
+			desc:       "files excluded with extension",
+			glob:       "**",
+			expected:   1,
+			exclusions: []string{"**/*.txt"},
+		},
+		{
+			input:      "test-fixtures/image-simple",
+			desc:       "keep files with different extensions",
+			glob:       "**",
+			expected:   4,
+			exclusions: []string{"**/target/**/*.jar"},
+		},
+		{
+			input:      "test-fixtures/path-detected",
+			desc:       "file directly excluded",
+			glob:       "**",
+			expected:   1,
+			exclusions: []string{"**/empty"},
+		},
+		{
+			input:      "test-fixtures/path-detected",
+			desc:       "pattern error containing **/",
+			glob:       "**",
+			expected:   1,
+			exclusions: []string{"/**/empty"},
+			err:        true,
+		},
+		{
+			input:      "test-fixtures/path-detected",
+			desc:       "pattern error incorrect start",
+			glob:       "**",
+			expected:   1,
+			exclusions: []string{"empty"},
+			err:        true,
+		},
+		{
+			input:      "test-fixtures/path-detected",
+			desc:       "pattern error starting with /",
+			glob:       "**",
+			expected:   1,
+			exclusions: []string{"/empty"},
+			err:        true,
+		},
+	}
+	registryOpts := &image.RegistryOptions{}
+	for _, test := range testCases {
+		t.Run(test.desc, func(t *testing.T) {
+			src, fn, err := New("dir:"+test.input, registryOpts, test.exclusions)
+			defer fn()
+
+			if test.err {
+				_, err = src.FileResolver(SquashedScope)
+				if err == nil {
+					t.Errorf("expected an error for patterns: %s", strings.Join(test.exclusions, " or "))
+				}
+				return
+			}
+
+			if err != nil {
+				t.Errorf("could not create NewDirScope: %+v", err)
+			}
+			resolver, err := src.FileResolver(SquashedScope)
+			if err != nil {
+				t.Errorf("could not get resolver error: %+v", err)
+			}
+			contents, err := resolver.FilesByGlob(test.glob)
+			if err != nil {
+				t.Errorf("could not get files by glob: %s+v", err)
+			}
+			if len(contents) != test.expected {
+				t.Errorf("wrong number of files after exclusions (%s): %d != %d", test.glob, len(contents), test.expected)
+			}
+		})
+	}
+}
+
+func TestImageExclusions(t *testing.T) {
+	testCases := []struct {
+		desc       string
+		input      string
+		glob       string
+		expected   int
+		exclusions []string
+	}{
+		// NOTE: in the Dockerfile, /target is moved to /, which makes /really a top-level dir
+		{
+			input:      "image-simple",
+			desc:       "a single path excluded",
+			glob:       "**",
+			expected:   2,
+			exclusions: []string{"/really/**"},
+		},
+		{
+			input:      "image-simple",
+			desc:       "a directly referenced directory is excluded",
+			glob:       "**",
+			expected:   2,
+			exclusions: []string{"/really"},
+		},
+		{
+			input:      "image-simple",
+			desc:       "a partial directory is not excluded",
+			glob:       "**",
+			expected:   3,
+			exclusions: []string{"/reall"},
+		},
+		{
+			input:      "image-simple",
+			desc:       "exclude files deeper",
+			glob:       "**",
+			expected:   2,
+			exclusions: []string{"**/nested/**"},
+		},
+		{
+			input:      "image-simple",
+			desc:       "files excluded with extension",
+			glob:       "**",
+			expected:   2,
+			exclusions: []string{"**/*1.txt"},
+		},
+		{
+			input:      "image-simple",
+			desc:       "keep files with different extensions",
+			glob:       "**",
+			expected:   3,
+			exclusions: []string{"**/target/**/*.jar"},
+		},
+		{
+			input:      "image-simple",
+			desc:       "file directly excluded",
+			glob:       "**",
+			expected:   2,
+			exclusions: []string{"**/somefile-1.txt"}, // file-1 renamed to somefile-1 in Dockerfile
+		},
+	}
+	registryOpts := &image.RegistryOptions{}
+	for _, test := range testCases {
+		t.Run(test.desc, func(t *testing.T) {
+			archiveLocation := imagetest.PrepareFixtureImage(t, "docker-archive", test.input)
+			src, fn, err := New(archiveLocation, registryOpts, test.exclusions)
+			defer fn()
+
+			if err != nil {
+				t.Errorf("could not create NewDirScope: %+v", err)
+			}
+			resolver, err := src.FileResolver(SquashedScope)
+			if err != nil {
+				t.Errorf("could not get resolver error: %+v", err)
+			}
+			contents, err := resolver.FilesByGlob(test.glob)
+			if err != nil {
+				t.Errorf("could not get files by glob: %s+v", err)
+			}
+			if len(contents) != test.expected {
+				t.Errorf("wrong number of files after exclusions (%s): %d != %d", test.glob, len(contents), test.expected)
+			}
+		})
+	}
+}
+
 // createArchive creates a new archive file at destinationArchivePath based on the directory found at sourceDirPath.
 func createArchive(t testing.TB, sourceDirPath, destinationArchivePath string) {
 	t.Helper()

test/cli/packages_cmd_test.go

@@ -7,7 +7,7 @@ import (
 
 func TestPackagesCmdFlags(t *testing.T) {
 	coverageImage := "docker-archive:" + getFixtureImage(t, "image-pkg-coverage")
-	badBinariesImage := "docker-archive:" + getFixtureImage(t, "image-bad-binaries")
+	//badBinariesImage := "docker-archive:" + getFixtureImage(t, "image-bad-binaries")
 
 	tests := []struct {
 		name       string
@@ -32,20 +32,26 @@ func TestPackagesCmdFlags(t *testing.T) {
 				assertSuccessfulReturnCode,
 			},
 		},
-		{
-			name: "regression-survive-bad-binaries",
-			// this image has all sorts of rich binaries from the clang-13 test suite that should do pretty bad things
-			// to the go cataloger binary path. We should NEVER let a panic stop the cataloging process for these
-			// specific cases.
-
-			// this is more of an integration test, however, to assert the output we want to see from the application
-			// a CLI test is much easier.
-			args: []string{"packages", "-vv", badBinariesImage},
-			assertions: []traitAssertion{
-				assertInOutput("could not parse possible go binary"),
-				assertSuccessfulReturnCode,
-			},
-		},
+		// I haven't been able to reproduce locally yet, but in CI this has proven to be unstable:
+		// For the same commit:
+		//   pass: https://github.com/anchore/syft/runs/4611344142?check_suite_focus=true
+		//   fail: https://github.com/anchore/syft/runs/4611343586?check_suite_focus=true
+		// For the meantime this test will be commented out, but should be added back in as soon as possible.
+		//
+		//{
+		//	name: "regression-survive-bad-binaries",
+		//	// this image has all sorts of rich binaries from the clang-13 test suite that should do pretty bad things
+		//	// to the go cataloger binary path. We should NEVER let a panic stop the cataloging process for these
+		//	// specific cases.
+		//
+		//	// this is more of an integration test, however, to assert the output we want to see from the application
+		//	// a CLI test is much easier.
+		//	args: []string{"packages", "-vv", badBinariesImage},
+		//	assertions: []traitAssertion{
+		//		assertInOutput("could not parse possible go binary"),
+		//		assertSuccessfulReturnCode,
+		//	},
+		//},
 		{
 			name: "output-env-binding",
 			env: map[string]string{

test/integration/catalog_packages_test.go

@@ -22,7 +22,7 @@ func BenchmarkImagePackageCatalogers(b *testing.B) {
 	var pc *pkg.Catalog
 	for _, c := range cataloger.ImageCatalogers() {
 		// in case of future alteration where state is persisted, assume no dependency is safe to reuse
-		theSource, cleanupSource, err := source.New("docker-archive:"+tarPath, nil)
+		theSource, cleanupSource, err := source.New("docker-archive:"+tarPath, nil, nil)
 		b.Cleanup(cleanupSource)
 		if err != nil {
 			b.Fatalf("unable to get source: %+v", err)

test/integration/utils_test.go

@@ -14,7 +14,7 @@ func catalogFixtureImage(t *testing.T, fixtureImageName string) (sbom.SBOM, *sou
 	imagetest.GetFixtureImage(t, "docker-archive", fixtureImageName)
 	tarPath := imagetest.GetFixtureImageTarPath(t, fixtureImageName)
 
-	theSource, cleanupSource, err := source.New("docker-archive:"+tarPath, nil)
+	theSource, cleanupSource, err := source.New("docker-archive:"+tarPath, nil, nil)
 	t.Cleanup(cleanupSource)
 	if err != nil {
 		t.Fatalf("unable to get source: %+v", err)
@@ -45,7 +45,7 @@ func catalogFixtureImage(t *testing.T, fixtureImageName string) (sbom.SBOM, *sou
 }
 
 func catalogDirectory(t *testing.T, dir string) (sbom.SBOM, *source.Source) {
-	theSource, cleanupSource, err := source.New("dir:"+dir, nil)
+	theSource, cleanupSource, err := source.New("dir:"+dir, nil, nil)
 	t.Cleanup(cleanupSource)
 	if err != nil {
 		t.Fatalf("unable to get source: %+v", err)