Merge pull request #328 from anchore/issue-327

Be lenient on invalid fields in PKG-INFO

.github/workflows/acceptance-test.yaml

@@ -1,4 +1,4 @@
-name: 'Acceptance'
+name: "Acceptance"
 on:
   workflow_dispatch:
   push:
@@ -16,7 +16,6 @@ jobs:
   Build-Snapshot-Artifacts:
     runs-on: macos-latest # We're creating these snapshot builds on macOS to be consistent with our release workflow's build process, which also takes place on macOS (due to code signing requirements).
     steps:
-
       - uses: actions/setup-go@v2
         with:
           go-version: ${{ env.GO_VERSION }}
@@ -25,7 +24,7 @@ jobs:
 
       - name: Restore bootstrap cache
         id: cache
-        uses: actions/cache@v2
+        uses: actions/cache@v2.1.3
         with:
           path: |
             ~/go/pkg/mod
@@ -75,11 +74,10 @@ jobs:
 
   # Note: changing this job name requires making the same update in the .github/workflows/release.yaml pipeline
   Acceptance-Linux:
-    needs: [ Build-Snapshot-Artifacts ]
+    needs: [Build-Snapshot-Artifacts]
     # come Nov 30 2020 ubuntu-latest will be ubuntu-20.04, until then it needs to be explicitly referenced due to python 3.7 specific features being used
     runs-on: ubuntu-20.04
     steps:
-
       - uses: actions/checkout@v2
 
       - uses: actions/download-artifact@v2
@@ -101,10 +99,9 @@ jobs:
 
   # Note: changing this job name requires making the same update in the .github/workflows/release.yaml pipeline
   Acceptance-Mac:
-    needs: [ Build-Snapshot-Artifacts ]
+    needs: [Build-Snapshot-Artifacts]
     runs-on: macos-latest
     steps:
-
       - uses: actions/checkout@v2
 
       - uses: actions/download-artifact@v2
@@ -126,11 +123,10 @@ jobs:
 
   # Note: changing this job name requires making the same update in the .github/workflows/release.yaml pipeline
   Inline-Compare:
-    needs: [ Build-Snapshot-Artifacts ]
+    needs: [Build-Snapshot-Artifacts]
     # come Nov 30 2020 ubuntu-latest will be ubuntu-20.04, until then it needs to be explicitly referenced due to python 3.7 specific features being used
     runs-on: ubuntu-20.04
     steps:
-
       - uses: actions/checkout@v2
 
       - name: Fingerprint inline-compare sources
@@ -138,7 +134,7 @@ jobs:
 
       - name: Restore inline reports cache
         id: cache
-        uses: actions/cache@v2
+        uses: actions/cache@v2.1.3
         with:
           path: ${{ github.workspace }}/test/inline-compare/inline-reports
           key: inline-reports-${{ hashFiles('**/inline-compare.fingerprint') }}

.github/workflows/release.yaml

@@ -1,12 +1,12 @@
-name: 'Release'
+name: "Release"
 on:
   push:
     # take no actions on push to any branch...
     branches-ignore:
-      - '**'
+      - "**"
     # ... only act on release tags
     tags:
-      - 'v*'
+      - "v*"
 
 env:
   GO_VERSION: "1.14.x"
@@ -15,7 +15,6 @@ jobs:
   wait-for-checks:
     runs-on: ubuntu-latest # This OS choice is arbitrary. None of the steps in this job are specific to either Linux or macOS.
     steps:
-
       - uses: actions/checkout@v2
 
       # we don't want to release commits that have been pushed and tagged, but not necessarily merged onto main
@@ -81,10 +80,9 @@ jobs:
           false
 
   release:
-    needs: [ wait-for-checks ]
+    needs: [wait-for-checks]
     runs-on: macos-latest # Due to our code signing process, it's vital that we run our release steps on macOS.
     steps:
-
       - uses: actions/setup-go@v2
         with:
           go-version: ${{ env.GO_VERSION }}
@@ -96,7 +94,7 @@ jobs:
       # We are expecting this cache to have been created during the "Build-Snapshot-Artifacts" job in the "Acceptance" workflow.
       - name: Restore bootstrap cache
         id: cache
-        uses: actions/cache@v2
+        uses: actions/cache@v2.1.3
         with:
           path: |
             ~/go/pkg/mod

.github/workflows/static-unit-integration.yaml

@@ -1,10 +1,9 @@
-name: 'Static Analysis + Unit + Integration'
+name: "Static Analysis + Unit + Integration"
 on:
   workflow_dispatch:
   push:
   pull_request:
 jobs:
-
   Static-Analysis:
     strategy:
       matrix:
@@ -12,7 +11,6 @@ jobs:
         platform: [ubuntu-latest]
     runs-on: ${{ matrix.platform }}
     steps:
-
       - uses: actions/setup-go@v2
         with:
           go-version: ${{ matrix.go-version }}
@@ -21,7 +19,7 @@ jobs:
 
       - name: Restore bootstrap cache
         id: bootstrap-cache
-        uses: actions/cache@v2
+        uses: actions/cache@v2.1.3
         with:
           path: |
             ~/go/pkg/mod
@@ -49,7 +47,6 @@ jobs:
         platform: [ubuntu-latest]
     runs-on: ${{ matrix.platform }}
     steps:
-
       - uses: actions/setup-go@v2
         with:
           go-version: ${{ matrix.go-version }}
@@ -58,7 +55,7 @@ jobs:
 
       - name: Restore bootstrap cache
         id: bootstrap-cache
-        uses: actions/cache@v2
+        uses: actions/cache@v2.1.3
         with:
           path: |
             ~/go/pkg/mod
@@ -80,7 +77,7 @@ jobs:
 
       - name: Restore Java test-fixture cache
         id: unit-java-cache
-        uses: actions/cache@v2
+        uses: actions/cache@v2.1.3
         with:
           path: syft/cataloger/java/test-fixtures/java-builds/packages
           key: ${{ runner.os }}-unit-java-cache-${{ hashFiles( 'syft/cataloger/java/test-fixtures/java-builds/packages.fingerprint' ) }}
@@ -95,7 +92,7 @@ jobs:
         run: make integration-fingerprint
 
       - name: Restore integration test cache
-        uses: actions/cache@v2
+        uses: actions/cache@v2.1.3
         with:
           path: ${{ github.workspace }}/integration/test-fixtures/cache
           key: ${{ runner.os }}-integration-test-cache-${{ hashFiles('integration/test-fixtures/cache.fingerprint') }}

.gitignore

@@ -33,3 +33,5 @@ coverage.txt
 
 # macOS Finder metadata
 .DS_STORE
+
+*.profile

.golangci.yaml

@@ -31,7 +31,6 @@ linters:
     - misspell
     - nakedret
     - nolintlint
-    - prealloc
     - rowserrcheck
     - scopelint
     - staticcheck
@@ -55,5 +54,6 @@ linters:
 #    - lll         # without a way to specify per-line exception cases, this is not usable
 #    - maligned    # this is an excellent linter, but tricky to optimize and we are not sensitive to memory layout optimizations
 #    - nestif
+#    - prealloc    # following this rule isn't consistently a good idea, as it sometimes forces unnecessary allocations that result in less idiomatic code
 #    - testpackage
-#    - wsl
+#    - wsl

Makefile

@@ -16,6 +16,10 @@ TITLE := $(BOLD)$(PURPLE)
 SUCCESS := $(BOLD)$(GREEN)
 # the quality gate lower threshold for unit test total % coverage (by function statements)
 COVERAGE_THRESHOLD := 68
+# CI cache busting values; change these if you want CI to not use previous stored cache
+COMPARE_CACHE_BUSTER="f7e689d76a9"
+INTEGRATION_CACHE_BUSTER="789bacdf"
+BOOTSTRAP_CACHE="789bacdf"
 
 ## Build variables
 DISTDIR=./dist
@@ -140,7 +144,7 @@ integration: ## Run integration tests
 
 # note: this is used by CI to determine if the integration test fixture cache (docker image tars) should be busted
 integration-fingerprint:
-	find test/integration/test-fixtures/image-* -type f -exec md5sum {} + | awk '{print $1}' | sort | md5sum | tee test/integration/test-fixtures/cache.fingerprint
+	find test/integration/test-fixtures/image-* -type f -exec md5sum {} + | awk '{print $1}' | sort | md5sum | tee test/integration/test-fixtures/cache.fingerprint && echo "$(INTEGRATION_CACHE_BUSTER)" >> test/integration/test-fixtures/cache.fingerprint
 
 .PHONY: java-packages-fingerprint
 java-packages-fingerprint:
@@ -154,8 +158,7 @@ fixtures:
 
 .PHONY: generate-json-schema
 generate-json-schema:  ## Generate a new json schema
-	cd schema/json
-	go run generate.go
+	cd schema/json && go run generate.go
 
 .PHONY: clear-test-cache
 clear-test-cache: ## Delete all test cache (built docker image tars)
@@ -174,6 +177,7 @@ $(SNAPSHOTDIR): ## Build snapshot release binaries and packages
 	BUILD_GIT_TREE_STATE=$(GITTREESTATE) \
 	$(TEMPDIR)/goreleaser release --skip-publish --rm-dist --snapshot --config $(TEMPDIR)/goreleaser.yaml
 
+# note: we cannot clean the snapshot directory since the pipeline builds the snapshot separately
 .PHONY: acceptance-mac
 acceptance-mac: $(SNAPSHOTDIR) ## Run acceptance tests on build snapshot binaries and packages (Mac)
 	$(call title,Running acceptance test: Run on Mac)
@@ -183,13 +187,14 @@ acceptance-mac: $(SNAPSHOTDIR) ## Run acceptance tests on build snapshot binarie
 			$(ACC_TEST_IMAGE) \
 			$(RESULTSDIR)
 
+# note: we cannot clean the snapshot directory since the pipeline builds the snapshot separately
 .PHONY: acceptance-linux
 acceptance-linux: acceptance-test-deb-package-install acceptance-test-rpm-package-install ## Run acceptance tests on build snapshot binaries and packages (Linux)
 
 # note: this is used by CI to determine if the inline-scan report cache should be busted for the inline-compare tests
 .PHONY: compare-fingerprint
 compare-fingerprint:
-	find test/inline-compare/* -type f -exec md5sum {} + | grep -v '\-reports' | grep -v 'fingerprint' | awk '{print $1}' | sort | md5sum | tee test/inline-compare/inline-compare.fingerprint
+	find test/inline-compare/* -type f -exec md5sum {} + | grep -v '\-reports' | grep -v 'fingerprint' | awk '{print $1}' | sort | md5sum | tee test/inline-compare/inline-compare.fingerprint && echo "$(COMPARE_CACHE_BUSTER)" >> test/inline-compare/inline-compare.fingerprint
 
 .PHONY: compare-snapshot
 compare-snapshot: $(SNAPSHOTDIR) ## Compare the reports of a run of a snapshot build of syft against inline-scan

cmd/cmd.go

@@ -86,7 +86,7 @@ func setGlobalUploadOptions() {
 	flag := "host"
 	rootCmd.Flags().StringP(
 		flag, "H", "",
-		"the hostname or URL of the Anchore Engine/Enterprise instance to upload to",
+		"the hostname or URL of the Anchore Enterprise instance to upload to",
 	)
 	if err := viper.BindPFlag("anchore.host", rootCmd.Flags().Lookup(flag)); err != nil {
 		fmt.Printf("unable to bind flag '%s': %+v", flag, err)
@@ -96,7 +96,7 @@ func setGlobalUploadOptions() {
 	flag = "username"
 	rootCmd.Flags().StringP(
 		flag, "u", "",
-		"the username to authenticate against Anchore Engine/Enterprise",
+		"the username to authenticate against Anchore Enterprise",
 	)
 	if err := viper.BindPFlag("anchore.username", rootCmd.Flags().Lookup(flag)); err != nil {
 		fmt.Printf("unable to bind flag '%s': %+v", flag, err)
@@ -106,7 +106,7 @@ func setGlobalUploadOptions() {
 	flag = "password"
 	rootCmd.Flags().StringP(
 		flag, "p", "",
-		"the password to authenticate against Anchore Engine/Enterprise",
+		"the password to authenticate against Anchore Enterprise",
 	)
 	if err := viper.BindPFlag("anchore.password", rootCmd.Flags().Lookup(flag)); err != nil {
 		fmt.Printf("unable to bind flag '%s': %+v", flag, err)
@@ -116,12 +116,22 @@ func setGlobalUploadOptions() {
 	flag = "dockerfile"
 	rootCmd.Flags().StringP(
 		flag, "d", "",
-		"include dockerfile for upload to Anchore Engine/Enterprise",
+		"include dockerfile for upload to Anchore Enterprise",
 	)
 	if err := viper.BindPFlag("anchore.dockerfile", rootCmd.Flags().Lookup(flag)); err != nil {
 		fmt.Printf("unable to bind flag '#{flag}': #{err}")
 		os.Exit(1)
 	}
+
+	flag = "overwrite-existing-image"
+	rootCmd.Flags().Bool(
+		flag, false,
+		"overwrite an existing image during the upload to Anchore Enterprise",
+	)
+	if err := viper.BindPFlag("anchore.overwrite-existing-image", rootCmd.Flags().Lookup(flag)); err != nil {
+		fmt.Printf("unable to bind flag '#{flag}': #{err}")
+		os.Exit(1)
+	}
 }
 
 func initAppConfig() {

cmd/root.go

@@ -7,11 +7,7 @@ import (
 	"os"
 	"strings"
 
-	"github.com/anchore/syft/syft/distro"
-
-	"github.com/anchore/syft/syft/pkg"
-
-	"github.com/anchore/syft/syft/source"
+	"github.com/pkg/profile"
 
 	"github.com/anchore/syft/internal"
 	"github.com/anchore/syft/internal/anchore"
@@ -20,8 +16,11 @@ import (
 	"github.com/anchore/syft/internal/ui"
 	"github.com/anchore/syft/internal/version"
 	"github.com/anchore/syft/syft"
+	"github.com/anchore/syft/syft/distro"
 	"github.com/anchore/syft/syft/event"
+	"github.com/anchore/syft/syft/pkg"
 	"github.com/anchore/syft/syft/presenter"
+	"github.com/anchore/syft/syft/source"
 	"github.com/docker/docker/api/types"
 	"github.com/docker/docker/api/types/filters"
 	"github.com/docker/docker/client"
@@ -56,7 +55,20 @@ You can also explicitly specify the scheme to use:
 			}
 			os.Exit(1)
 		}
+
+		if appConfig.Dev.ProfileCPU && appConfig.Dev.ProfileMem {
+			log.Errorf("cannot profile CPU and memory simultaneously")
+			os.Exit(1)
+		}
+
+		if appConfig.Dev.ProfileCPU {
+			defer profile.Start(profile.CPUProfile).Stop()
+		} else if appConfig.Dev.ProfileMem {
+			defer profile.Start(profile.MemProfile).Stop()
+		}
+
 		err := doRunCmd(cmd, args)
+
 		if err != nil {
 			log.Errorf(err.Error())
 			os.Exit(1)
@@ -146,25 +158,25 @@ func doImport(src source.Source, s source.Metadata, catalog *pkg.Catalog, d *dis
 		}
 	}
 
-	var scheme string
-	var hostname = appConfig.Anchore.Host
-	urlFields := strings.Split(hostname, "://")
-	if len(urlFields) > 1 {
-		scheme = urlFields[0]
-		hostname = urlFields[1]
-	}
-
 	c, err := anchore.NewClient(anchore.Configuration{
-		Hostname: hostname,
+		BaseURL:  appConfig.Anchore.Host,
 		Username: appConfig.Anchore.Username,
 		Password: appConfig.Anchore.Password,
-		Scheme:   scheme,
 	})
 	if err != nil {
-		return fmt.Errorf("failed to create anchore client: %+v", err)
+		return fmt.Errorf("unable to upload results: %w", err)
 	}
 
-	if err := c.Import(context.Background(), src.Image.Metadata, s, catalog, d, dockerfileContents); err != nil {
+	importCfg := anchore.ImportConfig{
+		ImageMetadata:           src.Image.Metadata,
+		SourceMetadata:          s,
+		Catalog:                 catalog,
+		Distro:                  d,
+		Dockerfile:              dockerfileContents,
+		OverwriteExistingUpload: appConfig.Anchore.OverwriteExistingImage,
+	}
+
+	if err := c.Import(context.Background(), importCfg); err != nil {
 		return fmt.Errorf("failed to upload results to host=%s: %+v", appConfig.Anchore.Host, err)
 	}
 	return nil

go.mod

@@ -5,12 +5,13 @@ go 1.14
 require (
 	github.com/adrg/xdg v0.2.1
 	github.com/alecthomas/jsonschema v0.0.0-20200530073317-71f438968921
-	github.com/anchore/client-go v0.0.0-20201210022459-59e7a0749c74
+	github.com/anchore/client-go v0.0.0-20201216213038-a486b838e238
 	github.com/anchore/go-rpmdb v0.0.0-20201106153645-0043963c2e12
 	github.com/anchore/go-testutils v0.0.0-20200925183923-d5f45b0d3c04
 	github.com/anchore/go-version v1.2.2-0.20200701162849-18adb9c92b9b
-	github.com/anchore/stereoscope v0.0.0-20201210022249-091f9bddb42e
-	github.com/bmatcuk/doublestar v1.3.3
+	github.com/anchore/stereoscope v0.0.0-20210201165248-e94c52b4052d
+	github.com/antihax/optional v1.0.0
+	github.com/bmatcuk/doublestar/v2 v2.0.4
 	github.com/docker/docker v17.12.0-ce-rc1.0.20200309214505-aa6a9891b09c+incompatible
 	github.com/dustin/go-humanize v1.0.0
 	github.com/facebookincubator/nvdtools v0.1.4
@@ -24,6 +25,7 @@ require (
 	github.com/olekukonko/tablewriter v0.0.4
 	github.com/package-url/packageurl-go v0.1.0
 	github.com/pelletier/go-toml v1.8.0
+	github.com/pkg/profile v1.5.0
 	github.com/scylladb/go-set v1.0.2
 	github.com/sergi/go-diff v1.1.0
 	github.com/sirupsen/logrus v1.6.0

go.sum

@@ -126,25 +126,22 @@ github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuy
 github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
 github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
 github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho=
-github.com/anchore/client-go v0.0.0-20201120223920-9f812673f4d6/go.mod h1:FaODhIA06mxO1E6R32JE0TL1JWZZkmjRIAd4ULvHUKk=
-github.com/anchore/client-go v0.0.0-20201210022459-59e7a0749c74 h1:9kkKTIyXJC+/syUcY6KWxFoJZJ+GWwrIscF+gBY067k=
-github.com/anchore/client-go v0.0.0-20201210022459-59e7a0749c74/go.mod h1:FaODhIA06mxO1E6R32JE0TL1JWZZkmjRIAd4ULvHUKk=
+github.com/anchore/client-go v0.0.0-20201216213038-a486b838e238 h1:/iI+1cj1a27ow0wj378pPJIm8sCSy6I21Tz6oLbLDQY=
+github.com/anchore/client-go v0.0.0-20201216213038-a486b838e238/go.mod h1:FaODhIA06mxO1E6R32JE0TL1JWZZkmjRIAd4ULvHUKk=
 github.com/anchore/go-rpmdb v0.0.0-20201106153645-0043963c2e12 h1:xbeIbn5F52JVx3RUIajxCj8b0y+9lywspql4sFhcxWQ=
 github.com/anchore/go-rpmdb v0.0.0-20201106153645-0043963c2e12/go.mod h1:juoyWXIj7sJ1IDl4E/KIfyLtovbs5XQVSIdaQifFQT8=
 github.com/anchore/go-testutils v0.0.0-20200925183923-d5f45b0d3c04 h1:VzprUTpc0vW0nnNKJfJieyH/TZ9UYAnTZs5/gHTdAe8=
 github.com/anchore/go-testutils v0.0.0-20200925183923-d5f45b0d3c04/go.mod h1:6dK64g27Qi1qGQZ67gFmBFvEHScy0/C8qhQhNe5B5pQ=
 github.com/anchore/go-version v1.2.2-0.20200701162849-18adb9c92b9b h1:e1bmaoJfZVsCYMrIZBpFxwV26CbsuoEh5muXD5I1Ods=
 github.com/anchore/go-version v1.2.2-0.20200701162849-18adb9c92b9b/go.mod h1:Bkc+JYWjMCF8OyZ340IMSIi2Ebf3uwByOk6ho4wne1E=
-github.com/anchore/stereoscope v0.0.0-20201106140100-12e75c48f409 h1:xKSpDRjmYrEFrdMeDh4AuSUAFc99pdro6YFBKxy2um0=
-github.com/anchore/stereoscope v0.0.0-20201106140100-12e75c48f409/go.mod h1:2Jja/4l0zYggW52og+nn0rut4i+OYjCf9vTyrM8RT4E=
-github.com/anchore/stereoscope v0.0.0-20201130153727-b3f1fad856b0 h1:wa0hdnvBeCpI+rmzDbPG7k5SKlsGkot7aZ8Az1i/vws=
-github.com/anchore/stereoscope v0.0.0-20201130153727-b3f1fad856b0/go.mod h1:2Jja/4l0zYggW52og+nn0rut4i+OYjCf9vTyrM8RT4E=
-github.com/anchore/stereoscope v0.0.0-20201203153145-3f9a05a624d7 h1:G3LnRqHL/IIeQZTAMtDOJNYfSYsXLNCZX4DCiS0R0FY=
-github.com/anchore/stereoscope v0.0.0-20201203153145-3f9a05a624d7/go.mod h1:2Jja/4l0zYggW52og+nn0rut4i+OYjCf9vTyrM8RT4E=
-github.com/anchore/stereoscope v0.0.0-20201203222654-09e79bf5fef4 h1:XDuCqOWKyQQlKhd9kEDnyKbvSCwShKBDCsyBmD/ALYs=
-github.com/anchore/stereoscope v0.0.0-20201203222654-09e79bf5fef4/go.mod h1:/dHAFjYflH/1tzhdHAcnMCjprMch+YzHJKi59m/1KCM=
-github.com/anchore/stereoscope v0.0.0-20201210022249-091f9bddb42e h1:vHUqHTvH9/oxdDDh1fxS9Ls9gWGytKO7XbbzcQ9MBwI=
-github.com/anchore/stereoscope v0.0.0-20201210022249-091f9bddb42e/go.mod h1:/dHAFjYflH/1tzhdHAcnMCjprMch+YzHJKi59m/1KCM=
+github.com/anchore/stereoscope v0.0.0-20210104203718-4c1d1bd9a255 h1:Ng7BDr9PQTCztANogjfEdEjjWUylhlPyZPhtarIGo00=
+github.com/anchore/stereoscope v0.0.0-20210104203718-4c1d1bd9a255/go.mod h1:BMdPL0QEIYfpjQ3M7sHYZvuh6+vcomqF3TMHL8gr6Vw=
+github.com/anchore/stereoscope v0.0.0-20210105000809-428eda0b2ec6 h1:JWpsV/8x1fuCYjJmNjT43cVFblLTpO/ISDnePukiTNw=
+github.com/anchore/stereoscope v0.0.0-20210105000809-428eda0b2ec6/go.mod h1:BMdPL0QEIYfpjQ3M7sHYZvuh6+vcomqF3TMHL8gr6Vw=
+github.com/anchore/stereoscope v0.0.0-20210105001222-7beea73cb7e5 h1:NGRfS6BZKElgiMbqdoH9iQn+6oxT7CJdZYrqgwvGkWY=
+github.com/anchore/stereoscope v0.0.0-20210105001222-7beea73cb7e5/go.mod h1:BMdPL0QEIYfpjQ3M7sHYZvuh6+vcomqF3TMHL8gr6Vw=
+github.com/anchore/stereoscope v0.0.0-20210201165248-e94c52b4052d h1:2hv5NOZ0fD8tPk1UdGiW9PHxmjBmBLL+sFlhLXjjKgo=
+github.com/anchore/stereoscope v0.0.0-20210201165248-e94c52b4052d/go.mod h1:lhSEYyGLXTXMIFHAz7Ls/MNQ5EjYd5ziLxovKZp1xOs=
 github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883/go.mod h1:rCTlJbsFo29Kk6CurOXKm700vrz8f0KW0JNfpkRJY/8=
 github.com/antihax/optional v1.0.0 h1:xK2lYat7ZLaVVcIuj82J8kIro4V6kDe0AUDFboUCwcg=
 github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY=
@@ -175,8 +172,8 @@ github.com/bketelsen/crypt v0.0.3-0.20200106085610-5cbc8cc4026c/go.mod h1:MKsuJm
 github.com/blakesmith/ar v0.0.0-20190502131153-809d4375e1fb h1:m935MPodAbYS46DG4pJSv7WO+VECIWUQ7OJYSoTrMh4=
 github.com/blakesmith/ar v0.0.0-20190502131153-809d4375e1fb/go.mod h1:PkYb9DJNAwrSvRx5DYA+gUcOIgTGVMNkfSCbZM8cWpI=
 github.com/blang/semver v3.5.0+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk=
-github.com/bmatcuk/doublestar v1.3.3 h1:pVP1d49CcQQaNOl+PI6sPybIrIOD/6sux31PFdmhTH0=
-github.com/bmatcuk/doublestar v1.3.3/go.mod h1:wiQtGV+rzVYxB7WIlirSN++5HPtPlXEo9MEoZQC/PmE=
+github.com/bmatcuk/doublestar/v2 v2.0.4 h1:6I6oUiT/sU27eE2OFcWqBhL1SwjyvQuOssxT4a1yidI=
+github.com/bmatcuk/doublestar/v2 v2.0.4/go.mod h1:QMmcs3H2AUQICWhfzLXz+IYln8lRQmTZRptLie8RgRw=
 github.com/bombsimon/wsl/v2 v2.0.0/go.mod h1:mf25kr/SqFEPhhcxW1+7pxzGlW+hIl/hYTKY95VwV8U=
 github.com/bombsimon/wsl/v2 v2.2.0/go.mod h1:Azh8c3XGEJl9LyX0/sFC+CKMc7Ssgua0g+6abzXN4Pg=
 github.com/bombsimon/wsl/v3 v3.0.0/go.mod h1:st10JtZYLE4D5sC7b8xV4zTKZwAQjCH/Hy2Pm1FNZIc=
@@ -274,6 +271,7 @@ github.com/facebookincubator/nvdtools v0.1.4/go.mod h1:0/FIVnSEl9YHXLq3tKBPpKaI0
 github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4=
 github.com/fatih/color v1.9.0 h1:8xPHl4/q1VyqGIPif1F+1V3Y3lSmrq01EabUW3CoW5s=
 github.com/fatih/color v1.9.0/go.mod h1:eQcE1qtQxscV5RaZvpXrrb8Drkc3/DdQ+uUYCNjL+zU=
+github.com/fatih/set v0.2.1 h1:nn2CaJyknWE/6txyUDGwysr3G5QC6xWB/PtVjPBbeaA=
 github.com/fatih/set v0.2.1/go.mod h1:+RKtMCH+favT2+3YecHGxcc0b4KyVWA1QWWJUs4E0CI=
 github.com/fortytw2/leaktest v1.2.0/go.mod h1:jDsjWgpAGjm2CA7WthBh/CdZYEPF31XHquHwclZch5g=
 github.com/fortytw2/leaktest v1.3.0/go.mod h1:jDsjWgpAGjm2CA7WthBh/CdZYEPF31XHquHwclZch5g=
@@ -700,6 +698,8 @@ github.com/pkg/errors v0.8.1-0.20171018195549-f15c970de5b7/go.mod h1:bwawxfHBFNV
 github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
 github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pkg/profile v1.5.0 h1:042Buzk+NhDI+DeSAA62RwJL8VAuZUMQZUjCsRz1Mug=
+github.com/pkg/profile v1.5.0/go.mod h1:qBsxPvzyUincmltOk6iyRVxHYg4adc0OFOv72ZdLa18=
 github.com/pmezard/go-difflib v0.0.0-20151028094244-d8ed2627bdf0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
@@ -824,6 +824,7 @@ github.com/tetafro/godot v0.4.2/go.mod h1:/7NLHhv08H1+8DNj0MElpAACw1ajsCuf3TKNQx
 github.com/timakin/bodyclose v0.0.0-20190930140734-f7f2e9bca95e/go.mod h1:Qimiffbc6q9tBWlVV6x0P9sat/ao1xEkREYPPj9hphk=
 github.com/timakin/bodyclose v0.0.0-20200424151742-cb6215831a94 h1:ig99OeTyDwQWhPe2iw9lwfQVF1KB3Q4fpP3X7/2VBG8=
 github.com/timakin/bodyclose v0.0.0-20200424151742-cb6215831a94/go.mod h1:Qimiffbc6q9tBWlVV6x0P9sat/ao1xEkREYPPj9hphk=
+github.com/tj/assert v0.0.0-20171129193455-018094318fb0 h1:Rw8kxzWo1mr6FSaYXjQELRe88y2KdfynXdnK72rdjtA=
 github.com/tj/assert v0.0.0-20171129193455-018094318fb0/go.mod h1:mZ9/Rh9oLWpLLDRpvE+3b7gP/C2YyLFYxNmcLnPTMe0=
 github.com/tj/go-elastic v0.0.0-20171221160941-36157cbbebc2/go.mod h1:WjeM0Oo1eNAjXGDx2yma7uG2XoyRZTq1uv3M/o7imD0=
 github.com/tj/go-kinesis v0.0.0-20171128231115-08b17f58cb1b/go.mod h1:/yhzCV0xPfx6jb1bBgRFjl5lytqVqZXEaeqWP8lTEao=
@@ -850,6 +851,7 @@ github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyC
 github.com/valyala/fasthttp v1.2.0/go.mod h1:4vX61m6KN+xDduDNwXrhIAVZaZaZiQ1luJk8LWSxF3s=
 github.com/valyala/quicktemplate v1.2.0/go.mod h1:EH+4AkTd43SvgIbQHYu59/cJyxDoOVRUAfrukLPuGJ4=
 github.com/valyala/tcplisten v0.0.0-20161114210144-ceec8f93295a/go.mod h1:v3UYOV9WzVtRmSR+PDvWpU/qWl4Wa5LApYYX4ZtKbio=
+github.com/vbatts/tar-split v0.11.1/go.mod h1:LEuURwDEiWjRjwu46yU3KVGuUdVv/dcnpcEPSzR8z6g=
 github.com/vdemeester/k8s-pkg-credentialprovider v1.17.4/go.mod h1:inCTmtUdr5KJbreVojo06krnTgaeAz/Z7lynpPk/Q2c=
 github.com/vmware/govmomi v0.20.3/go.mod h1:URlwyTFZX72RmxtxuaFL2Uj3fD1JTvZdx59bHWk6aFU=
 github.com/wagoodman/go-partybus v0.0.0-20200526224238-eb215533f07d h1:KOxOL6qpmqwoPloNwi+CEgc1ayjHNOFNrvoOmeDOjDg=

internal/anchore/client.go

@@ -2,7 +2,11 @@ package anchore
 
 import (
 	"context"
+	"errors"
 	"fmt"
+	"path"
+	"strings"
+	"unicode"
 
 	"github.com/anchore/client-go/pkg/external"
 	"github.com/anchore/syft/internal"
@@ -10,11 +14,10 @@ import (
 )
 
 type Configuration struct {
-	Hostname  string
+	BaseURL   string
 	Username  string
 	Password  string
 	UserAgent string
-	Scheme    string
 }
 
 type Client struct {
@@ -29,16 +32,16 @@ func NewClient(cfg Configuration) (*Client, error) {
 		cfg.UserAgent = fmt.Sprintf("%s / %s %s", internal.ApplicationName, versionInfo.Version, versionInfo.Platform)
 	}
 
-	if cfg.Scheme == "" {
-		cfg.Scheme = "https"
+	baseURL, err := prepareBaseURLForClient(cfg.BaseURL)
+	if err != nil {
+		return nil, fmt.Errorf("unable to create client: %w", err)
 	}
 
 	return &Client{
 		config: cfg,
 		client: external.NewAPIClient(&external.Configuration{
-			Host:      cfg.Hostname,
+			BasePath:  baseURL,
 			UserAgent: cfg.UserAgent,
-			Scheme:    cfg.Scheme,
 		}),
 	}, nil
 }
@@ -56,3 +59,57 @@ func (c *Client) newRequestContext(parentContext context.Context) context.Contex
 		},
 	)
 }
+
+var ErrInvalidBaseURLInput = errors.New("invalid base URL input")
+
+func prepareBaseURLForClient(baseURL string) (string, error) {
+	if err := checkBaseURLInput(baseURL); err != nil {
+		return "", err
+	}
+
+	scheme, urlWithoutScheme := splitSchemeFromURL(baseURL)
+
+	if scheme == "" {
+		scheme = "http"
+	}
+
+	urlWithoutScheme = path.Clean(urlWithoutScheme)
+
+	const requiredSuffix = "v1"
+	if path.Base(urlWithoutScheme) != requiredSuffix {
+		urlWithoutScheme = path.Join(urlWithoutScheme, requiredSuffix)
+	}
+
+	preparedBaseURL := scheme + "://" + urlWithoutScheme
+	return preparedBaseURL, nil
+}
+
+func checkBaseURLInput(url string) error {
+	if url == "" {
+		return ErrInvalidBaseURLInput
+	}
+
+	firstCharacter := rune(url[0])
+	if !(unicode.IsLetter(firstCharacter)) {
+		return ErrInvalidBaseURLInput
+	}
+
+	return nil
+}
+
+func splitSchemeFromURL(url string) (scheme, urlWithoutScheme string) {
+	if hasScheme(url) {
+		urlParts := strings.SplitN(url, "://", 2)
+		scheme = urlParts[0]
+		urlWithoutScheme = urlParts[1]
+		return
+	}
+
+	return "", url
+}
+
+func hasScheme(url string) bool {
+	parts := strings.Split(url, "://")
+
+	return len(parts) > 1
+}

internal/anchore/client_test.go

@@ -0,0 +1,210 @@
+package anchore
+
+import "testing"
+
+func TestHasScheme(t *testing.T) {
+	cases := []struct {
+		url      string
+		expected bool
+	}{
+		{
+			url:      "http://localhost",
+			expected: true,
+		},
+		{
+			url:      "https://anchore.com:8443",
+			expected: true,
+		},
+		{
+			url:      "google.com",
+			expected: false,
+		},
+		{
+			url:      "",
+			expected: false,
+		},
+	}
+
+	for _, testCase := range cases {
+		t.Run(testCase.url, func(t *testing.T) {
+			result := hasScheme(testCase.url)
+
+			if testCase.expected != result {
+				t.Errorf("expected %t but got %t", testCase.expected, result)
+			}
+		})
+	}
+}
+
+func TestPrepareBaseURLForClient(t *testing.T) {
+	cases := []struct {
+		inputURL    string
+		expectedURL string
+		expectedErr error
+	}{
+		{
+			inputURL:    "",
+			expectedURL: "",
+			expectedErr: ErrInvalidBaseURLInput,
+		},
+		{
+			inputURL:    "localhost",
+			expectedURL: "http://localhost/v1",
+			expectedErr: nil,
+		},
+		{
+			inputURL:    "https://localhost",
+			expectedURL: "https://localhost/v1",
+			expectedErr: nil,
+		},
+		{
+			inputURL:    "https://localhost/",
+			expectedURL: "https://localhost/v1",
+			expectedErr: nil,
+		},
+		{
+			inputURL:    "https://localhost/v1/",
+			expectedURL: "https://localhost/v1",
+			expectedErr: nil,
+		},
+		{
+			inputURL:    "https://localhost/v1//",
+			expectedURL: "https://localhost/v1",
+			expectedErr: nil,
+		},
+		{
+			inputURL:    "http://something.com/platform/v1/services/anchore",
+			expectedURL: "http://something.com/platform/v1/services/anchore/v1",
+			expectedErr: nil,
+		},
+		{
+			inputURL:    "my-host:8228",
+			expectedURL: "http://my-host:8228/v1",
+			expectedErr: nil,
+		},
+		{
+			inputURL:    "v1/v1",
+			expectedURL: "http://v1/v1",
+			expectedErr: nil,
+		},
+		{
+			inputURL:    "/v1",
+			expectedURL: "",
+			expectedErr: ErrInvalidBaseURLInput,
+		},
+		{
+			inputURL:    "/imports/images",
+			expectedURL: "",
+			expectedErr: ErrInvalidBaseURLInput,
+		},
+	}
+
+	for _, testCase := range cases {
+		t.Run(testCase.inputURL, func(t *testing.T) {
+			resultURL, err := prepareBaseURLForClient(testCase.inputURL)
+			if err != testCase.expectedErr {
+				t.Errorf("expected err to be '%v' but got '%v'", testCase.expectedErr, err)
+			}
+
+			if resultURL != testCase.expectedURL {
+				t.Errorf("expected URL to be '%v' but got '%v'", testCase.expectedURL, resultURL)
+			}
+		})
+	}
+}
+
+func TestCheckBaseURLInput(t *testing.T) {
+	cases := []struct {
+		input    string
+		expected error
+	}{
+		{
+			input:    "",
+			expected: ErrInvalidBaseURLInput,
+		},
+		{
+			input:    "x",
+			expected: nil,
+		},
+		{
+			input:    "localhost:8000",
+			expected: nil,
+		},
+		{
+			input:    ":80",
+			expected: ErrInvalidBaseURLInput,
+		},
+		{
+			input:    "/v1",
+			expected: ErrInvalidBaseURLInput,
+		},
+	}
+
+	for _, testCase := range cases {
+		t.Run(testCase.input, func(t *testing.T) {
+			resultErr := checkBaseURLInput(testCase.input)
+
+			if testCase.expected != resultErr {
+				t.Errorf("expected err to be '%v' but got '%v'", testCase.expected, resultErr)
+			}
+		})
+	}
+}
+
+func TestSplitSchemeFromURL(t *testing.T) {
+	cases := []struct {
+		input                    string
+		expectedScheme           string
+		expectedURLWithoutScheme string
+	}{
+		{
+			input:                    "",
+			expectedScheme:           "",
+			expectedURLWithoutScheme: "",
+		},
+		{
+			input:                    "localhost",
+			expectedScheme:           "",
+			expectedURLWithoutScheme: "localhost",
+		},
+		{
+			input:                    "https://anchore.com/path",
+			expectedScheme:           "https",
+			expectedURLWithoutScheme: "anchore.com/path",
+		},
+		{
+			input:                    "tcp://host:1234",
+			expectedScheme:           "tcp",
+			expectedURLWithoutScheme: "host:1234",
+		},
+		{
+			input:                    "/hello",
+			expectedScheme:           "",
+			expectedURLWithoutScheme: "/hello",
+		},
+		{
+			input:                    "://host",
+			expectedScheme:           "",
+			expectedURLWithoutScheme: "host",
+		},
+		{
+			input:                    "http//localhost",
+			expectedScheme:           "",
+			expectedURLWithoutScheme: "http//localhost",
+		},
+	}
+
+	for _, testCase := range cases {
+		t.Run(testCase.input, func(t *testing.T) {
+			resultScheme, resultURLWithoutScheme := splitSchemeFromURL(testCase.input)
+
+			if testCase.expectedScheme != resultScheme {
+				t.Errorf("expected scheme to be '%s' but got '%s'", testCase.expectedScheme, resultScheme)
+			}
+
+			if testCase.expectedURLWithoutScheme != resultURLWithoutScheme {
+				t.Errorf("expected urlWithoutScheme to be '%s' but got '%s'", testCase.expectedURLWithoutScheme, resultURLWithoutScheme)
+			}
+		})
+	}
+}

internal/anchore/import.go

@@ -6,6 +6,8 @@ import (
 	"fmt"
 	"time"
 
+	"github.com/antihax/optional"
+
 	"github.com/anchore/client-go/pkg/external"
 	"github.com/anchore/stereoscope/pkg/image"
 	"github.com/anchore/syft/internal/bus"
@@ -17,6 +19,15 @@ import (
 	"github.com/wagoodman/go-progress"
 )
 
+type ImportConfig struct {
+	ImageMetadata           image.Metadata
+	SourceMetadata          source.Metadata
+	Catalog                 *pkg.Catalog
+	Distro                  *distro.Distro
+	Dockerfile              []byte
+	OverwriteExistingUpload bool
+}
+
 func importProgress(source string) (*progress.Stage, *progress.Manual) {
 	stage := &progress.Stage{}
 	prog := &progress.Manual{
@@ -39,8 +50,8 @@ func importProgress(source string) (*progress.Stage, *progress.Manual) {
 }
 
 // nolint:funlen
-func (c *Client) Import(ctx context.Context, imageMetadata image.Metadata, s source.Metadata, catalog *pkg.Catalog, d *distro.Distro, dockerfile []byte) error {
-	stage, prog := importProgress(imageMetadata.ID)
+func (c *Client) Import(ctx context.Context, cfg ImportConfig) error {
+	stage, prog := importProgress(c.config.BaseURL)
 
 	ctxWithTimeout, cancel := context.WithTimeout(ctx, time.Second*30)
 	defer cancel()
@@ -60,33 +71,37 @@ func (c *Client) Import(ctx context.Context, imageMetadata image.Metadata, s sou
 	prog.N++
 	sessionID := startOperation.Uuid
 
-	packageDigest, err := importPackageSBOM(authedCtx, c.client.ImportsApi, sessionID, s, catalog, d, stage)
+	packageDigest, err := importPackageSBOM(authedCtx, c.client.ImportsApi, sessionID, cfg.SourceMetadata, cfg.Catalog, cfg.Distro, stage)
 	if err != nil {
 		return fmt.Errorf("failed to import Package SBOM: %w", err)
 	}
 	prog.N++
 
-	manifestDigest, err := importManifest(authedCtx, c.client.ImportsApi, sessionID, imageMetadata.RawManifest, stage)
+	manifestDigest, err := importManifest(authedCtx, c.client.ImportsApi, sessionID, cfg.ImageMetadata.RawManifest, stage)
 	if err != nil {
 		return fmt.Errorf("failed to import Manifest: %w", err)
 	}
 	prog.N++
 
-	configDigest, err := importConfig(authedCtx, c.client.ImportsApi, sessionID, imageMetadata.RawConfig, stage)
+	configDigest, err := importConfig(authedCtx, c.client.ImportsApi, sessionID, cfg.ImageMetadata.RawConfig, stage)
 	if err != nil {
 		return fmt.Errorf("failed to import Config: %w", err)
 	}
 	prog.N++
 
-	dockerfileDigest, err := importDockerfile(authedCtx, c.client.ImportsApi, sessionID, dockerfile, stage)
+	dockerfileDigest, err := importDockerfile(authedCtx, c.client.ImportsApi, sessionID, cfg.Dockerfile, stage)
 	if err != nil {
 		return fmt.Errorf("failed to import Dockerfile: %w", err)
 	}
 	prog.N++
 
 	stage.Current = "finalizing"
-	imageModel := addImageModel(imageMetadata, packageDigest, manifestDigest, dockerfileDigest, configDigest, sessionID)
-	_, _, err = c.client.ImagesApi.AddImage(authedCtx, imageModel, nil)
+	imageModel := addImageModel(cfg.ImageMetadata, packageDigest, manifestDigest, dockerfileDigest, configDigest, sessionID)
+	opts := external.AddImageOpts{
+		Force: optional.NewBool(cfg.OverwriteExistingUpload),
+	}
+
+	_, _, err = c.client.ImagesApi.AddImage(authedCtx, imageModel, &opts)
 	if err != nil {
 		var detail = "no details given"
 		var openAPIErr external.GenericOpenAPIError

internal/anchore/import_config.go

@@ -3,6 +3,7 @@ package anchore
 
 import (
 	"context"
+	"encoding/json"
 	"errors"
 	"fmt"
 	"net/http"
@@ -14,15 +15,21 @@ import (
 )
 
 type configImportAPI interface {
-	ImportImageConfig(ctx context.Context, sessionID string, contents string) (external.ImageImportContentResponse, *http.Response, error)
+	ImportImageConfig(ctx context.Context, sessionID string, contents interface{}) (external.ImageImportContentResponse, *http.Response, error)
 }
 
-func importConfig(ctx context.Context, api configImportAPI, sessionID string, manifest []byte, stage *progress.Stage) (string, error) {
-	if len(manifest) > 0 {
+func importConfig(ctx context.Context, api configImportAPI, sessionID string, config []byte, stage *progress.Stage) (string, error) {
+	if len(config) > 0 {
 		log.Debug("importing image config")
 		stage.Current = "image config"
 
-		response, httpResponse, err := api.ImportImageConfig(ctx, sessionID, string(manifest))
+		// API requires an object, but we do not verify the shape of this object locally
+		var sender map[string]interface{}
+		if err := json.Unmarshal(config, &sender); err != nil {
+			return "", err
+		}
+
+		response, httpResponse, err := api.ImportImageConfig(ctx, sessionID, sender)
 		if err != nil {
 			var openAPIErr external.GenericOpenAPIError
 			if errors.As(err, &openAPIErr) {

internal/anchore/import_config_test.go

@@ -2,6 +2,7 @@ package anchore
 
 import (
 	"context"
+	"encoding/json"
 	"fmt"
 	"net/http"
 	"strings"
@@ -16,7 +17,7 @@ import (
 
 type mockConfigImportAPI struct {
 	sessionID      string
-	model          string
+	model          interface{}
 	httpResponse   *http.Response
 	err            error
 	ctx            context.Context
@@ -24,7 +25,7 @@ type mockConfigImportAPI struct {
 	wasCalled      bool
 }
 
-func (m *mockConfigImportAPI) ImportImageConfig(ctx context.Context, sessionID string, contents string) (external.ImageImportContentResponse, *http.Response, error) {
+func (m *mockConfigImportAPI) ImportImageConfig(ctx context.Context, sessionID string, contents interface{}) (external.ImageImportContentResponse, *http.Response, error) {
 	m.wasCalled = true
 	m.model = contents
 	m.sessionID = sessionID
@@ -41,16 +42,16 @@ func TestConfigImport(t *testing.T) {
 	sessionID := "my-session"
 
 	tests := []struct {
-		name         string
-		manifest     string
-		api          *mockConfigImportAPI
-		expectsError bool
-		expectsCall  bool
+		name            string
+		manifestJSONStr string
+		api             *mockConfigImportAPI
+		expectsError    bool
+		expectsCall     bool
 	}{
 
 		{
-			name:     "Go case: import works",
-			manifest: "the-manifest-contents!",
+			name:            "Go case: import works",
+			manifestJSONStr: `{ "key": "the-manifest-contents!" }`,
 			api: &mockConfigImportAPI{
 				httpResponse:   &http.Response{StatusCode: 200},
 				responseDigest: "digest!",
@@ -58,14 +59,14 @@ func TestConfigImport(t *testing.T) {
 			expectsCall: true,
 		},
 		{
-			name:        "No manifest provided",
-			manifest:    "",
-			api:         &mockConfigImportAPI{},
-			expectsCall: false,
+			name:            "No manifest provided",
+			manifestJSONStr: "",
+			api:             &mockConfigImportAPI{},
+			expectsCall:     false,
 		},
 		{
-			name:     "API returns an error",
-			manifest: "the-manifest-contents!",
+			name:            "API returns an error",
+			manifestJSONStr: `{ "key": "the-manifest-contents!" }`,
 			api: &mockConfigImportAPI{
 				err: fmt.Errorf("api error, something went wrong"),
 			},
@@ -73,8 +74,8 @@ func TestConfigImport(t *testing.T) {
 			expectsCall:  true,
 		},
 		{
-			name:     "API HTTP-level error",
-			manifest: "the-manifest-contents!",
+			name:            "API HTTP-level error",
+			manifestJSONStr: `{ "key": "the-manifest-contents!" }`,
 			api: &mockConfigImportAPI{
 				httpResponse: &http.Response{StatusCode: 404},
 			},
@@ -86,7 +87,7 @@ func TestConfigImport(t *testing.T) {
 	for _, test := range tests {
 		t.Run(test.name, func(t *testing.T) {
 
-			digest, err := importConfig(context.TODO(), test.api, sessionID, []byte(test.manifest), &progress.Stage{})
+			digest, err := importConfig(context.TODO(), test.api, sessionID, []byte(test.manifestJSONStr), &progress.Stage{})
 
 			// validate error handling
 			if err != nil && !test.expectsError {
@@ -114,7 +115,12 @@ func TestConfigImport(t *testing.T) {
 				t.Errorf("different session ID: %s != %s", test.api.sessionID, sessionID)
 			}
 
-			for _, d := range deep.Equal(test.api.model, test.manifest) {
+			var expected map[string]interface{}
+			if err := json.Unmarshal([]byte(test.manifestJSONStr), &expected); err != nil {
+				t.Fatalf("could not unmarshal expected results")
+			}
+
+			for _, d := range deep.Equal(test.api.model, expected) {
 				t.Errorf("model difference: %s", d)
 			}
 

internal/anchore/import_dockerfile.go

@@ -1,4 +1,3 @@
-// nolint:dupl
 package anchore
 
 import (

internal/anchore/import_manifest.go

@@ -3,6 +3,7 @@ package anchore
 
 import (
 	"context"
+	"encoding/json"
 	"errors"
 	"fmt"
 	"net/http"
@@ -14,7 +15,7 @@ import (
 )
 
 type manifestImportAPI interface {
-	ImportImageManifest(ctx context.Context, sessionID string, contents string) (external.ImageImportContentResponse, *http.Response, error)
+	ImportImageManifest(ctx context.Context, sessionID string, contents interface{}) (external.ImageImportContentResponse, *http.Response, error)
 }
 
 func importManifest(ctx context.Context, api manifestImportAPI, sessionID string, manifest []byte, stage *progress.Stage) (string, error) {
@@ -22,7 +23,13 @@ func importManifest(ctx context.Context, api manifestImportAPI, sessionID string
 		log.Debug("importing image manifest")
 		stage.Current = "image manifest"
 
-		response, httpResponse, err := api.ImportImageManifest(ctx, sessionID, string(manifest))
+		// API requires an object, but we do not verify the shape of this object locally
+		var sender map[string]interface{}
+		if err := json.Unmarshal(manifest, &sender); err != nil {
+			return "", err
+		}
+
+		response, httpResponse, err := api.ImportImageManifest(ctx, sessionID, sender)
 		if err != nil {
 			var openAPIErr external.GenericOpenAPIError
 			if errors.As(err, &openAPIErr) {

internal/anchore/import_manifest_test.go

@@ -2,6 +2,7 @@ package anchore
 
 import (
 	"context"
+	"encoding/json"
 	"fmt"
 	"net/http"
 	"strings"
@@ -16,7 +17,7 @@ import (
 
 type mockManifestImportAPI struct {
 	sessionID      string
-	model          string
+	model          interface{}
 	httpResponse   *http.Response
 	err            error
 	ctx            context.Context
@@ -24,7 +25,7 @@ type mockManifestImportAPI struct {
 	wasCalled      bool
 }
 
-func (m *mockManifestImportAPI) ImportImageManifest(ctx context.Context, sessionID string, contents string) (external.ImageImportContentResponse, *http.Response, error) {
+func (m *mockManifestImportAPI) ImportImageManifest(ctx context.Context, sessionID string, contents interface{}) (external.ImageImportContentResponse, *http.Response, error) {
 	m.wasCalled = true
 	m.model = contents
 	m.sessionID = sessionID
@@ -50,7 +51,7 @@ func TestManifestImport(t *testing.T) {
 
 		{
 			name:     "Go case: import works",
-			manifest: "the-manifest-contents!",
+			manifest: `{ "key": "the-config-contents!" }`,
 			api: &mockManifestImportAPI{
 				httpResponse:   &http.Response{StatusCode: 200},
 				responseDigest: "digest!",
@@ -65,7 +66,7 @@ func TestManifestImport(t *testing.T) {
 		},
 		{
 			name:     "API returns an error",
-			manifest: "the-manifest-contents!",
+			manifest: `{ "key": "the-config-contents!" }`,
 			api: &mockManifestImportAPI{
 				err: fmt.Errorf("api error, something went wrong"),
 			},
@@ -74,7 +75,7 @@ func TestManifestImport(t *testing.T) {
 		},
 		{
 			name:     "API HTTP-level error",
-			manifest: "the-manifest-contents!",
+			manifest: `{ "key": "the-config-contents!" }`,
 			api: &mockManifestImportAPI{
 				httpResponse: &http.Response{StatusCode: 404},
 			},
@@ -114,7 +115,12 @@ func TestManifestImport(t *testing.T) {
 				t.Errorf("different session ID: %s != %s", test.api.sessionID, sessionID)
 			}
 
-			for _, d := range deep.Equal(test.api.model, test.manifest) {
+			var expected map[string]interface{}
+			if err := json.Unmarshal([]byte(test.manifest), &expected); err != nil {
+				t.Fatalf("could not unmarshal expected results")
+			}
+
+			for _, d := range deep.Equal(test.api.model, expected) {
 				t.Errorf("model difference: %s", d)
 			}
 

internal/anchore/import_package_sbom_test.go

@@ -61,7 +61,7 @@ func TestPackageSbomToModel(t *testing.T) {
 		FoundBy: "foundBy",
 		Locations: []source.Location{
 			{
-				Path:         "path",
+				RealPath:     "path",
 				FileSystemID: "layerID",
 			},
 		},
@@ -139,7 +139,7 @@ func TestPackageSbomImport(t *testing.T) {
 		FoundBy: "foundBy",
 		Locations: []source.Location{
 			{
-				Path:         "path",
+				RealPath:     "path",
 				FileSystemID: "layerID",
 			},
 		},

internal/config/config.go

@@ -17,14 +17,15 @@ import (
 
 // Application is the main syft application configuration.
 type Application struct {
-	ConfigPath        string           `yaml:",omitempty"`                                               // the location where the application config was read from (either from -c or discovered while loading)
-	PresenterOpt      presenter.Option `yaml:"-"`                                                        // -o, the native Presenter.Option to use for report formatting
-	Output            string           `yaml:"output" mapstructure:"output"`                             // -o, the Presenter hint string to use for report formatting
-	ScopeOpt          source.Scope     `yaml:"-"`                                                        // -s, the native source.Scope option to use for how to catalog the container image
-	Scope             string           `yaml:"scope" mapstructure:"scope"`                               // -s, the source.Scope string hint for how to catalog the container image
-	Quiet             bool             `yaml:"quiet" mapstructure:"quiet"`                               // -q, indicates to not show any status output to stderr (ETUI or logging UI)
-	Log               logging          `yaml:"log"  mapstructure:"log"`                                  // all logging-related options
-	CliOptions        CliOnlyOptions   `yaml:"-"`                                                        // all options only available through the CLI (not via env vars or config)
+	ConfigPath        string           `yaml:",omitempty"`                   // the location where the application config was read from (either from -c or discovered while loading)
+	PresenterOpt      presenter.Option `yaml:"-"`                            // -o, the native Presenter.Option to use for report formatting
+	Output            string           `yaml:"output" mapstructure:"output"` // -o, the Presenter hint string to use for report formatting
+	ScopeOpt          source.Scope     `yaml:"-"`                            // -s, the native source.Scope option to use for how to catalog the container image
+	Scope             string           `yaml:"scope" mapstructure:"scope"`   // -s, the source.Scope string hint for how to catalog the container image
+	Quiet             bool             `yaml:"quiet" mapstructure:"quiet"`   // -q, indicates to not show any status output to stderr (ETUI or logging UI)
+	Log               logging          `yaml:"log"  mapstructure:"log"`      // all logging-related options
+	CliOptions        CliOnlyOptions   `yaml:"-"`                            // all options only available through the CLI (not via env vars or config)
+	Dev               Development      `mapstructure:"dev"`
 	CheckForAppUpdate bool             `yaml:"check-for-app-update" mapstructure:"check-for-app-update"` // whether to check for an application update on start up or not
 	Anchore           anchore          `yaml:"anchore" mapstructure:"anchore"`                           // options for interacting with Anchore Engine/Enterprise
 }
@@ -45,12 +46,18 @@ type logging struct {
 
 type anchore struct {
 	// upload options
-	UploadEnabled bool   `yaml:"upload-enabled"  mapstructure:"upload-enabled"` // whether to upload results to Anchore Engine/Enterprise (defaults to "false" unless there is the presence of -h CLI option)
-	Host          string `yaml:"host" mapstructure:"host"`                      // -H , hostname of the engine/enterprise instance to upload to
-	Path          string `yaml:"path" mapstructure:"path"`                      // override the engine/enterprise API upload path
-	Username      string `yaml:"username" mapstructure:"username"`              // -u , username to authenticate upload
-	Password      string `yaml:"password" mapstructure:"password"`              // -p , password to authenticate upload
-	Dockerfile    string `yaml:"dockerfile" mapstructure:"dockerfile"`          // -d , dockerfile to attach for upload
+	UploadEnabled          bool   `yaml:"upload-enabled"  mapstructure:"upload-enabled"`                    // whether to upload results to Anchore Engine/Enterprise (defaults to "false" unless there is the presence of -h CLI option)
+	Host                   string `yaml:"host" mapstructure:"host"`                                         // -H , hostname of the engine/enterprise instance to upload to
+	Path                   string `yaml:"path" mapstructure:"path"`                                         // override the engine/enterprise API upload path
+	Username               string `yaml:"username" mapstructure:"username"`                                 // -u , username to authenticate upload
+	Password               string `yaml:"password" mapstructure:"password"`                                 // -p , password to authenticate upload
+	Dockerfile             string `yaml:"dockerfile" mapstructure:"dockerfile"`                             // -d , dockerfile to attach for upload
+	OverwriteExistingImage bool   `yaml:"overwrite-existing-image" mapstructure:"overwrite-existing-image"` // --overwrite-existing-image , if any of the SBOM components have already been uploaded this flag will ensure they are overwritten with the current upload
+}
+
+type Development struct {
+	ProfileCPU bool `mapstructure:"profile-cpu"`
+	ProfileMem bool `mapstructure:"profile-mem"`
 }
 
 // LoadApplicationConfig populates the given viper object with application configuration discovered on disk
@@ -216,4 +223,6 @@ func setNonCliDefaultValues(v *viper.Viper) {
 	v.SetDefault("log.file", "")
 	v.SetDefault("log.structured", false)
 	v.SetDefault("check-for-app-update", true)
+	v.SetDefault("dev.profile-cpu", false)
+	v.SetDefault("dev.profile-mem", false)
 }

internal/constants.go

@@ -6,5 +6,5 @@ const (
 
 	// JSONSchemaVersion is the current schema version output by the JSON presenter
 	// This is roughly following the "SchemaVer" guidelines for versioning the JSON schema. Please see schema/json/README.md for details on how to increment.
-	JSONSchemaVersion = "1.0.0"
+	JSONSchemaVersion = "1.0.1"
 )

internal/ui/etui/ephemeral_tui.go

@@ -23,7 +23,9 @@ import (
 	"context"
 	"fmt"
 	"os"
+	"os/signal"
 	"sync"
+	"syscall"
 
 	"github.com/anchore/syft/internal/logger"
 
@@ -102,6 +104,8 @@ func OutputToEphemeralTUI(workerErrs <-chan error, subscription *partybus.Subscr
 	ctx := context.Background()
 	syftUIHandler := ui.NewHandler()
 
+	signals := interruptingSignals()
+
 eventLoop:
 	for {
 		select {
@@ -154,8 +158,23 @@ eventLoop:
 				log.Errorf("cancelled (%+v)", err)
 			}
 			break eventLoop
+		case <-signals:
+			break eventLoop
 		}
 	}
 
 	return nil
 }
+
+func interruptingSignals() chan os.Signal {
+	c := make(chan os.Signal, 1) // Note: A buffered channel is recommended for this; see https://golang.org/pkg/os/signal/#Notify
+
+	interruptions := []os.Signal{
+		syscall.SIGINT,
+		syscall.SIGTERM,
+	}
+
+	signal.Notify(c, interruptions...)
+
+	return c
+}

schema/json/schema-1.0.1.json

@@ -0,0 +1,682 @@
+{
+  "$schema": "http://json-schema.org/draft-04/schema#",
+  "$ref": "#/definitions/Document",
+  "definitions": {
+    "ApkFileRecord": {
+      "required": [
+        "path"
+      ],
+      "properties": {
+        "path": {
+          "type": "string"
+        },
+        "ownerUid": {
+          "type": "string"
+        },
+        "ownerGid": {
+          "type": "string"
+        },
+        "permissions": {
+          "type": "string"
+        },
+        "checksum": {
+          "type": "string"
+        }
+      },
+      "additionalProperties": false,
+      "type": "object"
+    },
+    "ApkMetadata": {
+      "required": [
+        "package",
+        "originPackage",
+        "maintainer",
+        "version",
+        "license",
+        "architecture",
+        "url",
+        "description",
+        "size",
+        "installedSize",
+        "pullDependencies",
+        "pullChecksum",
+        "gitCommitOfApkPort",
+        "files"
+      ],
+      "properties": {
+        "package": {
+          "type": "string"
+        },
+        "originPackage": {
+          "type": "string"
+        },
+        "maintainer": {
+          "type": "string"
+        },
+        "version": {
+          "type": "string"
+        },
+        "license": {
+          "type": "string"
+        },
+        "architecture": {
+          "type": "string"
+        },
+        "url": {
+          "type": "string"
+        },
+        "description": {
+          "type": "string"
+        },
+        "size": {
+          "type": "integer"
+        },
+        "installedSize": {
+          "type": "integer"
+        },
+        "pullDependencies": {
+          "type": "string"
+        },
+        "pullChecksum": {
+          "type": "string"
+        },
+        "gitCommitOfApkPort": {
+          "type": "string"
+        },
+        "files": {
+          "items": {
+            "$schema": "http://json-schema.org/draft-04/schema#",
+            "$ref": "#/definitions/ApkFileRecord"
+          },
+          "type": "array"
+        }
+      },
+      "additionalProperties": false,
+      "type": "object"
+    },
+    "Descriptor": {
+      "required": [
+        "name",
+        "version"
+      ],
+      "properties": {
+        "name": {
+          "type": "string"
+        },
+        "version": {
+          "type": "string"
+        }
+      },
+      "additionalProperties": false,
+      "type": "object"
+    },
+    "Distribution": {
+      "required": [
+        "name",
+        "version",
+        "idLike"
+      ],
+      "properties": {
+        "name": {
+          "type": "string"
+        },
+        "version": {
+          "type": "string"
+        },
+        "idLike": {
+          "type": "string"
+        }
+      },
+      "additionalProperties": false,
+      "type": "object"
+    },
+    "Document": {
+      "required": [
+        "artifacts",
+        "source",
+        "distro",
+        "descriptor",
+        "schema"
+      ],
+      "properties": {
+        "artifacts": {
+          "items": {
+            "$schema": "http://json-schema.org/draft-04/schema#",
+            "$ref": "#/definitions/Package"
+          },
+          "type": "array"
+        },
+        "source": {
+          "$schema": "http://json-schema.org/draft-04/schema#",
+          "$ref": "#/definitions/Source"
+        },
+        "distro": {
+          "$schema": "http://json-schema.org/draft-04/schema#",
+          "$ref": "#/definitions/Distribution"
+        },
+        "descriptor": {
+          "$schema": "http://json-schema.org/draft-04/schema#",
+          "$ref": "#/definitions/Descriptor"
+        },
+        "schema": {
+          "$schema": "http://json-schema.org/draft-04/schema#",
+          "$ref": "#/definitions/Schema"
+        }
+      },
+      "additionalProperties": false,
+      "type": "object"
+    },
+    "DpkgFileRecord": {
+      "required": [
+        "path",
+        "md5"
+      ],
+      "properties": {
+        "path": {
+          "type": "string"
+        },
+        "md5": {
+          "type": "string"
+        }
+      },
+      "additionalProperties": false,
+      "type": "object"
+    },
+    "DpkgMetadata": {
+      "required": [
+        "package",
+        "source",
+        "version",
+        "sourceVersion",
+        "architecture",
+        "maintainer",
+        "installedSize",
+        "files"
+      ],
+      "properties": {
+        "package": {
+          "type": "string"
+        },
+        "source": {
+          "type": "string"
+        },
+        "version": {
+          "type": "string"
+        },
+        "sourceVersion": {
+          "type": "string"
+        },
+        "architecture": {
+          "type": "string"
+        },
+        "maintainer": {
+          "type": "string"
+        },
+        "installedSize": {
+          "type": "integer"
+        },
+        "files": {
+          "items": {
+            "$schema": "http://json-schema.org/draft-04/schema#",
+            "$ref": "#/definitions/DpkgFileRecord"
+          },
+          "type": "array"
+        }
+      },
+      "additionalProperties": false,
+      "type": "object"
+    },
+    "GemMetadata": {
+      "required": [
+        "name",
+        "version"
+      ],
+      "properties": {
+        "name": {
+          "type": "string"
+        },
+        "version": {
+          "type": "string"
+        },
+        "files": {
+          "items": {
+            "type": "string"
+          },
+          "type": "array"
+        },
+        "authors": {
+          "items": {
+            "type": "string"
+          },
+          "type": "array"
+        },
+        "licenses": {
+          "items": {
+            "type": "string"
+          },
+          "type": "array"
+        },
+        "homepage": {
+          "type": "string"
+        }
+      },
+      "additionalProperties": false,
+      "type": "object"
+    },
+    "JavaManifest": {
+      "properties": {
+        "main": {
+          "patternProperties": {
+            ".*": {
+              "type": "string"
+            }
+          },
+          "type": "object"
+        },
+        "namedSections": {
+          "patternProperties": {
+            ".*": {
+              "patternProperties": {
+                ".*": {
+                  "type": "string"
+                }
+              },
+              "type": "object"
+            }
+          },
+          "type": "object"
+        }
+      },
+      "additionalProperties": false,
+      "type": "object"
+    },
+    "JavaMetadata": {
+      "required": [
+        "virtualPath"
+      ],
+      "properties": {
+        "virtualPath": {
+          "type": "string"
+        },
+        "manifest": {
+          "$schema": "http://json-schema.org/draft-04/schema#",
+          "$ref": "#/definitions/JavaManifest"
+        },
+        "pomProperties": {
+          "$schema": "http://json-schema.org/draft-04/schema#",
+          "$ref": "#/definitions/PomProperties"
+        }
+      },
+      "additionalProperties": false,
+      "type": "object"
+    },
+    "Location": {
+      "required": [
+        "path"
+      ],
+      "properties": {
+        "path": {
+          "type": "string"
+        },
+        "layerID": {
+          "type": "string"
+        }
+      },
+      "additionalProperties": false,
+      "type": "object"
+    },
+    "NpmPackageJSONMetadata": {
+      "required": [
+        "author",
+        "licenses",
+        "homepage",
+        "description",
+        "url"
+      ],
+      "properties": {
+        "files": {
+          "items": {
+            "type": "string"
+          },
+          "type": "array"
+        },
+        "author": {
+          "type": "string"
+        },
+        "licenses": {
+          "items": {
+            "type": "string"
+          },
+          "type": "array"
+        },
+        "homepage": {
+          "type": "string"
+        },
+        "description": {
+          "type": "string"
+        },
+        "url": {
+          "type": "string"
+        }
+      },
+      "additionalProperties": false,
+      "type": "object"
+    },
+    "Package": {
+      "required": [
+        "name",
+        "version",
+        "type",
+        "foundBy",
+        "locations",
+        "licenses",
+        "language",
+        "cpes",
+        "purl",
+        "metadataType",
+        "metadata"
+      ],
+      "properties": {
+        "name": {
+          "type": "string"
+        },
+        "version": {
+          "type": "string"
+        },
+        "type": {
+          "type": "string"
+        },
+        "foundBy": {
+          "type": "string"
+        },
+        "locations": {
+          "items": {
+            "$schema": "http://json-schema.org/draft-04/schema#",
+            "$ref": "#/definitions/Location"
+          },
+          "type": "array"
+        },
+        "licenses": {
+          "items": {
+            "type": "string"
+          },
+          "type": "array"
+        },
+        "language": {
+          "type": "string"
+        },
+        "cpes": {
+          "items": {
+            "type": "string"
+          },
+          "type": "array"
+        },
+        "purl": {
+          "type": "string"
+        },
+        "metadataType": {
+          "type": "string"
+        },
+        "metadata": {
+          "anyOf": [
+            {
+              "type": "null"
+            },
+            {
+              "$ref": "#/definitions/ApkMetadata"
+            },
+            {
+              "$ref": "#/definitions/DpkgMetadata"
+            },
+            {
+              "$ref": "#/definitions/GemMetadata"
+            },
+            {
+              "$ref": "#/definitions/JavaMetadata"
+            },
+            {
+              "$ref": "#/definitions/NpmPackageJSONMetadata"
+            },
+            {
+              "$ref": "#/definitions/PythonPackageMetadata"
+            },
+            {
+              "$ref": "#/definitions/RpmdbMetadata"
+            }
+          ]
+        }
+      },
+      "additionalProperties": false,
+      "type": "object"
+    },
+    "PomProperties": {
+      "required": [
+        "path",
+        "name",
+        "groupId",
+        "artifactId",
+        "version",
+        "extraFields"
+      ],
+      "properties": {
+        "path": {
+          "type": "string"
+        },
+        "name": {
+          "type": "string"
+        },
+        "groupId": {
+          "type": "string"
+        },
+        "artifactId": {
+          "type": "string"
+        },
+        "version": {
+          "type": "string"
+        },
+        "extraFields": {
+          "patternProperties": {
+            ".*": {
+              "type": "string"
+            }
+          },
+          "type": "object"
+        }
+      },
+      "additionalProperties": false,
+      "type": "object"
+    },
+    "PythonFileDigest": {
+      "required": [
+        "algorithm",
+        "value"
+      ],
+      "properties": {
+        "algorithm": {
+          "type": "string"
+        },
+        "value": {
+          "type": "string"
+        }
+      },
+      "additionalProperties": false,
+      "type": "object"
+    },
+    "PythonFileRecord": {
+      "required": [
+        "path"
+      ],
+      "properties": {
+        "path": {
+          "type": "string"
+        },
+        "digest": {
+          "$schema": "http://json-schema.org/draft-04/schema#",
+          "$ref": "#/definitions/PythonFileDigest"
+        },
+        "size": {
+          "type": "string"
+        }
+      },
+      "additionalProperties": false,
+      "type": "object"
+    },
+    "PythonPackageMetadata": {
+      "required": [
+        "name",
+        "version",
+        "license",
+        "author",
+        "authorEmail",
+        "platform",
+        "sitePackagesRootPath"
+      ],
+      "properties": {
+        "name": {
+          "type": "string"
+        },
+        "version": {
+          "type": "string"
+        },
+        "license": {
+          "type": "string"
+        },
+        "author": {
+          "type": "string"
+        },
+        "authorEmail": {
+          "type": "string"
+        },
+        "platform": {
+          "type": "string"
+        },
+        "files": {
+          "items": {
+            "$schema": "http://json-schema.org/draft-04/schema#",
+            "$ref": "#/definitions/PythonFileRecord"
+          },
+          "type": "array"
+        },
+        "sitePackagesRootPath": {
+          "type": "string"
+        },
+        "topLevelPackages": {
+          "items": {
+            "type": "string"
+          },
+          "type": "array"
+        }
+      },
+      "additionalProperties": false,
+      "type": "object"
+    },
+    "RpmdbFileRecord": {
+      "required": [
+        "path",
+        "mode",
+        "size",
+        "sha256"
+      ],
+      "properties": {
+        "path": {
+          "type": "string"
+        },
+        "mode": {
+          "type": "integer"
+        },
+        "size": {
+          "type": "integer"
+        },
+        "sha256": {
+          "type": "string"
+        }
+      },
+      "additionalProperties": false,
+      "type": "object"
+    },
+    "RpmdbMetadata": {
+      "required": [
+        "name",
+        "version",
+        "epoch",
+        "architecture",
+        "release",
+        "sourceRpm",
+        "size",
+        "license",
+        "vendor",
+        "files"
+      ],
+      "properties": {
+        "name": {
+          "type": "string"
+        },
+        "version": {
+          "type": "string"
+        },
+        "epoch": {
+          "type": "integer"
+        },
+        "architecture": {
+          "type": "string"
+        },
+        "release": {
+          "type": "string"
+        },
+        "sourceRpm": {
+          "type": "string"
+        },
+        "size": {
+          "type": "integer"
+        },
+        "license": {
+          "type": "string"
+        },
+        "vendor": {
+          "type": "string"
+        },
+        "files": {
+          "items": {
+            "$schema": "http://json-schema.org/draft-04/schema#",
+            "$ref": "#/definitions/RpmdbFileRecord"
+          },
+          "type": "array"
+        }
+      },
+      "additionalProperties": false,
+      "type": "object"
+    },
+    "Schema": {
+      "required": [
+        "version",
+        "url"
+      ],
+      "properties": {
+        "version": {
+          "type": "string"
+        },
+        "url": {
+          "type": "string"
+        }
+      },
+      "additionalProperties": false,
+      "type": "object"
+    },
+    "Source": {
+      "required": [
+        "type",
+        "target"
+      ],
+      "properties": {
+        "type": {
+          "type": "string"
+        },
+        "target": {
+          "additionalProperties": true
+        }
+      },
+      "additionalProperties": false,
+      "type": "object"
+    }
+  }
+}

syft/cataloger/common/generic_cataloger.go

@@ -4,7 +4,7 @@ Package common provides generic utilities used by multiple catalogers.
 package common
 
 import (
-	"strings"
+	"io"
 
 	"github.com/anchore/syft/internal/log"
 	"github.com/anchore/syft/syft/pkg"
@@ -89,7 +89,7 @@ func (c *GenericCataloger) selectFiles(resolver source.FileResolver) []source.Lo
 }
 
 // catalog takes a set of file contents and uses any configured parser functions to resolve and return discovered packages
-func (c *GenericCataloger) catalog(contents map[source.Location]string) ([]pkg.Package, error) {
+func (c *GenericCataloger) catalog(contents map[source.Location]io.ReadCloser) ([]pkg.Package, error) {
 	defer c.clear()
 
 	packages := make([]pkg.Package, 0)
@@ -101,7 +101,7 @@ func (c *GenericCataloger) catalog(contents map[source.Location]string) ([]pkg.P
 			continue
 		}
 
-		entries, err := parser(location.Path, strings.NewReader(content))
+		entries, err := parser(location.RealPath, content)
 		if err != nil {
 			// TODO: should we fail? or only log?
 			log.Warnf("cataloger '%s' failed to parse entries (location=%+v): %+v", c.upstreamCataloger, location, err)

syft/cataloger/common/generic_cataloger_test.go

@@ -4,28 +4,32 @@ import (
 	"fmt"
 	"io"
 	"io/ioutil"
+	"strings"
 	"testing"
 
-	"github.com/anchore/syft/syft/source"
-
 	"github.com/anchore/syft/syft/pkg"
+	"github.com/anchore/syft/syft/source"
 )
 
 type testResolverMock struct {
-	contents map[source.Location]string
+	contents map[source.Location]io.ReadCloser
 }
 
 func newTestResolver() *testResolverMock {
 	return &testResolverMock{
-		contents: make(map[source.Location]string),
+		contents: make(map[source.Location]io.ReadCloser),
 	}
 }
 
-func (r *testResolverMock) FileContentsByLocation(_ source.Location) (string, error) {
-	return "", fmt.Errorf("not implemented")
+func (r testResolverMock) HasPath(path string) bool {
+	panic("not implemented")
 }
 
-func (r *testResolverMock) MultipleFileContentsByLocation([]source.Location) (map[source.Location]string, error) {
+func (r *testResolverMock) FileContentsByLocation(_ source.Location) (io.ReadCloser, error) {
+	return nil, fmt.Errorf("not implemented")
+}
+
+func (r *testResolverMock) MultipleFileContentsByLocation([]source.Location) (map[source.Location]io.ReadCloser, error) {
 	return r.contents, nil
 }
 
@@ -34,7 +38,7 @@ func (r *testResolverMock) FilesByPath(paths ...string) ([]source.Location, erro
 
 	for idx, p := range paths {
 		results[idx] = source.NewLocation(p)
-		r.contents[results[idx]] = fmt.Sprintf("%s file contents!", p)
+		r.contents[results[idx]] = ioutil.NopCloser(strings.NewReader(fmt.Sprintf("%s file contents!", p)))
 	}
 
 	return results, nil
@@ -43,7 +47,7 @@ func (r *testResolverMock) FilesByPath(paths ...string) ([]source.Location, erro
 func (r *testResolverMock) FilesByGlob(_ ...string) ([]source.Location, error) {
 	path := "/a-path.txt"
 	location := source.NewLocation(path)
-	r.contents[location] = fmt.Sprintf("%s file contents!", path)
+	r.contents[location] = ioutil.NopCloser(strings.NewReader(fmt.Sprintf("%s file contents!", path)))
 	return []source.Location{location}, nil
 }
 
@@ -97,7 +101,7 @@ func TestGenericCataloger(t *testing.T) {
 
 	for _, p := range actualPkgs {
 		ref := p.Locations[0]
-		exP, ok := expectedPkgs[ref.Path]
+		exP, ok := expectedPkgs[ref.RealPath]
 		if !ok {
 			t.Errorf("missing expected pkg: ref=%+v", ref)
 			continue

syft/cataloger/cpe.go

@@ -2,6 +2,7 @@ package cataloger
 
 import (
 	"fmt"
+	"sort"
 	"strings"
 
 	"github.com/anchore/syft/internal"
@@ -12,6 +13,17 @@ import (
 // this is functionally equivalent to "*" and consistent with no input given (thus easier to test)
 const any = ""
 
+func newCPE(product, vendor, version, targetSW string) wfn.Attributes {
+	cpe := *(wfn.NewAttributesWithAny())
+	cpe.Part = "a"
+	cpe.Product = product
+	cpe.Vendor = vendor
+	cpe.Version = version
+	cpe.TargetSW = targetSW
+
+	return cpe
+}
+
 // generatePackageCPEs Create a list of CPEs, trying to guess the vendor, product tuple and setting TargetSoftware if possible
 func generatePackageCPEs(p pkg.Package) []pkg.CPE {
 	targetSws := candidateTargetSoftwareAttrs(p)
@@ -31,18 +43,14 @@ func generatePackageCPEs(p pkg.Package) []pkg.CPE {
 				keys.Add(key)
 
 				// add a new entry...
-				candidateCpe := wfn.NewAttributesWithAny()
-				candidateCpe.Part = "a"
-				candidateCpe.Product = product
-				candidateCpe.Vendor = vendor
-				candidateCpe.Version = p.Version
-				candidateCpe.TargetSW = targetSw
-
-				cpes = append(cpes, *candidateCpe)
+				c := newCPE(product, vendor, p.Version, targetSw)
+				cpes = append(cpes, c)
 			}
 		}
 	}
 
+	sort.Sort(ByCPESpecificity(cpes))
+
 	return cpes
 }
 

syft/cataloger/cpe_specificity.go

@@ -0,0 +1,31 @@
+package cataloger
+
+import "github.com/facebookincubator/nvdtools/wfn"
+
+type ByCPESpecificity []wfn.Attributes
+
+// Implementing sort.Interface
+func (c ByCPESpecificity) Len() int      { return len(c) }
+func (c ByCPESpecificity) Swap(i, j int) { c[i], c[j] = c[j], c[i] }
+func (c ByCPESpecificity) Less(i, j int) bool {
+	return countSpecifiedFields(c[i]) > countSpecifiedFields(c[j])
+}
+
+func countSpecifiedFields(cpe wfn.Attributes) int {
+	checksForSpecifiedField := []func(cpe wfn.Attributes) bool{
+		func(cpe wfn.Attributes) bool { return cpe.Part != "" },
+		func(cpe wfn.Attributes) bool { return cpe.Product != "" },
+		func(cpe wfn.Attributes) bool { return cpe.Vendor != "" },
+		func(cpe wfn.Attributes) bool { return cpe.Version != "" },
+		func(cpe wfn.Attributes) bool { return cpe.TargetSW != "" },
+	}
+
+	count := 0
+	for _, fieldIsSpecified := range checksForSpecifiedField {
+		if fieldIsSpecified(cpe) {
+			count++
+		}
+	}
+
+	return count
+}

syft/cataloger/deb/cataloger.go

@@ -8,7 +8,6 @@ import (
 	"io"
 	"path"
 	"path/filepath"
-	"strings"
 
 	"github.com/anchore/syft/syft/pkg"
 	"github.com/anchore/syft/syft/source"
@@ -47,9 +46,9 @@ func (c *Cataloger) Catalog(resolver source.Resolver) ([]pkg.Package, error) {
 			return nil, err
 		}
 
-		pkgs, err = parseDpkgStatus(strings.NewReader(dbContents))
+		pkgs, err = parseDpkgStatus(dbContents)
 		if err != nil {
-			return nil, fmt.Errorf("unable to catalog dpkg package=%+v: %w", dbLocation.Path, err)
+			return nil, fmt.Errorf("unable to catalog dpkg package=%+v: %w", dbLocation.RealPath, err)
 		}
 
 		md5ContentsByName, md5RefsByName, err := fetchMd5Contents(resolver, dbLocation, pkgs)
@@ -57,7 +56,7 @@ func (c *Cataloger) Catalog(resolver source.Resolver) ([]pkg.Package, error) {
 			return nil, fmt.Errorf("unable to find dpkg md5 contents: %w", err)
 		}
 
-		copyrightContentsByName, copyrightRefsByName, err := fetchCopyrightContents(resolver, dbLocation, pkgs)
+		copyrightContentsByName, copyrightLocationByName, err := fetchCopyrightContents(resolver, dbLocation, pkgs)
 		if err != nil {
 			return nil, fmt.Errorf("unable to find dpkg copyright contents: %w", err)
 		}
@@ -91,7 +90,7 @@ func (c *Cataloger) Catalog(resolver source.Resolver) ([]pkg.Package, error) {
 				p.Licenses = parseLicensesFromCopyright(copyrightReader)
 
 				// keep a record of the file where this was discovered
-				if ref, ok := copyrightRefsByName[p.Name]; ok {
+				if ref, ok := copyrightLocationByName[p.Name]; ok {
 					p.Locations = append(p.Locations, ref)
 				}
 			}
@@ -105,20 +104,17 @@ func fetchMd5Contents(resolver source.Resolver, dbLocation source.Location, pkgs
 
 	var md5FileMatches []source.Location
 	var nameByRef = make(map[source.Location]string)
-	parentPath := filepath.Dir(dbLocation.Path)
+	parentPath := filepath.Dir(dbLocation.RealPath)
 
 	for _, p := range pkgs {
 		// look for /var/lib/dpkg/info/NAME:ARCH.md5sums
 		name := md5Key(p)
-		md5sumPath := path.Join(parentPath, "info", name+md5sumsExt)
-		md5SumLocation := resolver.RelativeFileByPath(dbLocation, md5sumPath)
+		md5SumLocation := resolver.RelativeFileByPath(dbLocation, path.Join(parentPath, "info", name+md5sumsExt))
 
 		if md5SumLocation == nil {
 			// the most specific key did not work, fallback to just the name
 			// look for /var/lib/dpkg/info/NAME.md5sums
-			name := p.Name
-			md5sumPath := path.Join(parentPath, "info", name+md5sumsExt)
-			md5SumLocation = resolver.RelativeFileByPath(dbLocation, md5sumPath)
+			md5SumLocation = resolver.RelativeFileByPath(dbLocation, path.Join(parentPath, "info", p.Name+md5sumsExt))
 		}
 		// we should have at least one reference
 		if md5SumLocation != nil {
@@ -135,14 +131,14 @@ func fetchMd5Contents(resolver source.Resolver, dbLocation source.Location, pkgs
 
 	// organize content results and refs by a combination of name and architecture
 	var contentsByName = make(map[string]io.Reader)
-	var refsByName = make(map[string]source.Location)
+	var locationByName = make(map[string]source.Location)
 	for location, contents := range md5ContentsByLocation {
 		name := nameByRef[location]
-		contentsByName[name] = strings.NewReader(contents)
-		refsByName[name] = location
+		contentsByName[name] = contents
+		locationByName[name] = location
 	}
 
-	return contentsByName, refsByName, nil
+	return contentsByName, locationByName, nil
 }
 
 func fetchCopyrightContents(resolver source.Resolver, dbLocation source.Location, pkgs []pkg.Package) (map[string]io.Reader, map[string]source.Location, error) {
@@ -174,7 +170,7 @@ func fetchCopyrightContents(resolver source.Resolver, dbLocation source.Location
 	var refsByName = make(map[string]source.Location)
 	for location, contents := range copyrightContentsByLocation {
 		name := nameByLocation[location]
-		contentsByName[name] = strings.NewReader(contents)
+		contentsByName[name] = contents
 		refsByName[name] = location
 	}
 

syft/cataloger/deb/cataloger_test.go

@@ -54,7 +54,7 @@ func TestDpkgCataloger(t *testing.T) {
 			img, cleanup := imagetest.GetFixtureImage(t, "docker-archive", "image-dpkg")
 			defer cleanup()
 
-			s, err := source.NewFromImage(img, source.AllLayersScope, "")
+			s, err := source.NewFromImage(img, source.SquashedScope, "")
 			if err != nil {
 				t.Fatal(err)
 			}
@@ -79,7 +79,7 @@ func TestDpkgCataloger(t *testing.T) {
 				// we will test the sources separately
 				var sourcesList = make([]string, len(a.Locations))
 				for i, s := range a.Locations {
-					sourcesList[i] = s.Path
+					sourcesList[i] = s.RealPath
 				}
 				a.Locations = nil
 

syft/cataloger/deb/parse_copyright.go

@@ -3,6 +3,7 @@ package deb
 import (
 	"bufio"
 	"io"
+	"regexp"
 	"sort"
 	"strings"
 
@@ -11,14 +12,22 @@ import (
 
 // For more information see: https://www.debian.org/doc/packaging-manuals/copyright-format/1.0/#license-syntax
 
+var licensePattern = regexp.MustCompile(`^License: (?P<license>\S*)`)
+
 func parseLicensesFromCopyright(reader io.Reader) []string {
 	findings := internal.NewStringSet()
 	scanner := bufio.NewScanner(reader)
 
 	for scanner.Scan() {
 		line := scanner.Text()
-		if strings.HasPrefix(line, "License:") {
-			candidate := strings.Replace(line, "License:", "", 1)
+
+		matchesByGroup := internal.MatchCaptureGroups(licensePattern, line)
+		if len(matchesByGroup) > 0 {
+			candidate, ok := matchesByGroup["license"]
+			if !ok {
+				continue
+			}
+
 			candidate = strings.TrimSpace(candidate)
 			if strings.Contains(candidate, " or ") || strings.Contains(candidate, " and ") {
 				// this is a multi-license summary, ignore this as other recurrent license lines should cover this

syft/cataloger/deb/parse_copyright_test.go

@@ -24,6 +24,11 @@ func TestParseLicensesFromCopyright(t *testing.T) {
 			fixture:  "test-fixtures/copyright/libaudit-common",
 			expected: []string{"GPL-2", "LGPL-2.1"},
 		},
+		{
+			fixture: "test-fixtures/copyright/python",
+			// note: this should not capture #, Permission, This, see ... however it's not clear how to fix this (this is probably good enough)
+			expected: []string{"#", "Apache", "Apache-2", "Apache-2.0", "Expat", "ISC", "LGPL-2.1+", "PSF-2", "Permission", "Python", "This", "see"},
+		},
 	}
 
 	for _, test := range tests {

syft/cataloger/deb/parse_dpkg_status.go

@@ -5,9 +5,12 @@ import (
 	"errors"
 	"fmt"
 	"io"
+	"regexp"
 	"strconv"
 	"strings"
 
+	"github.com/anchore/syft/internal"
+
 	"github.com/anchore/syft/syft/pkg"
 	"github.com/mitchellh/mapstructure"
 )
@@ -45,6 +48,7 @@ func parseDpkgStatus(reader io.Reader) ([]pkg.Package, error) {
 }
 
 // parseDpkgStatusEntry returns an individual Dpkg entry, or returns errEndOfPackages if there are no more packages to parse from the reader.
+// nolint:funlen
 func parseDpkgStatusEntry(reader *bufio.Reader) (entry pkg.DpkgMetadata, err error) {
 	dpkgFields := make(map[string]interface{})
 	var retErr error
@@ -105,9 +109,29 @@ func parseDpkgStatusEntry(reader *bufio.Reader) (entry pkg.DpkgMetadata, err err
 		return pkg.DpkgMetadata{}, err
 	}
 
+	name, version := extractSourceVersion(entry.Source)
+	if version != "" {
+		entry.SourceVersion = version
+		entry.Source = name
+	}
+
 	return entry, retErr
 }
 
+// match examples:
+// "a-thing (1.2.3)"     name="a-thing" version="1.2.3"
+// "a-thing"             name="a-thing" version=""
+// ""                    name=""        version=""
+var sourceRegexp = regexp.MustCompile(`(?P<name>\S+)( \((?P<version>.*)\))?`)
+
+// If the source entry string is of the form "<name> (<version>)" then parse and return the components, if
+// of the "<name>" form, then return name and nil
+func extractSourceVersion(source string) (string, string) {
+	// special handling for the Source field since it has formatted data
+	match := internal.MatchCaptureGroups(sourceRegexp, source)
+	return match["name"], match["version"]
+}
+
 // handleNewKeyValue parse a new key-value pair from the given unprocessed line
 func handleNewKeyValue(line string) (string, interface{}, error) {
 	if i := strings.Index(line, ":"); i > 0 {

syft/cataloger/deb/parse_dpkg_status_test.go

@@ -115,3 +115,43 @@ func TestMultiplePackages(t *testing.T) {
 		})
 	}
 }
+
+func TestSourceVersionExtract(t *testing.T) {
+
+	tests := []struct {
+		name     string
+		input    string
+		expected []string
+	}{
+		{
+			name:     "name and version",
+			input:    "test (1.2.3)",
+			expected: []string{"test", "1.2.3"},
+		},
+		{
+			name:     "only name",
+			input:    "test",
+			expected: []string{"test", ""},
+		},
+		{
+			name:     "empty",
+			input:    "",
+			expected: []string{"", ""},
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			name, version := extractSourceVersion(test.input)
+
+			if name != test.expected[0] {
+				t.Errorf("mismatch name for %q : %q!=%q", test.input, name, test.expected[0])
+			}
+
+			if version != test.expected[1] {
+				t.Errorf("mismatch version for %q : %q!=%q", test.input, version, test.expected[1])
+			}
+
+		})
+	}
+}

syft/cataloger/deb/test-fixtures/copyright/python

@@ -0,0 +1,958 @@
+This package was put together by Klee Dienes <klee@debian.org> from 
+sources from ftp.python.org:/pub/python, based on the Debianization by 
+the previous maintainers Bernd S. Brentrup <bsb@uni-muenster.de> and 
+Bruce Perens. Current maintainer is Matthias Klose <doko@debian.org>. 
+
+It was downloaded from http://python.org/
+
+Copyright:
+
+Upstream Author: Guido van Rossum <guido@cwi.nl> and others.
+
+License:
+
+The following text includes the Python license and licenses and
+acknowledgements for incorporated software. The licenses can be read
+in the HTML and texinfo versions of the documentation as well, after
+installing the pythonx.y-doc package. Licenses for files not licensed
+under the Python Licenses are found at the end of this file.
+
+
+Python License
+==============
+
+A. HISTORY OF THE SOFTWARE
+==========================
+
+Python was created in the early 1990s by Guido van Rossum at Stichting
+Mathematisch Centrum (CWI, see http://www.cwi.nl) in the Netherlands
+as a successor of a language called ABC.  Guido remains Python's
+principal author, although it includes many contributions from others.
+
+In 1995, Guido continued his work on Python at the Corporation for
+National Research Initiatives (CNRI, see http://www.cnri.reston.va.us)
+in Reston, Virginia where he released several versions of the
+software.
+
+In May 2000, Guido and the Python core development team moved to
+BeOpen.com to form the BeOpen PythonLabs team.  In October of the same
+year, the PythonLabs team moved to Digital Creations (now Zope
+Corporation, see http://www.zope.com).  In 2001, the Python Software
+Foundation (PSF, see http://www.python.org/psf/) was formed, a
+non-profit organization created specifically to own Python-related
+Intellectual Property.  Zope Corporation is a sponsoring member of
+the PSF.
+
+All Python releases are Open Source (see http://www.opensource.org for
+the Open Source Definition).  Historically, most, but not all, Python
+releases have also been GPL-compatible; the table below summarizes
+the various releases.
+
+    Release         Derived     Year        Owner       GPL-
+                    from                                compatible? (1)
+
+    0.9.0 thru 1.2              1991-1995   CWI         yes
+    1.3 thru 1.5.2  1.2         1995-1999   CNRI        yes
+    1.6             1.5.2       2000        CNRI        no
+    2.0             1.6         2000        BeOpen.com  no
+    1.6.1           1.6         2001        CNRI        yes (2)
+    2.1             2.0+1.6.1   2001        PSF         no
+    2.0.1           2.0+1.6.1   2001        PSF         yes
+    2.1.1           2.1+2.0.1   2001        PSF         yes
+    2.2             2.1.1       2001        PSF         yes
+    2.1.2           2.1.1       2002        PSF         yes
+    2.1.3           2.1.2       2002        PSF         yes
+    2.2.1           2.2         2002        PSF         yes
+    2.2.2           2.2.1       2002        PSF         yes
+    2.2.3           2.2.2       2003        PSF         yes
+    2.3             2.2.2       2002-2003   PSF         yes
+    2.3.1           2.3         2002-2003   PSF         yes
+    2.3.2           2.3.1       2002-2003   PSF         yes
+    2.3.3           2.3.2       2002-2003   PSF         yes
+    2.3.4           2.3.3       2004        PSF         yes
+    2.3.5           2.3.4       2005        PSF         yes
+    2.4             2.3         2004        PSF         yes
+    2.4.1           2.4         2005        PSF         yes
+    2.4.2           2.4.1       2005        PSF         yes
+    2.4.3           2.4.2       2006        PSF         yes
+    2.5             2.4         2006        PSF         yes
+    2.5.1           2.5         2007        PSF         yes
+    2.5.2           2.5.1       2008        PSF         yes
+    2.5.3           2.5.2       2008        PSF         yes
+    2.6             2.5         2008        PSF         yes
+    2.6.1           2.6         2008        PSF         yes
+    2.6.2           2.6.1       2009        PSF         yes
+    2.6.3           2.6.2       2009        PSF         yes
+    2.6.4           2.6.3       2009        PSF         yes
+    2.6.5           2.6.4       2010        PSF         yes
+    2.7             2.6         2010        PSF         yes
+
+Footnotes:
+
+(1) GPL-compatible doesn't mean that we're distributing Python under
+    the GPL.  All Python licenses, unlike the GPL, let you distribute
+    a modified version without making your changes open source.  The
+    GPL-compatible licenses make it possible to combine Python with
+    other software that is released under the GPL; the others don't.
+
+(2) According to Richard Stallman, 1.6.1 is not GPL-compatible,
+    because its license has a choice of law clause.  According to
+    CNRI, however, Stallman's lawyer has told CNRI's lawyer that 1.6.1
+    is "not incompatible" with the GPL.
+
+Thanks to the many outside volunteers who have worked under Guido's
+direction to make these releases possible.
+
+
+B. TERMS AND CONDITIONS FOR ACCESSING OR OTHERWISE USING PYTHON
+===============================================================
+
+PYTHON SOFTWARE FOUNDATION LICENSE VERSION 2
+--------------------------------------------
+
+1. This LICENSE AGREEMENT is between the Python Software Foundation
+("PSF"), and the Individual or Organization ("Licensee") accessing and
+otherwise using this software ("Python") in source or binary form and
+its associated documentation.
+
+2. Subject to the terms and conditions of this License Agreement, PSF
+hereby grants Licensee a nonexclusive, royalty-free, world-wide
+license to reproduce, analyze, test, perform and/or display publicly,
+prepare derivative works, distribute, and otherwise use Python
+alone or in any derivative version, provided, however, that PSF's
+License Agreement and PSF's notice of copyright, i.e., "Copyright (c)
+2001, 2002, 2003, 2004, 2005, 2006, 2007 Python Software Foundation; 
+All Rights Reserved" are retained in Python alone or in any derivative 
+version prepared by Licensee.
+
+3. In the event Licensee prepares a derivative work that is based on
+or incorporates Python or any part thereof, and wants to make
+the derivative work available to others as provided herein, then
+Licensee hereby agrees to include in any such work a brief summary of
+the changes made to Python.
+
+4. PSF is making Python available to Licensee on an "AS IS"
+basis.  PSF MAKES NO REPRESENTATIONS OR WARRANTIES, EXPRESS OR
+IMPLIED.  BY WAY OF EXAMPLE, BUT NOT LIMITATION, PSF MAKES NO AND
+DISCLAIMS ANY REPRESENTATION OR WARRANTY OF MERCHANTABILITY OR FITNESS
+FOR ANY PARTICULAR PURPOSE OR THAT THE USE OF PYTHON WILL NOT
+INFRINGE ANY THIRD PARTY RIGHTS.
+
+5. PSF SHALL NOT BE LIABLE TO LICENSEE OR ANY OTHER USERS OF PYTHON
+FOR ANY INCIDENTAL, SPECIAL, OR CONSEQUENTIAL DAMAGES OR LOSS AS
+A RESULT OF MODIFYING, DISTRIBUTING, OR OTHERWISE USING PYTHON,
+OR ANY DERIVATIVE THEREOF, EVEN IF ADVISED OF THE POSSIBILITY THEREOF.
+
+6. This License Agreement will automatically terminate upon a material
+breach of its terms and conditions.
+
+7. Nothing in this License Agreement shall be deemed to create any
+relationship of agency, partnership, or joint venture between PSF and
+Licensee.  This License Agreement does not grant permission to use PSF
+trademarks or trade name in a trademark sense to endorse or promote
+products or services of Licensee, or any third party.
+
+8. By copying, installing or otherwise using Python, Licensee
+agrees to be bound by the terms and conditions of this License
+Agreement.
+
+
+BEOPEN.COM LICENSE AGREEMENT FOR PYTHON 2.0
+-------------------------------------------
+
+BEOPEN PYTHON OPEN SOURCE LICENSE AGREEMENT VERSION 1
+
+1. This LICENSE AGREEMENT is between BeOpen.com ("BeOpen"), having an
+office at 160 Saratoga Avenue, Santa Clara, CA 95051, and the
+Individual or Organization ("Licensee") accessing and otherwise using
+this software in source or binary form and its associated
+documentation ("the Software").
+
+2. Subject to the terms and conditions of this BeOpen Python License
+Agreement, BeOpen hereby grants Licensee a non-exclusive,
+royalty-free, world-wide license to reproduce, analyze, test, perform
+and/or display publicly, prepare derivative works, distribute, and
+otherwise use the Software alone or in any derivative version,
+provided, however, that the BeOpen Python License is retained in the
+Software, alone or in any derivative version prepared by Licensee.
+
+3. BeOpen is making the Software available to Licensee on an "AS IS"
+basis.  BEOPEN MAKES NO REPRESENTATIONS OR WARRANTIES, EXPRESS OR
+IMPLIED.  BY WAY OF EXAMPLE, BUT NOT LIMITATION, BEOPEN MAKES NO AND
+DISCLAIMS ANY REPRESENTATION OR WARRANTY OF MERCHANTABILITY OR FITNESS
+FOR ANY PARTICULAR PURPOSE OR THAT THE USE OF THE SOFTWARE WILL NOT
+INFRINGE ANY THIRD PARTY RIGHTS.
+
+4. BEOPEN SHALL NOT BE LIABLE TO LICENSEE OR ANY OTHER USERS OF THE
+SOFTWARE FOR ANY INCIDENTAL, SPECIAL, OR CONSEQUENTIAL DAMAGES OR LOSS
+AS A RESULT OF USING, MODIFYING OR DISTRIBUTING THE SOFTWARE, OR ANY
+DERIVATIVE THEREOF, EVEN IF ADVISED OF THE POSSIBILITY THEREOF.
+
+5. This License Agreement will automatically terminate upon a material
+breach of its terms and conditions.
+
+6. This License Agreement shall be governed by and interpreted in all
+respects by the law of the State of California, excluding conflict of
+law provisions.  Nothing in this License Agreement shall be deemed to
+create any relationship of agency, partnership, or joint venture
+between BeOpen and Licensee.  This License Agreement does not grant
+permission to use BeOpen trademarks or trade names in a trademark
+sense to endorse or promote products or services of Licensee, or any
+third party.  As an exception, the "BeOpen Python" logos available at
+http://www.pythonlabs.com/logos.html may be used according to the
+permissions granted on that web page.
+
+7. By copying, installing or otherwise using the software, Licensee
+agrees to be bound by the terms and conditions of this License
+Agreement.
+
+
+CNRI LICENSE AGREEMENT FOR PYTHON 1.6.1
+---------------------------------------
+
+1. This LICENSE AGREEMENT is between the Corporation for National
+Research Initiatives, having an office at 1895 Preston White Drive,
+Reston, VA 20191 ("CNRI"), and the Individual or Organization
+("Licensee") accessing and otherwise using Python 1.6.1 software in
+source or binary form and its associated documentation.
+
+2. Subject to the terms and conditions of this License Agreement, CNRI
+hereby grants Licensee a nonexclusive, royalty-free, world-wide
+license to reproduce, analyze, test, perform and/or display publicly,
+prepare derivative works, distribute, and otherwise use Python 1.6.1
+alone or in any derivative version, provided, however, that CNRI's
+License Agreement and CNRI's notice of copyright, i.e., "Copyright (c)
+1995-2001 Corporation for National Research Initiatives; All Rights
+Reserved" are retained in Python 1.6.1 alone or in any derivative
+version prepared by Licensee.  Alternately, in lieu of CNRI's License
+Agreement, Licensee may substitute the following text (omitting the
+quotes): "Python 1.6.1 is made available subject to the terms and
+conditions in CNRI's License Agreement.  This Agreement together with
+Python 1.6.1 may be located on the Internet using the following
+unique, persistent identifier (known as a handle): 1895.22/1013.  This
+Agreement may also be obtained from a proxy server on the Internet
+using the following URL: http://hdl.handle.net/1895.22/1013".
+
+3. In the event Licensee prepares a derivative work that is based on
+or incorporates Python 1.6.1 or any part thereof, and wants to make
+the derivative work available to others as provided herein, then
+Licensee hereby agrees to include in any such work a brief summary of
+the changes made to Python 1.6.1.
+
+4. CNRI is making Python 1.6.1 available to Licensee on an "AS IS"
+basis.  CNRI MAKES NO REPRESENTATIONS OR WARRANTIES, EXPRESS OR
+IMPLIED.  BY WAY OF EXAMPLE, BUT NOT LIMITATION, CNRI MAKES NO AND
+DISCLAIMS ANY REPRESENTATION OR WARRANTY OF MERCHANTABILITY OR FITNESS
+FOR ANY PARTICULAR PURPOSE OR THAT THE USE OF PYTHON 1.6.1 WILL NOT
+INFRINGE ANY THIRD PARTY RIGHTS.
+
+5. CNRI SHALL NOT BE LIABLE TO LICENSEE OR ANY OTHER USERS OF PYTHON
+1.6.1 FOR ANY INCIDENTAL, SPECIAL, OR CONSEQUENTIAL DAMAGES OR LOSS AS
+A RESULT OF MODIFYING, DISTRIBUTING, OR OTHERWISE USING PYTHON 1.6.1,
+OR ANY DERIVATIVE THEREOF, EVEN IF ADVISED OF THE POSSIBILITY THEREOF.
+
+6. This License Agreement will automatically terminate upon a material
+breach of its terms and conditions.
+
+7. This License Agreement shall be governed by the federal
+intellectual property law of the United States, including without
+limitation the federal copyright law, and, to the extent such
+U.S. federal law does not apply, by the law of the Commonwealth of
+Virginia, excluding Virginia's conflict of law provisions.
+Notwithstanding the foregoing, with regard to derivative works based
+on Python 1.6.1 that incorporate non-separable material that was
+previously distributed under the GNU General Public License (GPL), the
+law of the Commonwealth of Virginia shall govern this License
+Agreement only as to issues arising under or with respect to
+Paragraphs 4, 5, and 7 of this License Agreement.  Nothing in this
+License Agreement shall be deemed to create any relationship of
+agency, partnership, or joint venture between CNRI and Licensee.  This
+License Agreement does not grant permission to use CNRI trademarks or
+trade name in a trademark sense to endorse or promote products or
+services of Licensee, or any third party.
+
+8. By clicking on the "ACCEPT" button where indicated, or by copying,
+installing or otherwise using Python 1.6.1, Licensee agrees to be
+bound by the terms and conditions of this License Agreement.
+
+        ACCEPT
+
+
+CWI LICENSE AGREEMENT FOR PYTHON 0.9.0 THROUGH 1.2
+--------------------------------------------------
+
+Copyright (c) 1991 - 1995, Stichting Mathematisch Centrum Amsterdam,
+The Netherlands.  All rights reserved.
+
+Permission to use, copy, modify, and distribute this software and its
+documentation for any purpose and without fee is hereby granted,
+provided that the above copyright notice appear in all copies and that
+both that copyright notice and this permission notice appear in
+supporting documentation, and that the name of Stichting Mathematisch
+Centrum or CWI not be used in advertising or publicity pertaining to
+distribution of the software without specific, written prior
+permission.
+
+STICHTING MATHEMATISCH CENTRUM DISCLAIMS ALL WARRANTIES WITH REGARD TO
+THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND
+FITNESS, IN NO EVENT SHALL STICHTING MATHEMATISCH CENTRUM BE LIABLE
+FOR ANY SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
+ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT
+OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+
+
+Licenses and Acknowledgements for Incorporated Software
+=======================================================
+
+Mersenne Twister
+----------------
+
+The `_random' module includes code based on a download from
+`http://www.math.keio.ac.jp/~matumoto/MT2002/emt19937ar.html'.  The
+following are the verbatim comments from the original code:
+
+     A C-program for MT19937, with initialization improved 2002/1/26.
+     Coded by Takuji Nishimura and Makoto Matsumoto.
+
+     Before using, initialize the state by using init_genrand(seed)
+     or init_by_array(init_key, key_length).
+
+     Copyright (C) 1997 - 2002, Makoto Matsumoto and Takuji Nishimura,
+     All rights reserved.
+
+     Redistribution and use in source and binary forms, with or without
+     modification, are permitted provided that the following conditions
+     are met:
+
+      1. Redistributions of source code must retain the above copyright
+         notice, this list of conditions and the following disclaimer.
+
+      2. Redistributions in binary form must reproduce the above copyright
+         notice, this list of conditions and the following disclaimer in the
+         documentation and/or other materials provided with the distribution.
+
+      3. The names of its contributors may not be used to endorse or promote
+         products derived from this software without specific prior written
+         permission.
+
+     THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+     "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+     LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+     A PARTICULAR PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE COPYRIGHT
+     OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+     SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED
+     TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+     PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
+     LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
+     NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+     SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+     Any feedback is very welcome.
+     http://www.math.keio.ac.jp/matumoto/emt.html
+     email: matumoto@math.keio.ac.jp
+
+
+Sockets
+-------
+
+The `socket' module uses the functions, `getaddrinfo', and
+`getnameinfo', which are coded in separate source files from the WIDE
+Project, `http://www.wide.ad.jp/about/index.html'.
+
+     Copyright (C) 1995, 1996, 1997, and 1998 WIDE Project.
+     All rights reserved.
+
+     Redistribution and use in source and binary forms, with or without
+     modification, are permitted provided that the following conditions
+     are met:
+     1. Redistributions of source code must retain the above copyright
+        notice, this list of conditions and the following disclaimer.
+     2. Redistributions in binary form must reproduce the above copyright
+        notice, this list of conditions and the following disclaimer in the
+        documentation and/or other materials provided with the distribution.
+     3. Neither the name of the project nor the names of its contributors
+        may be used to endorse or promote products derived from this software
+        without specific prior written permission.
+
+     THIS SOFTWARE IS PROVIDED BY THE PROJECT AND CONTRIBUTORS ``AS IS'' AND
+     GAI_ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+     IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+     ARE DISCLAIMED.  IN NO EVENT SHALL THE PROJECT OR CONTRIBUTORS BE LIABLE
+     FOR GAI_ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+     CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+     SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+     INTERRUPTION) HOWEVER CAUSED AND ON GAI_ANY THEORY OF LIABILITY, WHETHER
+     IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+     ARISING IN GAI_ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED
+     OF THE POSSIBILITY OF SUCH DAMAGE.
+
+
+Floating point exception control
+--------------------------------
+
+The source for the `fpectl' module includes the following notice:
+
+     ---------------------------------------------------------------------  
+    /                       Copyright (c) 1996.                           \ 
+   |          The Regents of the University of California.                 |
+   |                        All rights reserved.                           |
+   |                                                                       |
+   |   Permission to use, copy, modify, and distribute this software for   |
+   |   any purpose without fee is hereby granted, provided that this en-   |
+   |   tire notice is included in all copies of any software which is or   |
+   |   includes  a  copy  or  modification  of  this software and in all   |
+   |   copies of the supporting documentation for such software.           |
+   |                                                                       |
+   |   This  work was produced at the University of California, Lawrence   |
+   |   Livermore National Laboratory under  contract  no.  W-7405-ENG-48   |
+   |   between  the  U.S.  Department  of  Energy and The Regents of the   |
+   |   University of California for the operation of UC LLNL.              |
+   |                                                                       |
+   |                              DISCLAIMER                               |
+   |                                                                       |
+   |   This  software was prepared as an account of work sponsored by an   |
+   |   agency of the United States Government. Neither the United States   |
+   |   Government  nor the University of California nor any of their em-   |
+   |   ployees, makes any warranty, express or implied, or  assumes  any   |
+   |   liability  or  responsibility  for the accuracy, completeness, or   |
+   |   usefulness of any information,  apparatus,  product,  or  process   |
+   |   disclosed,   or  represents  that  its  use  would  not  infringe   |
+   |   privately-owned rights. Reference herein to any specific  commer-   |
+   |   cial  products,  process,  or  service  by trade name, trademark,   |
+   |   manufacturer, or otherwise, does not  necessarily  constitute  or   |
+   |   imply  its endorsement, recommendation, or favoring by the United   |
+   |   States Government or the University of California. The views  and   |
+   |   opinions  of authors expressed herein do not necessarily state or   |
+   |   reflect those of the United States Government or  the  University   |
+   |   of  California,  and shall not be used for advertising or product   |
+    \  endorsement purposes.                                              / 
+     ---------------------------------------------------------------------
+
+
+Cookie management
+-----------------
+
+The `Cookie' module contains the following notice:
+
+      Copyright 2000 by Timothy O'Malley <timo@alum.mit.edu>
+
+                     All Rights Reserved
+
+      Permission to use, copy, modify, and distribute this software
+      and its documentation for any purpose and without fee is hereby
+      granted, provided that the above copyright notice appear in all
+      copies and that both that copyright notice and this permission
+      notice appear in supporting documentation, and that the name of
+      Timothy O'Malley  not be used in advertising or publicity
+      pertaining to distribution of the software without specific, written
+      prior permission.
+
+      Timothy O'Malley DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS
+      SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
+      AND FITNESS, IN NO EVENT SHALL Timothy O'Malley BE LIABLE FOR
+      ANY SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+      WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS,
+      WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS
+      ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
+      PERFORMANCE OF THIS SOFTWARE.
+
+
+Execution tracing
+-----------------
+
+The `trace' module contains the following notice:
+
+      portions copyright 2001, Autonomous Zones Industries, Inc., all rights...
+      err...  reserved and offered to the public under the terms of the
+      Python 2.2 license.
+      Author: Zooko O'Whielacronx
+      http://zooko.com/
+      mailto:zooko@zooko.com
+
+      Copyright 2000, Mojam Media, Inc., all rights reserved.
+      Author: Skip Montanaro
+
+      Copyright 1999, Bioreason, Inc., all rights reserved.
+      Author: Andrew Dalke
+
+      Copyright 1995-1997, Automatrix, Inc., all rights reserved.
+      Author: Skip Montanaro
+
+      Copyright 1991-1995, Stichting Mathematisch Centrum, all rights reserved.
+
+      Permission to use, copy, modify, and distribute this Python software and
+      its associated documentation for any purpose without fee is hereby
+      granted, provided that the above copyright notice appears in all copies,
+      and that both that copyright notice and this permission notice appear in
+      supporting documentation, and that the name of neither Automatrix,
+      Bioreason or Mojam Media be used in advertising or publicity pertaining
+      to distribution of the software without specific, written prior
+      permission.
+
+
+UUencode and UUdecode functions
+-------------------------------
+
+The `uu' module contains the following notice:
+
+      Copyright 1994 by Lance Ellinghouse
+      Cathedral City, California Republic, United States of America.
+                             All Rights Reserved
+      Permission to use, copy, modify, and distribute this software and its
+      documentation for any purpose and without fee is hereby granted,
+      provided that the above copyright notice appear in all copies and that
+      both that copyright notice and this permission notice appear in
+      supporting documentation, and that the name of Lance Ellinghouse
+      not be used in advertising or publicity pertaining to distribution
+      of the software without specific, written prior permission.
+      LANCE ELLINGHOUSE DISCLAIMS ALL WARRANTIES WITH REGARD TO
+      THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND
+      FITNESS, IN NO EVENT SHALL LANCE ELLINGHOUSE CENTRUM BE LIABLE
+      FOR ANY SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+      WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
+      ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT
+      OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+
+      Modified by Jack Jansen, CWI, July 1995:
+      - Use binascii module to do the actual line-by-line conversion
+        between ascii and binary. This results in a 1000-fold speedup. The C
+        version is still 5 times faster, though.
+      - Arguments more compliant with python standard
+
+
+XML Remote Procedure Calls
+--------------------------
+
+The `xmlrpclib' module contains the following notice:
+
+          The XML-RPC client interface is
+
+      Copyright (c) 1999-2002 by Secret Labs AB
+      Copyright (c) 1999-2002 by Fredrik Lundh
+
+      By obtaining, using, and/or copying this software and/or its
+      associated documentation, you agree that you have read, understood,
+      and will comply with the following terms and conditions:
+
+      Permission to use, copy, modify, and distribute this software and
+      its associated documentation for any purpose and without fee is
+      hereby granted, provided that the above copyright notice appears in
+      all copies, and that both that copyright notice and this permission
+      notice appear in supporting documentation, and that the name of
+      Secret Labs AB or the author not be used in advertising or publicity
+      pertaining to distribution of the software without specific, written
+      prior permission.
+
+      SECRET LABS AB AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH REGARD
+      TO THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANT-
+      ABILITY AND FITNESS.  IN NO EVENT SHALL SECRET LABS AB OR THE AUTHOR
+      BE LIABLE FOR ANY SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY
+      DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS,
+      WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS
+      ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE
+      OF THIS SOFTWARE.
+
+Licenses for Software linked to
+===============================
+
+Note that the choice of GPL compatibility outlined above doesn't extend
+to modules linked to particular libraries, since they change the
+effective License of the module binary.
+
+
+GNU Readline
+------------
+
+The 'readline' module makes use of GNU Readline.
+
+      The GNU Readline Library is free software; you can redistribute it
+      and/or modify it under the terms of the GNU General Public License as
+      published by the Free Software Foundation; either version 2, or (at
+      your option) any later version.
+
+      On Debian systems, you can find the complete statement in
+      /usr/share/doc/readline-common/copyright'. A copy of the GNU General
+      Public License is available in /usr/share/common-licenses/GPL-2'.
+
+
+OpenSSL
+-------
+
+The '_ssl' module makes use of OpenSSL.
+
+      The OpenSSL toolkit stays under a dual license, i.e. both the
+      conditions of the OpenSSL License and the original SSLeay license
+      apply to the toolkit. Actually both licenses are BSD-style Open
+      Source licenses. Note that both licenses are incompatible with
+      the GPL.
+
+      On Debian systems, you can find the complete license text in
+      /usr/share/doc/openssl/copyright'.
+
+
+Files with other licenses than the Python License
+-------------------------------------------------
+
+Files: Lib/profile.py Lib/pstats.py
+Copyright: Disney Enterprises, Inc.  All Rights Reserved.
+License: # Licensed to PSF under a Contributor Agreement
+  Licensed under the Apache License, Version 2.0 (the "License");
+  you may not use this file except in compliance with the License.
+  You may obtain a copy of the License at
+
+  http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
+  either express or implied.  See the License for the specific language
+  overning permissions and limitations under the License.
+
+  On Debian systems, the Apache 2.0 license can be found in
+  /usr/share/common-licenses/Apache-2.0.
+
+Files: Modules/zlib/*
+Copyright: (C) 1995-2010 Jean-loup Gailly and Mark Adler
+License: This software is provided 'as-is', without any express or implied
+  warranty.  In no event will the authors be held liable for any damages
+  arising from the use of this software.
+
+  Permission is granted to anyone to use this software for any purpose,
+  including commercial applications, and to alter it and redistribute it
+  freely, subject to the following restrictions:
+
+  1. The origin of this software must not be misrepresented; you must not
+     claim that you wrote the original software. If you use this software
+     in a product, an acknowledgment in the product documentation would be
+     appreciated but is not required.
+  2. Altered source versions must be plainly marked as such, and must not be
+     misrepresented as being the original software.
+  3. This notice may not be removed or altered from any source distribution.
+
+  Jean-loup Gailly        Mark Adler
+  jloup@gzip.org          madler@alumni.caltech.edu
+
+ If you use the zlib library in a product, we would appreciate *not* receiving
+ lengthy legal documents to sign.  The sources are provided for free but without
+ warranty of any kind.  The library has been entirely written by Jean-loup
+ Gailly and Mark Adler; it does not include third-party code.
+
+Files: Modules/_ctypes/libffi/*
+Copyright: Copyright (C) 1996-2009 Red Hat, Inc and others.
+License: Permission is hereby granted, free of charge, to any person obtaining
+   a copy of this software and associated documentation files (the
+   ``Software''), to deal in the Software without restriction, including
+   without limitation the rights to use, copy, modify, merge, publish,
+   distribute, sublicense, and/or sell copies of the Software, and to
+   permit persons to whom the Software is furnished to do so, subject to
+   the following conditions:
+
+   The above copyright notice and this permission notice shall be included
+   in all copies or substantial portions of the Software.
+
+   THE SOFTWARE IS PROVIDED ``AS IS'', WITHOUT WARRANTY OF ANY KIND,
+   EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+   MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+   NONINFRINGEMENT.  IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+   HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+   WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+   OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+   DEALINGS IN THE SOFTWARE.
+
+   Documentation:
+   Permission is granted to copy, distribute and/or modify this document
+   under the terms of the GNU General Public License as published by the
+   Free Software Foundation; either version 2, or (at your option) any
+   later version.  A copy of the license is included in the
+   section entitled ``GNU General Public License''.
+
+Files: Modules/expat/*
+Copyright: Copyright (c) 1998, 1999, 2000 Thai Open Source Software Center Ltd
+  and Clark Cooper
+  Copyright (c) 2001, 2002, 2003, 2004, 2005, 2006 Expat maintainers
+License: Permission is hereby granted, free of charge, to any person obtaining
+  a copy of this software and associated documentation files (the
+  "Software"), to deal in the Software without restriction, including
+  without limitation the rights to use, copy, modify, merge, publish,
+  distribute, sublicense, and/or sell copies of the Software, and to
+  permit persons to whom the Software is furnished to do so, subject to
+  the following conditions:
+ 
+  The above copyright notice and this permission notice shall be included
+  in all copies or substantial portions of the Software.
+ 
+  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+  EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+  MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+  IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
+  CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
+  TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+  SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+Files: Misc/python-mode.el
+Copyright: Copyright (C) 1992,1993,1994  Tim Peters
+License: This software is provided as-is, without express or implied
+        warranty.  Permission to use, copy, modify, distribute or sell this
+        software, without fee, for any purpose and by any individual or
+        organization, is hereby granted, provided that the above copyright
+        notice and this paragraph appear in all copies.
+
+Files: PC/_subprocess.c
+Copyright: Copyright (c) 2004 by Fredrik Lundh <fredrik@pythonware.com>
+        Copyright (c) 2004 by Secret Labs AB, http://www.pythonware.com
+        Copyright (c) 2004 by Peter Astrand <astrand@lysator.liu.se>
+License:
+ * Permission to use, copy, modify, and distribute this software and
+ * its associated documentation for any purpose and without fee is
+ * hereby granted, provided that the above copyright notice appears in
+ * all copies, and that both that copyright notice and this permission
+ * notice appear in supporting documentation, and that the name of the
+ * authors not be used in advertising or publicity pertaining to
+ * distribution of the software without specific, written prior
+ * permission.
+ *
+ * THE AUTHORS DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE,
+ * INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS.
+ * IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY SPECIAL, INDIRECT OR
+ * CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS
+ * OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT,
+ * NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION
+ * WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+
+Files: PC/winsound.c
+Copyright: Copyright (c) 1999 Toby Dickenson
+License:  * Permission to use this software in any way is granted without
+ * fee, provided that the copyright notice above appears in all
+ * copies. This software is provided "as is" without any warranty.
+ */
+
+/* Modified by Guido van Rossum */
+/* Beep added by Mark Hammond */
+/* Win9X Beep and platform identification added by Uncle Timmy */
+
+Files: Lib/ensurepip/_bundled/setuptools-*.whl
+License: see above, some license as Python.
+
+Files: Lib/ensurepip/_bundled/pip-*.whl
+Copyright: Copyright © 2008-2013 The pip developers:
+    Alex Grönholm
+    Alex Morega
+    Alexandre Conrad
+    Andrey Bulgakov
+    Antti Kaihola
+    Armin Ronacher
+    Aziz Köksal
+    Ben Rosser
+    Brian Rosner
+    Carl Meyer
+    Chris McDonough
+    Christian Oudard
+    Clay McClure
+    Cody Soyland
+    Daniel Holth
+    Dave Abrahams
+    David (d1b)
+    Dmitry Gladkov
+    Donald Stufft
+    Francesco
+    Geoffrey Lehée
+    Georgi Valkov
+    Hugo Lopes Tavares
+    Ian Bicking
+    Igor Sobreira
+    Ionel Maries Cristian
+    Jakub Vysoky
+    James Cleveland
+    Jannis Leidel
+    Jay Graves
+    John-Scott Atlakson
+    Jon Parise
+    Jonas Nockert
+    Josh Bronson
+    Kamal Bin Mustafa
+    Kelsey Hightower
+    Kenneth Belitzky
+    Kumar McMillan
+    Luke Macken
+    Masklinn
+    Marc Abramowitz
+    Marcus Smith
+    Markus Hametner
+    Matt Maker
+    Maxime R.
+    Miguel Araujo
+    Nick Stenning
+    Nowell Strite
+    Oliver Tonnhofer
+    Olivier Girardot
+    Patrick Jenkins
+    Paul Moore
+    Paul Nasrat
+    Paul Oswald
+    Paul van der Linden
+    Peter Waller
+    Phil Whelan
+    Piet Delport
+    Przemek Wrzos
+    Qiangning Hong
+    Rafael Caricio
+    Rene Dudfield
+    Roey Berman
+    Ronny Pfannschmidt
+    Rory McCann
+    Simon Cross
+    Stavros Korokithakis
+    Thomas Fenzl
+    Thomas Johansson
+    Vinay Sajip
+    Vitaly Babiy
+    W Trevor King
+    Wil Tan
+    Hsiaoming Yang
+License: Expat
+    Permission is hereby granted, free of charge, to any person obtaining
+    a copy of this software and associated documentation files (the
+    "Software"), to deal in the Software without restriction, including
+    without limitation the rights to use, copy, modify, merge, publish,
+    distribute, sublicense, and/or sell copies of the Software, and to
+    permit persons to whom the Software is furnished to do so, subject to
+    the following conditions:
+    .
+    The above copyright notice and this permission notice shall be
+    included in all copies or substantial portions of the Software.
+    .
+    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+    EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+    MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+    NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+    LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+    OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+    WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+Files: Lib/ensurepip/_bundled/pip-*.whl/pip/_vendor/ipaddress.py
+Copyright: 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010, 2011, 2012, 2013, 2014 Python Software Foundation; All Rights Reserved
+           2007 Google Inc.
+License: PSF-2
+
+Files: Lib/ensurepip/_bundled/pip-*.whl/pip/_vendor/retrying.py
+Copyright: (c) 2013-2014, <ray@blacklocus.com>
+           (c) 2014, Derek Wilson <dwilson@domaintools.com>
+           (c) 2014, Alex Kuang <akuang@bizo.com>
+License: Apache-2
+
+License: Apache-2
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+ .
+    http://www.apache.org/licenses/LICENSE-2.0
+ .
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+ .
+ On Debian-based systems the full text of the Apache version 2.0 license
+ can be found in /usr/share/common-licenses/Apache-2.0.
+
+Files: Lib/ensurepip/_bundled/pip-*.whl/pip/_vendor/six.py
+Copyright: Copyright (c) 2010-2014 Benjamin Peterson
+License: Expat
+
+Files: Lib/ensurepip/_bundled/pip-*.whl/pip/_vendor/cachecontrol
+Copyright: 2015 Eric Larson
+License: Apache-2.0
+ Copyright 2015 Eric Larson
+ .
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+ .
+     http://www.apache.org/licenses/LICENSE-2.0
+ .
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+ implied.
+ .
+ See the License for the specific language governing permissions and
+ limitations under the License.
+ .
+ On Debian systems, the license is available at
+ /usr/share/common-licenses/Apache-2.0
+
+Files: Lib/ensurepip/_bundled/pip-*.whl/pip/_vendor/colorama
+Files: Lib/ensurepip/_bundled/pip-*.whl/pip/_vendor/distlib
+Copyright: Copyright (C) 2012-2013 The Python Software Foundation
+           Copyright (C) 2012-2013 Vinay Sajip
+License: Python
+
+Files: Lib/ensurepip/_bundled/pip-*.whl/pip/_vendor/html5lib
+html5lib is Copyright (c) 2006 The Authors
+Authors:
+  James Graham - james@hoppipolla.co.uk
+  Anne van Kesteren - annevankesteren@gmail.com
+  Lachlan Hunt - lachlan.hunt@lachy.id.au
+  Matt McDonald - kanashii@kanashii.ca
+  Sam Ruby - rubys@intertwingly.net
+  Ian Hickson (Google) - ian@hixie.ch
+  Thomas Broyer - t.broyer@gmail.com
+
+License:
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of
+this software and associated documentation files (the "Software"), to deal in
+the Software without restriction, including without limitation the rights to
+use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies
+of the Software, and to permit persons to whom the Software is furnished to do
+so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+
+Files: Lib/ensurepip/_bundled/pip-*.whl/pip/_vendor/lockfile
+Copyright:
+    © 2014–2015 OpenStack Foundation <openstack-dev@lists.openstack.org>
+    © 2007–2012 Skip Montanaro <skip@pobox.com>
+    © 2008–2014 Ben Finney <ben+python@benfinney.id.au>
+License: Expat
+
+Files: Lib/ensurepip/_bundled/pip-*.whl/pip/_vendor/packaging
+Copyright: Copyright (C) 2014 Donald Stufft
+           Copyright (C) 2012-2013 Vinay Sajip
+License: Apache-2.0
+
+Files: Lib/ensurepip/_bundled/pip-*.whl/pip/_vendor/progress
+Copyright: Copyright (c) 2012 Giorgos Verigakis <verigak@gmail.com>
+License: ISC
+
+Files: Lib/ensurepip/_bundled/pip-*.whl/pip/_vendor/requests
+Copyright: 2016, Kenneth Reitz
+License: Apache
+
+Files: Lib/ensurepip/_bundled/pip-*.whl/pip/_vendor/requests/packages/urllib3/*
+Copyright: 2008-2016, Andrey Petrov
+License: Expat
+
+Files: Lib/ensurepip/_bundled/pip-*.whl/pip/_vendor/requests/packages/urllib3/packages/ordered_dict.py
+Copyright: 2009, Raymond Hettinger
+License: Expat
+
+Files: Lib/ensurepip/_bundled/pip-*.whl/pip/_vendor/requests/packages/urllib3/packages/ssl_match_hostname/__init__.py
+Copyright: 2011, Python Software Foundation
+License: PSF-2
+
+Files: Lib/ensurepip/_bundled/pip-*.whl/pip/_vendor/requests/packages/chardet/*
+Copyright: 2006-2008, Mark Pilgrim
+           2012-2013, Ian Cordasco
+License: LGPL-2.1+

syft/cataloger/java/archive_parser.go

@@ -178,36 +178,40 @@ func (j *archiveParser) discoverPkgsFromPomProperties(parentPkg *pkg.Package) ([
 			return nil, fmt.Errorf("failed to parse pom.properties (%s): %w", j.virtualPath, err)
 		}
 
-		if propsObj != nil {
-			if propsObj.Version != "" && propsObj.ArtifactID != "" {
-				// TODO: if there is no parentPkg (no java manifest) one of these poms could be the parent. We should discover the right parent and attach the correct info accordingly to each discovered package
+		if propsObj == nil {
+			continue
+		}
 
-				// keep the artifact name within the virtual path if this package does not match the parent package
-				vPathSuffix := ""
-				if !strings.HasPrefix(propsObj.ArtifactID, parentPkg.Name) {
-					vPathSuffix += ":" + propsObj.ArtifactID
-				}
-				virtualPath := j.virtualPath + vPathSuffix
+		if propsObj.Version != "" && propsObj.ArtifactID != "" {
+			// TODO: if there is no parentPkg (no java manifest) one of these poms could be the parent. We should discover the right parent and attach the correct info accordingly to each discovered package
 
-				// discovered props = new package
-				p := pkg.Package{
-					Name:         propsObj.ArtifactID,
-					Version:      propsObj.Version,
-					Language:     pkg.Java,
-					Type:         pkg.JavaPkg,
-					MetadataType: pkg.JavaMetadataType,
-					Metadata: pkg.JavaMetadata{
-						VirtualPath:   virtualPath,
-						PomProperties: propsObj,
-						Parent:        parentPkg,
-					},
-				}
+			// keep the artifact name within the virtual path if this package does not match the parent package
+			vPathSuffix := ""
+			if parentPkg != nil && !strings.HasPrefix(propsObj.ArtifactID, parentPkg.Name) {
+				vPathSuffix += ":" + propsObj.ArtifactID
+			}
+			virtualPath := j.virtualPath + vPathSuffix
 
-				pkgKey := uniquePkgKey(&p)
+			// discovered props = new package
+			p := pkg.Package{
+				Name:         propsObj.ArtifactID,
+				Version:      propsObj.Version,
+				Language:     pkg.Java,
+				Type:         pkg.JavaPkg,
+				MetadataType: pkg.JavaMetadataType,
+				Metadata: pkg.JavaMetadata{
+					VirtualPath:   virtualPath,
+					PomProperties: propsObj,
+					Parent:        parentPkg,
+				},
+			}
 
-				// the name/version pair matches...
-				matchesParentPkg := pkgKey == parentKey
+			pkgKey := uniquePkgKey(&p)
 
+			// the name/version pair matches...
+			matchesParentPkg := pkgKey == parentKey
+
+			if parentPkg != nil {
 				// the virtual path matches...
 				matchesParentPkg = matchesParentPkg || parentPkg.Metadata.(pkg.JavaMetadata).VirtualPath == virtualPath
 
@@ -231,11 +235,13 @@ func (j *archiveParser) discoverPkgsFromPomProperties(parentPkg *pkg.Package) ([
 						parentMetadata.PomProperties = propsObj
 						parentPkg.Metadata = parentMetadata
 					}
-				} else if !j.discoveredPkgs.Contains(pkgKey) {
-					// only keep packages we haven't seen yet (and are not related to the parent package)
-					pkgs = append(pkgs, p)
 				}
 			}
+
+			if !matchesParentPkg && !j.discoveredPkgs.Contains(pkgKey) {
+				// only keep packages we haven't seen yet (and are not related to the parent package)
+				pkgs = append(pkgs, p)
+			}
 		}
 	}
 	return pkgs, nil

syft/cataloger/javascript/parse_package_json.go

@@ -7,6 +7,8 @@ import (
 	"io"
 	"regexp"
 
+	"github.com/anchore/syft/internal/log"
+
 	"github.com/anchore/syft/internal"
 
 	"github.com/mitchellh/mapstructure"
@@ -172,6 +174,11 @@ func parsePackageJSON(_ string, reader io.Reader) ([]pkg.Package, error) {
 			return nil, fmt.Errorf("failed to parse package.json file: %w", err)
 		}
 
+		if !p.hasNameAndVersionValues() {
+			log.Debug("encountered package.json file without a name and/or version field, ignoring this file")
+			return nil, nil
+		}
+
 		licenses, err := licensesFromJSON(p)
 		if err != nil {
 			return nil, fmt.Errorf("failed to parse package.json file: %w", err)
@@ -195,3 +202,7 @@ func parsePackageJSON(_ string, reader io.Reader) ([]pkg.Package, error) {
 
 	return packages, nil
 }
+
+func (p PackageJSON) hasNameAndVersionValues() bool {
+	return p.Name != "" && p.Version != ""
+}

syft/cataloger/javascript/parse_package_json_test.go

@@ -142,3 +142,20 @@ func TestParsePackageJSON(t *testing.T) {
 		})
 	}
 }
+
+func TestParsePackageJSON_Partial(t *testing.T) { // see https://github.com/anchore/syft/issues/311
+	const fixtureFile = "test-fixtures/pkg-json/package-partial.json"
+	fixture, err := os.Open(fixtureFile)
+	if err != nil {
+		t.Fatalf("failed to open fixture: %+v", err)
+	}
+
+	actual, err := parsePackageJSON("", fixture)
+	if err != nil {
+		t.Fatalf("failed to parse package-lock.json: %+v", err)
+	}
+
+	if actualCount := len(actual); actualCount != 0 {
+		t.Errorf("no packages should've been returned (but got %d packages)", actualCount)
+	}
+}

syft/cataloger/javascript/test-fixtures/pkg-json/package-partial.json

@@ -0,0 +1,5 @@
+{
+  "sideEffects": false,
+  "module": "../../esm/fp/isSaturday/index.js",
+  "typings": "../../typings.d.ts"
+}

syft/cataloger/python/package_cataloger.go

@@ -3,10 +3,6 @@ package python
 import (
 	"bufio"
 	"fmt"
-	"path/filepath"
-	"strings"
-
-	"github.com/anchore/syft/internal/log"
 
 	"github.com/anchore/syft/syft/pkg"
 
@@ -14,8 +10,9 @@ import (
 )
 
 const (
-	eggMetadataGlob   = "**/*egg-info/PKG-INFO"
-	wheelMetadataGlob = "**/*dist-info/METADATA"
+	eggMetadataGlob     = "**/*egg-info/PKG-INFO"
+	eggFileMetadataGlob = "**/*.egg-info"
+	wheelMetadataGlob   = "**/*dist-info/METADATA"
 )
 
 type PackageCataloger struct{}
@@ -32,37 +29,73 @@ func (c *PackageCataloger) Name() string {
 
 // Catalog is given an object to resolve file references and content, this function returns any discovered Packages after analyzing python egg and wheel installations.
 func (c *PackageCataloger) Catalog(resolver source.Resolver) ([]pkg.Package, error) {
-	// nolint:prealloc
-	var fileMatches []source.Location
-
-	for _, glob := range []string{eggMetadataGlob, wheelMetadataGlob} {
-		matches, err := resolver.FilesByGlob(glob)
-		if err != nil {
-			return nil, fmt.Errorf("failed to find files by glob: %s", glob)
-		}
-		fileMatches = append(fileMatches, matches...)
+	entries, err := c.getPackageEntries(resolver)
+	if err != nil {
+		return nil, err
 	}
 
-	var pkgs []pkg.Package
-	for _, location := range fileMatches {
-		p, err := c.catalogEggOrWheel(resolver, location)
+	var packages []pkg.Package
+	for _, entry := range entries {
+		p, err := c.catalogEggOrWheel(entry)
 		if err != nil {
-			return nil, fmt.Errorf("unable to catalog python package=%+v: %w", location.Path, err)
+			return nil, fmt.Errorf("unable to catalog python package=%+v: %w", entry.Metadata.Location.RealPath, err)
 		}
 		if p != nil {
-			pkgs = append(pkgs, *p)
+			packages = append(packages, *p)
 		}
 	}
-	return pkgs, nil
+
+	return packages, nil
+}
+
+// getPackageEntries fetches the contents for all python packages within the given resolver.
+func (c *PackageCataloger) getPackageEntries(resolver source.Resolver) ([]*packageEntry, error) {
+	var metadataLocations []source.Location
+
+	// find all primary record paths
+	matches, err := resolver.FilesByGlob(eggMetadataGlob, eggFileMetadataGlob, wheelMetadataGlob)
+	if err != nil {
+		return nil, fmt.Errorf("failed to find files by glob: %w", err)
+	}
+	metadataLocations = append(metadataLocations, matches...)
+
+	// for every primary record path, craft all secondary record paths and build a request object to gather all file contents for each record
+	requester := source.NewContentRequester()
+	entries := make([]*packageEntry, len(metadataLocations))
+	for i, metadataLocation := range metadataLocations {
+		// build the entry to process (holding only path information)
+		entry := newPackageEntry(resolver, metadataLocation)
+
+		// populate the data onto the requester object
+		requester.Add(&entry.Metadata)
+		if entry.FileRecord != nil {
+			requester.Add(entry.FileRecord)
+		}
+		if entry.TopPackage != nil {
+			requester.Add(entry.TopPackage)
+		}
+
+		// keep track of the entry for later package processing
+		entries[i] = entry
+	}
+
+	// return the set of entries and execute the request for fetching contents
+	return entries, requester.Execute(resolver)
 }
 
 // catalogEggOrWheel takes the primary metadata file reference and returns the python package it represents.
-func (c *PackageCataloger) catalogEggOrWheel(resolver source.Resolver, metadataLocation source.Location) (*pkg.Package, error) {
-	metadata, sources, err := c.assembleEggOrWheelMetadata(resolver, metadataLocation)
+func (c *PackageCataloger) catalogEggOrWheel(entry *packageEntry) (*pkg.Package, error) {
+	metadata, sources, err := c.assembleEggOrWheelMetadata(entry)
 	if err != nil {
 		return nil, err
 	}
 
+	// This can happen for Python 2.7 where it is reported from an egg-info, but Python is
+	// the actual runtime, it isn't a "package". The special-casing here allows to skip it
+	if metadata.Name == "Python" {
+		return nil, nil
+	}
+
 	var licenses []string
 	if metadata.License != "" {
 		licenses = []string{metadata.License}
@@ -81,26 +114,45 @@ func (c *PackageCataloger) catalogEggOrWheel(resolver source.Resolver, metadataL
 	}, nil
 }
 
-// fetchRecordFiles finds a corresponding RECORD file for the given python package metadata file and returns the set of file records contained.
-func (c *PackageCataloger) fetchRecordFiles(resolver source.Resolver, metadataLocation source.Location) (files []pkg.PythonFileRecord, sources []source.Location, err error) {
+// assembleEggOrWheelMetadata discovers and accumulates python package metadata from multiple file sources and returns a single metadata object as well as a list of files where the metadata was derived from.
+func (c *PackageCataloger) assembleEggOrWheelMetadata(entry *packageEntry) (*pkg.PythonPackageMetadata, []source.Location, error) {
+	var sources = []source.Location{entry.Metadata.Location}
+
+	metadata, err := parseWheelOrEggMetadata(entry.Metadata.Location.RealPath, entry.Metadata.Contents)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	// attach any python files found for the given wheel/egg installation
+	r, s, err := c.processRecordFiles(entry.FileRecord)
+	if err != nil {
+		return nil, nil, err
+	}
+	sources = append(sources, s...)
+	metadata.Files = r
+
+	// attach any top-level package names found for the given wheel/egg installation
+	p, s, err := c.processTopLevelPackages(entry.TopPackage)
+	if err != nil {
+		return nil, nil, err
+	}
+	sources = append(sources, s...)
+	metadata.TopLevelPackages = p
+
+	return &metadata, sources, nil
+}
+
+// processRecordFiles takes a corresponding RECORD file for the given python package metadata file and returns the set of file records contained.
+func (c *PackageCataloger) processRecordFiles(entry *source.FileData) (files []pkg.PythonFileRecord, sources []source.Location, err error) {
 	// we've been given a file reference to a specific wheel METADATA file. note: this may be for a directory
 	// or for an image... for an image the METADATA file may be present within multiple layers, so it is important
 	// to reconcile the RECORD path to the same layer (or the next adjacent lower layer).
 
-	// lets find the RECORD file relative to the directory where the METADATA file resides (in path AND layer structure)
-	recordPath := filepath.Join(filepath.Dir(metadataLocation.Path), "RECORD")
-	recordRef := resolver.RelativeFileByPath(metadataLocation, recordPath)
-
-	if recordRef != nil {
-		sources = append(sources, *recordRef)
-
-		recordContents, err := resolver.FileContentsByLocation(*recordRef)
-		if err != nil {
-			return nil, nil, err
-		}
+	if entry != nil {
+		sources = append(sources, entry.Location)
 
 		// parse the record contents
-		records, err := parseWheelOrEggRecord(strings.NewReader(recordContents))
+		records, err := parseWheelOrEggRecord(entry.Contents)
 		if err != nil {
 			return nil, nil, err
 		}
@@ -110,26 +162,15 @@ func (c *PackageCataloger) fetchRecordFiles(resolver source.Resolver, metadataLo
 	return files, sources, nil
 }
 
-// fetchTopLevelPackages finds a corresponding top_level.txt file for the given python package metadata file and returns the set of package names contained.
-func (c *PackageCataloger) fetchTopLevelPackages(resolver source.Resolver, metadataLocation source.Location) (pkgs []string, sources []source.Location, err error) {
-	// a top_level.txt file specifies the python top-level packages (provided by this python package) installed into site-packages
-	parentDir := filepath.Dir(metadataLocation.Path)
-	topLevelPath := filepath.Join(parentDir, "top_level.txt")
-	topLevelRef := resolver.RelativeFileByPath(metadataLocation, topLevelPath)
-
-	if topLevelRef == nil {
-		log.Warnf("missing python package top_level.txt (package=%q)", metadataLocation.Path)
+// processTopLevelPackages takes a corresponding top_level.txt file for the given python package metadata file and returns the set of package names contained.
+func (c *PackageCataloger) processTopLevelPackages(entry *source.FileData) (pkgs []string, sources []source.Location, err error) {
+	if entry == nil {
 		return nil, nil, nil
 	}
 
-	sources = append(sources, *topLevelRef)
+	sources = append(sources, entry.Location)
 
-	topLevelContents, err := resolver.FileContentsByLocation(*topLevelRef)
-	if err != nil {
-		return nil, nil, err
-	}
-
-	scanner := bufio.NewScanner(strings.NewReader(topLevelContents))
+	scanner := bufio.NewScanner(entry.Contents)
 	for scanner.Scan() {
 		pkgs = append(pkgs, scanner.Text())
 	}
@@ -140,36 +181,3 @@ func (c *PackageCataloger) fetchTopLevelPackages(resolver source.Resolver, metad
 
 	return pkgs, sources, nil
 }
-
-// assembleEggOrWheelMetadata discovers and accumulates python package metadata from multiple file sources and returns a single metadata object as well as a list of files where the metadata was derived from.
-func (c *PackageCataloger) assembleEggOrWheelMetadata(resolver source.Resolver, metadataLocation source.Location) (*pkg.PythonPackageMetadata, []source.Location, error) {
-	var sources = []source.Location{metadataLocation}
-
-	metadataContents, err := resolver.FileContentsByLocation(metadataLocation)
-	if err != nil {
-		return nil, nil, err
-	}
-
-	metadata, err := parseWheelOrEggMetadata(metadataLocation.Path, strings.NewReader(metadataContents))
-	if err != nil {
-		return nil, nil, err
-	}
-
-	// attach any python files found for the given wheel/egg installation
-	r, s, err := c.fetchRecordFiles(resolver, metadataLocation)
-	if err != nil {
-		return nil, nil, err
-	}
-	sources = append(sources, s...)
-	metadata.Files = r
-
-	// attach any top-level package names found for the given wheel/egg installation
-	p, s, err := c.fetchTopLevelPackages(resolver, metadataLocation)
-	if err != nil {
-		return nil, nil, err
-	}
-	sources = append(sources, s...)
-	metadata.TopLevelPackages = p
-
-	return &metadata, sources, nil
-}

syft/cataloger/python/package_cataloger_test.go

@@ -1,131 +1,27 @@
 package python
 
 import (
-	"fmt"
-	"io"
-	"io/ioutil"
-	"os"
-	"strings"
 	"testing"
 
-	"github.com/anchore/syft/syft/source"
-
 	"github.com/anchore/syft/syft/pkg"
+	"github.com/anchore/syft/syft/source"
 	"github.com/go-test/deep"
 )
 
-type pythonTestResolverMock struct {
-	metadataReader io.Reader
-	recordReader   io.Reader
-	topLevelReader io.Reader
-	metadataRef    *source.Location
-	recordRef      *source.Location
-	topLevelRef    *source.Location
-	contents       map[source.Location]string
-}
-
-func newTestResolver(metaPath, recordPath, topPath string) *pythonTestResolverMock {
-	metadataReader, err := os.Open(metaPath)
-	if err != nil {
-		panic(fmt.Errorf("failed to open metadata: %+v", err))
-	}
-
-	var recordReader io.Reader
-	if recordPath != "" {
-		recordReader, err = os.Open(recordPath)
-		if err != nil {
-			panic(fmt.Errorf("failed to open record: %+v", err))
-		}
-	}
-
-	var topLevelReader io.Reader
-	if topPath != "" {
-		topLevelReader, err = os.Open(topPath)
-		if err != nil {
-			panic(fmt.Errorf("failed to open top level: %+v", err))
-		}
-	}
-
-	var recordRef *source.Location
-	if recordReader != nil {
-		ref := source.NewLocation("test-fixtures/dist-info/RECORD")
-		recordRef = &ref
-	}
-	var topLevelRef *source.Location
-	if topLevelReader != nil {
-		ref := source.NewLocation("test-fixtures/dist-info/top_level.txt")
-		topLevelRef = &ref
-	}
-	metadataRef := source.NewLocation("test-fixtures/dist-info/METADATA")
-	return &pythonTestResolverMock{
-		recordReader:   recordReader,
-		metadataReader: metadataReader,
-		topLevelReader: topLevelReader,
-		metadataRef:    &metadataRef,
-		recordRef:      recordRef,
-		topLevelRef:    topLevelRef,
-		contents:       make(map[source.Location]string),
-	}
-}
-
-func (r *pythonTestResolverMock) FileContentsByLocation(ref source.Location) (string, error) {
-	switch {
-	case r.topLevelRef != nil && ref.Path == r.topLevelRef.Path:
-		b, err := ioutil.ReadAll(r.topLevelReader)
-		if err != nil {
-			return "", err
-		}
-		return string(b), nil
-	case ref.Path == r.metadataRef.Path:
-		b, err := ioutil.ReadAll(r.metadataReader)
-		if err != nil {
-			return "", err
-		}
-		return string(b), nil
-	case ref.Path == r.recordRef.Path:
-		b, err := ioutil.ReadAll(r.recordReader)
-		if err != nil {
-			return "", err
-		}
-		return string(b), nil
-	}
-	return "", fmt.Errorf("invalid value given")
-}
-
-func (r *pythonTestResolverMock) MultipleFileContentsByLocation(_ []source.Location) (map[source.Location]string, error) {
-	return nil, fmt.Errorf("not implemented")
-}
-
-func (r *pythonTestResolverMock) FilesByPath(_ ...string) ([]source.Location, error) {
-	return nil, fmt.Errorf("not implemented")
-}
-
-func (r *pythonTestResolverMock) FilesByGlob(_ ...string) ([]source.Location, error) {
-	return nil, fmt.Errorf("not implemented")
-}
-func (r *pythonTestResolverMock) RelativeFileByPath(_ source.Location, path string) *source.Location {
-	switch {
-	case strings.Contains(path, "RECORD"):
-		return r.recordRef
-	case strings.Contains(path, "top_level.txt"):
-		return r.topLevelRef
-	default:
-		panic(fmt.Errorf("invalid RelativeFileByPath value given: %q", path))
-	}
-}
-
 func TestPythonPackageWheelCataloger(t *testing.T) {
 	tests := []struct {
-		MetadataFixture string
-		RecordFixture   string
-		TopLevelFixture string
-		ExpectedPackage pkg.Package
+		name            string
+		fixtures        []string
+		expectedPackage pkg.Package
 	}{
 		{
-			MetadataFixture: "test-fixtures/egg-info/PKG-INFO",
-			RecordFixture:   "test-fixtures/egg-info/RECORD",
-			TopLevelFixture: "test-fixtures/egg-info/top_level.txt",
-			ExpectedPackage: pkg.Package{
+			name: "egg-info directory",
+			fixtures: []string{
+				"test-fixtures/egg-info/PKG-INFO",
+				"test-fixtures/egg-info/RECORD",
+				"test-fixtures/egg-info/top_level.txt",
+			},
+			expectedPackage: pkg.Package{
 				Name:         "requests",
 				Version:      "2.22.0",
 				Type:         pkg.PythonPkg,
@@ -154,10 +50,13 @@ func TestPythonPackageWheelCataloger(t *testing.T) {
 			},
 		},
 		{
-			MetadataFixture: "test-fixtures/dist-info/METADATA",
-			RecordFixture:   "test-fixtures/dist-info/RECORD",
-			TopLevelFixture: "test-fixtures/dist-info/top_level.txt",
-			ExpectedPackage: pkg.Package{
+			name: "dist-info directory",
+			fixtures: []string{
+				"test-fixtures/dist-info/METADATA",
+				"test-fixtures/dist-info/RECORD",
+				"test-fixtures/dist-info/top_level.txt",
+			},
+			expectedPackage: pkg.Package{
 				Name:         "Pygments",
 				Version:      "2.6.1",
 				Type:         pkg.PythonPkg,
@@ -187,8 +86,9 @@ func TestPythonPackageWheelCataloger(t *testing.T) {
 		{
 			// in cases where the metadata file is available and the record is not we should still record there is a package
 			// additionally empty top_level.txt files should not result in an error
-			MetadataFixture: "test-fixtures/partial.dist-info/METADATA",
-			ExpectedPackage: pkg.Package{
+			name:     "partial dist-info directory",
+			fixtures: []string{"test-fixtures/partial.dist-info/METADATA"},
+			expectedPackage: pkg.Package{
 				Name:         "Pygments",
 				Version:      "2.6.1",
 				Type:         pkg.PythonPkg,
@@ -207,34 +107,78 @@ func TestPythonPackageWheelCataloger(t *testing.T) {
 				},
 			},
 		},
+		{
+			name:     "egg-info regular file",
+			fixtures: []string{"test-fixtures/test.egg-info"},
+			expectedPackage: pkg.Package{
+				Name:         "requests",
+				Version:      "2.22.0",
+				Type:         pkg.PythonPkg,
+				Language:     pkg.Python,
+				Licenses:     []string{"Apache 2.0"},
+				FoundBy:      "python-package-cataloger",
+				MetadataType: pkg.PythonPackageMetadataType,
+				Metadata: pkg.PythonPackageMetadata{
+					Name:                 "requests",
+					Version:              "2.22.0",
+					License:              "Apache 2.0",
+					Platform:             "UNKNOWN",
+					Author:               "Kenneth Reitz",
+					AuthorEmail:          "me@kennethreitz.org",
+					SitePackagesRootPath: "test-fixtures",
+				},
+			},
+		},
 	}
 
 	for _, test := range tests {
-		t.Run(test.MetadataFixture, func(t *testing.T) {
-			resolver := newTestResolver(test.MetadataFixture, test.RecordFixture, test.TopLevelFixture)
+		t.Run(test.name, func(t *testing.T) {
+			resolver := source.NewMockResolverForPaths(test.fixtures...)
 
-			// note that the source is the record ref created by the resolver mock... attach the expected values
-			test.ExpectedPackage.Locations = []source.Location{*resolver.metadataRef}
-			if resolver.recordRef != nil {
-				test.ExpectedPackage.Locations = append(test.ExpectedPackage.Locations, *resolver.recordRef)
+			locations, err := resolver.FilesByPath(test.fixtures...)
+			if err != nil {
+				t.Fatal(err)
 			}
 
-			if resolver.topLevelRef != nil {
-				test.ExpectedPackage.Locations = append(test.ExpectedPackage.Locations, *resolver.topLevelRef)
-			}
-			// end patching expected values with runtime data...
+			test.expectedPackage.Locations = locations
 
-			pyPkgCataloger := NewPythonPackageCataloger()
-
-			actual, err := pyPkgCataloger.catalogEggOrWheel(resolver, *resolver.metadataRef)
+			actual, err := NewPythonPackageCataloger().Catalog(resolver)
 			if err != nil {
 				t.Fatalf("failed to catalog python package: %+v", err)
 			}
 
-			for _, d := range deep.Equal(actual, &test.ExpectedPackage) {
+			if len(actual) != 1 {
+				t.Fatalf("unexpected number of packages: %d", len(actual))
+			}
+
+			for _, d := range deep.Equal(actual[0], test.expectedPackage) {
 				t.Errorf("diff: %+v", d)
 			}
 		})
 	}
+}
 
+func TestIgnorePackage(t *testing.T) {
+	tests := []struct {
+		MetadataFixture string
+	}{
+		{
+			MetadataFixture: "test-fixtures/Python-2.7.egg-info",
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.MetadataFixture, func(t *testing.T) {
+			resolver := source.NewMockResolverForPaths(test.MetadataFixture)
+
+			actual, err := NewPythonPackageCataloger().Catalog(resolver)
+			if err != nil {
+				t.Fatalf("failed to catalog python package: %+v", err)
+			}
+
+			if len(actual) != 0 {
+				t.Fatalf("Expected 0 packages but found: %d", len(actual))
+			}
+		})
+	}
 }

syft/cataloger/python/package_entry.go

@@ -0,0 +1,49 @@
+package python
+
+import (
+	"path/filepath"
+
+	"github.com/anchore/syft/syft/source"
+)
+
+type packageEntry struct {
+	Metadata   source.FileData
+	FileRecord *source.FileData
+	TopPackage *source.FileData
+}
+
+// newPackageEntry returns a new packageEntry to be processed relative to what information is available in the given FileResolver.
+func newPackageEntry(resolver source.FileResolver, metadataLocation source.Location) *packageEntry {
+	// we've been given a file reference to a specific wheel METADATA file. note: this may be for a directory
+	// or for an image... for an image the METADATA file may be present within multiple layers, so it is important
+	// to reconcile the RECORD path to the same layer (or a lower layer). The same is true with the top_level.txt file.
+
+	// lets find the RECORD file relative to the directory where the METADATA file resides (in path AND layer structure)
+	recordPath := filepath.Join(filepath.Dir(metadataLocation.RealPath), "RECORD")
+	recordLocation := resolver.RelativeFileByPath(metadataLocation, recordPath)
+
+	// a top_level.txt file specifies the python top-level packages (provided by this python package) installed into site-packages
+	parentDir := filepath.Dir(metadataLocation.RealPath)
+	topLevelPath := filepath.Join(parentDir, "top_level.txt")
+	topLevelLocation := resolver.RelativeFileByPath(metadataLocation, topLevelPath)
+
+	// build an entry that will later be populated with contents when the request is executed
+	entry := &packageEntry{
+		Metadata: source.FileData{
+			Location: metadataLocation,
+		},
+	}
+
+	if recordLocation != nil {
+		entry.FileRecord = &source.FileData{
+			Location: *recordLocation,
+		}
+	}
+
+	if topLevelLocation != nil {
+		entry.TopPackage = &source.FileData{
+			Location: *topLevelLocation,
+		}
+	}
+	return entry
+}

syft/cataloger/python/parse_wheel_egg_metadata.go

@@ -7,6 +7,9 @@ import (
 	"path/filepath"
 	"strings"
 
+	"github.com/anchore/syft/internal/file"
+	"github.com/anchore/syft/internal/log"
+
 	"github.com/mitchellh/mapstructure"
 
 	"github.com/anchore/syft/syft/pkg"
@@ -54,7 +57,7 @@ func parseWheelOrEggMetadata(path string, reader io.Reader) (pkg.PythonPackageMe
 
 				fields[key] = val
 			} else {
-				return pkg.PythonPackageMetadata{}, fmt.Errorf("cannot parse field from line: '%s'", line)
+				log.Warnf("cannot parse field from path: %q from line: %q", path, line)
 			}
 		}
 	}
@@ -70,11 +73,28 @@ func parseWheelOrEggMetadata(path string, reader io.Reader) (pkg.PythonPackageMe
 
 	// add additional metadata not stored in the egg/wheel metadata file
 
-	metadata.SitePackagesRootPath = filepath.Clean(filepath.Join(filepath.Dir(path), ".."))
+	metadata.SitePackagesRootPath = determineSitePackagesRootPath(path)
 
 	return metadata, nil
 }
 
+// isEggRegularFile determines if the specified path is the regular file variant
+// of egg metadata (as opposed to a directory that contains more metadata
+// files).
+func isEggRegularFile(path string) bool {
+	return file.GlobMatch(eggFileMetadataGlob, path)
+}
+
+// determineSitePackagesRootPath returns the path of the site packages root,
+// given the egg metadata file or directory specified in the path.
+func determineSitePackagesRootPath(path string) string {
+	if isEggRegularFile(path) {
+		return filepath.Clean(filepath.Dir(path))
+	}
+
+	return filepath.Clean(filepath.Dir(filepath.Dir(path)))
+}
+
 // handleFieldBodyContinuation returns the updated value for the specified field after processing the specified line.
 // If the continuation cannot be processed, it returns an error.
 func handleFieldBodyContinuation(key, line string, fields map[string]string) (string, error) {

syft/cataloger/python/parse_wheel_egg_metadata_test.go

@@ -56,5 +56,98 @@ func TestParseWheelEggMetadata(t *testing.T) {
 			}
 		})
 	}
+}
 
+func TestIsRegularEggFile(t *testing.T) {
+	cases := []struct {
+		path     string
+		expected bool
+	}{
+		{
+			"/usr/lib64/python2.6/site-packages/M2Crypto-0.20.2-py2.6.egg-info",
+			true,
+		},
+		{
+			"/usr/lib64/python2.6/site-packages/M2Crypto-0.20.2-py2.6.egg-info/PKG-INFO",
+			false,
+		},
+		{
+			"/usr/lib64/python2.6/site-packages/M2Crypto-0.20.2-py2.6.dist-info/METADATA",
+			false,
+		},
+	}
+
+	for _, c := range cases {
+		t.Run(c.path, func(t *testing.T) {
+			actual := isEggRegularFile(c.path)
+
+			if actual != c.expected {
+				t.Errorf("expected %t but got %t", c.expected, actual)
+			}
+		})
+	}
+}
+
+func TestDetermineSitePackagesRootPath(t *testing.T) {
+	cases := []struct {
+		inputPath string
+		expected  string
+	}{
+		{
+			inputPath: "/usr/lib64/python2.6/site-packages/ethtool-0.6-py2.6.egg-info",
+			expected:  "/usr/lib64/python2.6/site-packages",
+		},
+		{
+			inputPath: "/usr/lib/python2.7/dist-packages/configobj-5.0.6.egg-info/top_level.txt",
+			expected:  "/usr/lib/python2.7/dist-packages",
+		},
+		{
+			inputPath: "/usr/lib/python2.7/dist-packages/six-1.10.0.egg-info/PKG-INFO",
+			expected:  "/usr/lib/python2.7/dist-packages",
+		},
+	}
+
+	for _, c := range cases {
+		t.Run(c.inputPath, func(t *testing.T) {
+			actual := determineSitePackagesRootPath(c.inputPath)
+
+			if actual != c.expected {
+				t.Errorf("expected %s but got %s", c.expected, actual)
+			}
+		})
+	}
+}
+
+func TestParseWheelEggMetadataInvalid(t *testing.T) {
+	tests := []struct {
+		Fixture          string
+		ExpectedMetadata pkg.PythonPackageMetadata
+	}{
+		{
+			Fixture: "test-fixtures/egg-info/PKG-INFO-INVALID",
+			ExpectedMetadata: pkg.PythonPackageMetadata{
+				Name:                 "mxnet",
+				Version:              "1.8.0",
+				SitePackagesRootPath: "test-fixtures",
+			},
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.Fixture, func(t *testing.T) {
+			fixture, err := os.Open(test.Fixture)
+			if err != nil {
+				t.Fatalf("failed to open fixture: %+v", err)
+			}
+
+			actual, err := parseWheelOrEggMetadata(test.Fixture, fixture)
+			if err != nil {
+				t.Fatalf("failed to parse: %+v", err)
+			}
+
+			for _, d := range deep.Equal(actual, test.ExpectedMetadata) {
+				t.Errorf("diff: %+v", d)
+			}
+		})
+	}
 }

syft/cataloger/python/test-fixtures/Python-2.7.egg-info

@@ -0,0 +1,31 @@
+Metadata-Version: 1.1
+Name: Python
+Version: 2.7.13
+Summary: A high-level object-oriented programming language
+Home-page: http://www.python.org/2.7
+Author: Guido van Rossum and the Python community
+Author-email: python-dev@python.org
+License: PSF license
+Description: Python is an interpreted, interactive, object-oriented programming
+        language. It is often compared to Tcl, Perl, Scheme or Java.
+
+        Python combines remarkable power with very clear syntax. It has
+        modules, classes, exceptions, very high level dynamic data types, and
+        dynamic typing. There are interfaces to many system calls and
+        libraries, as well as to various windowing systems (X11, Motif, Tk,
+        Mac, MFC). New built-in modules are easily written in C or C++. Python
+        is also usable as an extension language for applications that need a
+        programmable interface.
+
+        The Python implementation is portable: it runs on many brands of UNIX,
+        on Windows, DOS, OS/2, Mac, Amiga... If your favorite system isn't
+        listed here, it may still be supported, if there's a C compiler for
+        it. Ask around on comp.lang.python -- or just try compiling Python
+        yourself.
+Platform: Many
+Classifier: Development Status :: 6 - Mature
+Classifier: License :: OSI Approved :: Python Software Foundation License
+Classifier: Natural Language :: English
+Classifier: Programming Language :: C
+Classifier: Programming Language :: Python
+Classifier: Topic :: Software Development

syft/cataloger/python/test-fixtures/egg-info/PKG-INFO-INVALID

@@ -0,0 +1,69 @@
+Metadata-Version: 1.1
+Name: mxnet
+Version: 1.8.0
+Summary: <!--- Licensed to the Apache Software Foundation (ASF) under one -->
+<!--- or more contributor license agreements.  See the NOTICE file -->
+<!--- distributed with this work for additional information -->
+<!--- regarding copyright ownership.  The ASF licenses this file -->
+<!--- to you under the Apache License, Version 2.0 (the -->
+<!--- "License"); you may not use this file except in compliance -->
+<!--- with the License.  You may obtain a copy of the License at -->
+
+<!---   http://www.apache.org/licenses/LICENSE-2.0 -->
+
+<!--- Unless required by applicable law or agreed to in writing, -->
+<!--- software distributed under the License is distributed on an -->
+<!--- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -->
+<!--- KIND, either express or implied.  See the License for the -->
+<!--- specific language governing permissions and limitations -->
+<!--- under the License. -->
+
+MXNet Python Package
+====================
+This directory and nested files contain MXNet Python package and language binding.
+
+## Installation
+To install MXNet Python package, visit MXNet [Install Instruction](https://mxnet.apache.org/get_started)
+
+
+## Running the unit tests
+
+For running unit tests, you will need the [nose PyPi package](https://pypi.python.org/pypi/nose). To install:
+```bash
+pip install --upgrade nose
+```
+
+Once ```nose``` is installed, run the following from MXNet root directory (please make sure the installation path of ```nosetests``` is included in your ```$PATH``` environment variable):
+```
+nosetests tests/python/unittest
+nosetests tests/python/train
+
+```
+
+Home-page: https://github.com/apache/incubator-mxnet
+Author: UNKNOWN
+Author-email: UNKNOWN
+License: UNKNOWN
+Description: UNKNOWN
+Platform: UNKNOWN
+Classifier: Development Status :: 5 - Production/Stable
+Classifier: Intended Audience :: Developers
+Classifier: Intended Audience :: Education
+Classifier: Intended Audience :: Science/Research
+Classifier: License :: OSI Approved :: Apache Software License
+Classifier: Programming Language :: C++
+Classifier: Programming Language :: Cython
+Classifier: Programming Language :: Other
+Classifier: Programming Language :: Perl
+Classifier: Programming Language :: Python
+Classifier: Programming Language :: Python :: 2.7
+Classifier: Programming Language :: Python :: 3.4
+Classifier: Programming Language :: Python :: 3.5
+Classifier: Programming Language :: Python :: 3.6
+Classifier: Programming Language :: Python :: Implementation :: CPython
+Classifier: Topic :: Scientific/Engineering
+Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
+Classifier: Topic :: Scientific/Engineering :: Mathematics
+Classifier: Topic :: Software Development
+Classifier: Topic :: Software Development :: Libraries
+Classifier: Topic :: Software Development :: Libraries :: Python Modules

syft/cataloger/python/test-fixtures/test.egg-info

@@ -0,0 +1,134 @@
+Metadata-Version: 2.1
+Name: requests
+Version: 2.22.0
+Summary: Python HTTP for Humans.
+Home-page: http://python-requests.org
+Author: Kenneth Reitz
+Author-email: me@kennethreitz.org
+License: Apache 2.0
+Description: Requests: HTTP for Humans™
+        ==========================
+        
+        [![image](https://img.shields.io/pypi/v/requests.svg)](https://pypi.org/project/requests/)
+        [![image](https://img.shields.io/pypi/l/requests.svg)](https://pypi.org/project/requests/)
+        [![image](https://img.shields.io/pypi/pyversions/requests.svg)](https://pypi.org/project/requests/)
+        [![codecov.io](https://codecov.io/github/requests/requests/coverage.svg?branch=master)](https://codecov.io/github/requests/requests)
+        [![image](https://img.shields.io/github/contributors/requests/requests.svg)](https://github.com/requests/requests/graphs/contributors)
+        [![image](https://img.shields.io/badge/Say%20Thanks-!-1EAEDB.svg)](https://saythanks.io/to/kennethreitz)
+        
+        Requests is the only *Non-GMO* HTTP library for Python, safe for human
+        consumption.
+        
+        ![image](https://farm5.staticflickr.com/4317/35198386374_1939af3de6_k_d.jpg)
+        
+        Behold, the power of Requests:
+        
+        ``` {.sourceCode .python}
+        >>> import requests
+        >>> r = requests.get('https://api.github.com/user', auth=('user', 'pass'))
+        >>> r.status_code
+        200
+        >>> r.headers['content-type']
+        'application/json; charset=utf8'
+        >>> r.encoding
+        'utf-8'
+        >>> r.text
+        u'{"type":"User"...'
+        >>> r.json()
+        {u'disk_usage': 368627, u'private_gists': 484, ...}
+        ```
+        
+        See [the similar code, sans Requests](https://gist.github.com/973705).
+        
+        [![image](https://raw.githubusercontent.com/requests/requests/master/docs/_static/requests-logo-small.png)](http://docs.python-requests.org/)
+        
+        Requests allows you to send *organic, grass-fed* HTTP/1.1 requests,
+        without the need for manual labor. There's no need to manually add query
+        strings to your URLs, or to form-encode your POST data. Keep-alive and
+        HTTP connection pooling are 100% automatic, thanks to
+        [urllib3](https://github.com/shazow/urllib3).
+        
+        Besides, all the cool kids are doing it. Requests is one of the most
+        downloaded Python packages of all time, pulling in over 11,000,000
+        downloads every month. You don't want to be left out!
+        
+        Feature Support
+        ---------------
+        
+        Requests is ready for today's web.
+        
+        -   International Domains and URLs
+        -   Keep-Alive & Connection Pooling
+        -   Sessions with Cookie Persistence
+        -   Browser-style SSL Verification
+        -   Basic/Digest Authentication
+        -   Elegant Key/Value Cookies
+        -   Automatic Decompression
+        -   Automatic Content Decoding
+        -   Unicode Response Bodies
+        -   Multipart File Uploads
+        -   HTTP(S) Proxy Support
+        -   Connection Timeouts
+        -   Streaming Downloads
+        -   `.netrc` Support
+        -   Chunked Requests
+        
+        Requests officially supports Python 2.7 & 3.4–3.7, and runs great on
+        PyPy.
+        
+        Installation
+        ------------
+        
+        To install Requests, simply use [pipenv](http://pipenv.org/) (or pip, of
+        course):
+        
+        ``` {.sourceCode .bash}
+        $ pipenv install requests
+        ✨🍰✨
+        ```
+        
+        Satisfaction guaranteed.
+        
+        Documentation
+        -------------
+        
+        Fantastic documentation is available at
+        <http://docs.python-requests.org/>, for a limited time only.
+        
+        How to Contribute
+        -----------------
+        
+        1.  Become more familiar with the project by reading our [Contributor's Guide](http://docs.python-requests.org/en/latest/dev/contributing/) and our [development philosophy](http://docs.python-requests.org/en/latest/dev/philosophy/).
+        2.  Check for open issues or open a fresh issue to start a discussion
+            around a feature idea or a bug. There is a [Contributor
+            Friendly](https://github.com/requests/requests/issues?direction=desc&labels=Contributor+Friendly&page=1&sort=updated&state=open)
+            tag for issues that should be ideal for people who are not very
+            familiar with the codebase yet.
+        3.  Fork [the repository](https://github.com/requests/requests) on
+            GitHub to start making your changes to the **master** branch (or
+            branch off of it).
+        4.  Write a test which shows that the bug was fixed or that the feature
+            works as expected.
+        5.  Send a pull request and bug the maintainer until it gets merged and
+            published. :) Make sure to add yourself to
+            [AUTHORS](https://github.com/requests/requests/blob/master/AUTHORS.rst).
+        
+        
+Platform: UNKNOWN
+Classifier: Development Status :: 5 - Production/Stable
+Classifier: Intended Audience :: Developers
+Classifier: Natural Language :: English
+Classifier: License :: OSI Approved :: Apache Software License
+Classifier: Programming Language :: Python
+Classifier: Programming Language :: Python :: 2
+Classifier: Programming Language :: Python :: 2.7
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.5
+Classifier: Programming Language :: Python :: 3.6
+Classifier: Programming Language :: Python :: 3.7
+Classifier: Programming Language :: Python :: Implementation :: CPython
+Classifier: Programming Language :: Python :: Implementation :: PyPy
+Requires-Python: >=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*
+Description-Content-Type: text/markdown
+Provides-Extra: security
+Provides-Extra: socks

syft/cataloger/rpmdb/cataloger.go

@@ -5,7 +5,6 @@ package rpmdb
 
 import (
 	"fmt"
-	"strings"
 
 	"github.com/anchore/syft/syft/pkg"
 	"github.com/anchore/syft/syft/source"
@@ -37,14 +36,14 @@ func (c *Cataloger) Catalog(resolver source.Resolver) ([]pkg.Package, error) {
 
 	var pkgs []pkg.Package
 	for _, location := range fileMatches {
-		dbContents, err := resolver.FileContentsByLocation(location)
+		dbContentReader, err := resolver.FileContentsByLocation(location)
 		if err != nil {
 			return nil, err
 		}
 
-		pkgs, err = parseRpmDB(resolver, location, strings.NewReader(dbContents))
+		pkgs, err = parseRpmDB(resolver, location, dbContentReader)
 		if err != nil {
-			return nil, fmt.Errorf("unable to catalog rpmdb package=%+v: %w", location.Path, err)
+			return nil, fmt.Errorf("unable to catalog rpmdb package=%+v: %w", location.RealPath, err)
 		}
 	}
 	return pkgs, nil

syft/cataloger/rpmdb/parse_rpmdb.go

@@ -45,11 +45,6 @@ func parseRpmDB(resolver source.FileResolver, dbLocation source.Location, reader
 	allPkgs := make([]pkg.Package, 0)
 
 	for _, entry := range pkgList {
-		records, err := extractRpmdbFileRecords(resolver, entry)
-		if err != nil {
-			return nil, err
-		}
-
 		p := pkg.Package{
 			Name:         entry.Name,
 			Version:      fmt.Sprintf("%s-%s", entry.Version, entry.Release), // this is what engine does, instead of fmt.Sprintf("%d:%s-%s.%s", entry.Epoch, entry.Version, entry.Release, entry.Arch)
@@ -67,7 +62,7 @@ func parseRpmDB(resolver source.FileResolver, dbLocation source.Location, reader
 				Vendor:    entry.Vendor,
 				License:   entry.License,
 				Size:      entry.Size,
-				Files:     records,
+				Files:     extractRpmdbFileRecords(resolver, entry),
 			},
 		}
 
@@ -77,25 +72,19 @@ func parseRpmDB(resolver source.FileResolver, dbLocation source.Location, reader
 	return allPkgs, nil
 }
 
-func extractRpmdbFileRecords(resolver source.FileResolver, entry *rpmdb.PackageInfo) ([]pkg.RpmdbFileRecord, error) {
+func extractRpmdbFileRecords(resolver source.FileResolver, entry *rpmdb.PackageInfo) []pkg.RpmdbFileRecord {
 	var records = make([]pkg.RpmdbFileRecord, 0)
 
 	for _, record := range entry.Files {
-		refs, err := resolver.FilesByPath(record.Path)
-		if err != nil {
-			return nil, fmt.Errorf("failed to resolve path=%+v: %w", record.Path, err)
-		}
 		//only persist RPMDB file records which exist in the image/directory, otherwise ignore them
-		if len(refs) == 0 {
-			continue
+		if resolver.HasPath(record.Path) {
+			records = append(records, pkg.RpmdbFileRecord{
+				Path:   record.Path,
+				Mode:   pkg.RpmdbFileMode(record.Mode),
+				Size:   int(record.Size),
+				SHA256: record.SHA256,
+			})
 		}
-
-		records = append(records, pkg.RpmdbFileRecord{
-			Path:   record.Path,
-			Mode:   pkg.RpmdbFileMode(record.Mode),
-			Size:   int(record.Size),
-			SHA256: record.SHA256,
-		})
 	}
-	return records, nil
+	return records
 }

syft/cataloger/rpmdb/parse_rpmdb_test.go

@@ -21,6 +21,10 @@ func newTestFileResolver(ignorePaths bool) *rpmdbTestFileResolverMock {
 	}
 }
 
+func (r rpmdbTestFileResolverMock) HasPath(path string) bool {
+	return !r.ignorePaths
+}
+
 func (r *rpmdbTestFileResolverMock) FilesByPath(paths ...string) ([]source.Location, error) {
 	if r.ignorePaths {
 		// act as if no paths exist

syft/cataloger/ruby/catalogers.go

@@ -19,7 +19,7 @@ func NewGemFileLockCataloger() *common.GenericCataloger {
 // NewGemSpecCataloger returns a new Bundler cataloger object tailored for detecting installations of gems (e.g. Gemspec).
 func NewGemSpecCataloger() *common.GenericCataloger {
 	globParsers := map[string]common.ParserFn{
-		"**/specifications/*.gemspec": parseGemSpecEntries,
+		"**/specifications/**/*.gemspec": parseGemSpecEntries,
 	}
 
 	return common.NewGenericCataloger(nil, globParsers, "ruby-gemspec-cataloger")

syft/distro/identify.go

@@ -1,6 +1,7 @@
 package distro
 
 import (
+	"io/ioutil"
 	"regexp"
 	"strings"
 
@@ -16,28 +17,28 @@ type parseEntry struct {
 	fn   parseFunc
 }
 
+var identityFiles = []parseEntry{
+	{
+		// most distros provide a link at this location
+		path: "/etc/os-release",
+		fn:   parseOsRelease,
+	},
+	{
+		// standard location for rhel & debian distros
+		path: "/usr/lib/os-release",
+		fn:   parseOsRelease,
+	},
+	{
+		// check for busybox (important to check this last since other distros contain the busybox binary)
+		path: "/bin/busybox",
+		fn:   parseBusyBox,
+	},
+}
+
 // Identify parses distro-specific files to determine distro metadata like version and release.
 func Identify(resolver source.Resolver) *Distro {
 	var distro *Distro
 
-	identityFiles := []parseEntry{
-		{
-			// most distros provide a link at this location
-			path: "/etc/os-release",
-			fn:   parseOsRelease,
-		},
-		{
-			// standard location for rhel & debian distros
-			path: "/usr/lib/os-release",
-			fn:   parseOsRelease,
-		},
-		{
-			// check for busybox (important to check this last since other distros contain the busybox binary)
-			path: "/bin/busybox",
-			fn:   parseBusyBox,
-		},
-	}
-
 identifyLoop:
 	for _, entry := range identityFiles {
 		locations, err := resolver.FilesByPath(entry.path)
@@ -52,19 +53,25 @@ identifyLoop:
 		}
 
 		for _, location := range locations {
-			content, err := resolver.FileContentsByLocation(location)
+			contentReader, err := resolver.FileContentsByLocation(location)
 
 			if err != nil {
 				log.Debugf("unable to get contents from %s: %s", entry.path, err)
 				continue
 			}
 
-			if content == "" {
+			content, err := ioutil.ReadAll(contentReader)
+			if err != nil {
+				log.Errorf("unable to read %q: %+v", location.RealPath, err)
+				break
+			}
+
+			if len(content) == 0 {
 				log.Debugf("no contents in file, skipping: %s", entry.path)
 				continue
 			}
 
-			if candidateDistro := entry.fn(content); candidateDistro != nil {
+			if candidateDistro := entry.fn(string(content)); candidateDistro != nil {
 				distro = candidateDistro
 				break identifyLoop
 			}

syft/event/parsers/parsers.go

@@ -83,7 +83,7 @@ func ParseImportStarted(e partybus.Event) (string, progress.StagedProgressable,
 		return "", nil, err
 	}
 
-	imgName, ok := e.Source.(string)
+	host, ok := e.Source.(string)
 	if !ok {
 		return "", nil, newPayloadErr(e.Type, "Source", e.Source)
 	}
@@ -93,5 +93,5 @@ func ParseImportStarted(e partybus.Event) (string, progress.StagedProgressable,
 		return "", nil, newPayloadErr(e.Type, "Value", e.Value)
 	}
 
-	return imgName, prog, nil
+	return host, prog, nil
 }

syft/pkg/dpkg_metadata.go

@@ -11,6 +11,7 @@ type DpkgMetadata struct {
 	Package       string           `mapstructure:"Package" json:"package"`
 	Source        string           `mapstructure:"Source" json:"source"`
 	Version       string           `mapstructure:"Version" json:"version"`
+	SourceVersion string           `mapstructure:"SourceVersion" json:"sourceVersion"`
 	Architecture  string           `mapstructure:"Architecture" json:"architecture"`
 	Maintainer    string           `mapstructure:"Maintainer" json:"maintainer"`
 	InstalledSize int              `mapstructure:"InstalledSize" json:"installedSize"`

syft/presenter/cyclonedx/presenter_test.go

@@ -6,6 +6,8 @@ import (
 	"regexp"
 	"testing"
 
+	"github.com/anchore/stereoscope/pkg/filetree"
+
 	"github.com/anchore/stereoscope/pkg/imagetest"
 
 	"github.com/anchore/go-testutils"
@@ -28,7 +30,7 @@ func TestCycloneDxDirsPresenter(t *testing.T) {
 		Type:    pkg.DebPkg,
 		FoundBy: "the-cataloger-1",
 		Locations: []source.Location{
-			{Path: "/some/path/pkg1"},
+			{RealPath: "/some/path/pkg1"},
 		},
 		Metadata: pkg.DpkgMetadata{
 			Package:      "package1",
@@ -42,7 +44,7 @@ func TestCycloneDxDirsPresenter(t *testing.T) {
 		Type:    pkg.DebPkg,
 		FoundBy: "the-cataloger-2",
 		Locations: []source.Location{
-			{Path: "/some/path/pkg1"},
+			{RealPath: "/some/path/pkg1"},
 		},
 		Licenses: []string{
 			"MIT",
@@ -94,12 +96,15 @@ func TestCycloneDxImgsPresenter(t *testing.T) {
 	img, cleanup := imagetest.GetFixtureImage(t, "docker-archive", "image-simple")
 	defer cleanup()
 
+	_, ref1, _ := img.SquashedTree().File("/somefile-1.txt", filetree.FollowBasenameLinks)
+	_, ref2, _ := img.SquashedTree().File("/somefile-2.txt", filetree.FollowBasenameLinks)
+
 	// populate catalog with test data
 	catalog.Add(pkg.Package{
 		Name:    "package1",
 		Version: "1.0.1",
 		Locations: []source.Location{
-			source.NewLocationFromImage(*img.SquashedTree().File("/somefile-1.txt"), img),
+			source.NewLocationFromImage(string(ref1.RealPath), *ref1, img),
 		},
 		Type:    pkg.RpmPkg,
 		FoundBy: "the-cataloger-1",
@@ -109,7 +114,7 @@ func TestCycloneDxImgsPresenter(t *testing.T) {
 		Name:    "package2",
 		Version: "2.0.1",
 		Locations: []source.Location{
-			source.NewLocationFromImage(*img.SquashedTree().File("/somefile-2.txt"), img),
+			source.NewLocationFromImage(string(ref2.RealPath), *ref2, img),
 		},
 		Type:    pkg.RpmPkg,
 		FoundBy: "the-cataloger-2",
@@ -120,7 +125,7 @@ func TestCycloneDxImgsPresenter(t *testing.T) {
 		PURL: "the-purl-2",
 	})
 
-	s, err := source.NewFromImage(img, source.AllLayersScope, "user-image-input")
+	s, err := source.NewFromImage(img, source.SquashedScope, "user-image-input")
 	if err != nil {
 		t.Fatal(err)
 	}

syft/presenter/cyclonedx/test-fixtures/snapshot/TestCycloneDxDirsPresenter.golden

@@ -1,7 +1,7 @@
 <?xml version="1.0" encoding="UTF-8"?>
-<bom xmlns="http://cyclonedx.org/schema/bom/1.2" version="1" serialNumber="urn:uuid:553b2601-a7b0-4461-bce9-fd56e2e2a4be">
+<bom xmlns="http://cyclonedx.org/schema/bom/1.2" version="1" serialNumber="urn:uuid:544b9f08-b39d-4d0c-8723-32609567e8ed">
   <metadata>
-    <timestamp>2020-12-01T22:19:00-05:00</timestamp>
+    <timestamp>2020-12-28T13:56:29-05:00</timestamp>
     <tools>
       <tool>
         <vendor>anchore</vendor>

syft/presenter/cyclonedx/test-fixtures/snapshot/TestCycloneDxImgsPresenter.golden

@@ -1,7 +1,7 @@
 <?xml version="1.0" encoding="UTF-8"?>
-<bom xmlns="http://cyclonedx.org/schema/bom/1.2" version="1" serialNumber="urn:uuid:4a54c180-9469-4513-a1cb-af3fe1e83738">
+<bom xmlns="http://cyclonedx.org/schema/bom/1.2" version="1" serialNumber="urn:uuid:e1515688-2fd6-422a-acb2-97405a44cdfe">
   <metadata>
-    <timestamp>2020-12-01T22:19:00-05:00</timestamp>
+    <timestamp>2020-12-28T13:56:29-05:00</timestamp>
     <tools>
       <tool>
         <vendor>anchore</vendor>

syft/presenter/json/presenter_test.go

@@ -5,6 +5,8 @@ import (
 	"flag"
 	"testing"
 
+	"github.com/anchore/stereoscope/pkg/filetree"
+
 	"github.com/anchore/go-testutils"
 	"github.com/anchore/stereoscope/pkg/imagetest"
 	"github.com/anchore/syft/syft/distro"
@@ -34,7 +36,7 @@ func TestJsonDirsPresenter(t *testing.T) {
 		Type:    pkg.PythonPkg,
 		FoundBy: "the-cataloger-1",
 		Locations: []source.Location{
-			{Path: "/some/path/pkg1"},
+			{RealPath: "/some/path/pkg1"},
 		},
 		Language:     pkg.Python,
 		MetadataType: pkg.PythonPackageMetadataType,
@@ -54,7 +56,7 @@ func TestJsonDirsPresenter(t *testing.T) {
 		Type:    pkg.DebPkg,
 		FoundBy: "the-cataloger-2",
 		Locations: []source.Location{
-			{Path: "/some/path/pkg1"},
+			{RealPath: "/some/path/pkg1"},
 		},
 		MetadataType: pkg.DpkgMetadataType,
 		Metadata: pkg.DpkgMetadata{
@@ -106,12 +108,15 @@ func TestJsonImgsPresenter(t *testing.T) {
 	catalog := pkg.NewCatalog()
 	img := imagetest.GetGoldenFixtureImage(t, testImage)
 
+	_, ref1, _ := img.SquashedTree().File("/somefile-1.txt", filetree.FollowBasenameLinks)
+	_, ref2, _ := img.SquashedTree().File("/somefile-2.txt", filetree.FollowBasenameLinks)
+
 	// populate catalog with test data
 	catalog.Add(pkg.Package{
 		Name:    "package-1",
 		Version: "1.0.1",
 		Locations: []source.Location{
-			source.NewLocationFromImage(*img.SquashedTree().File("/somefile-1.txt"), img),
+			source.NewLocationFromImage(string(ref1.RealPath), *ref1, img),
 		},
 		Type:         pkg.PythonPkg,
 		FoundBy:      "the-cataloger-1",
@@ -131,7 +136,7 @@ func TestJsonImgsPresenter(t *testing.T) {
 		Name:    "package-2",
 		Version: "2.0.1",
 		Locations: []source.Location{
-			source.NewLocationFromImage(*img.SquashedTree().File("/somefile-2.txt"), img),
+			source.NewLocationFromImage(string(ref2.RealPath), *ref2, img),
 		},
 		Type:         pkg.DebPkg,
 		FoundBy:      "the-cataloger-2",
@@ -149,7 +154,7 @@ func TestJsonImgsPresenter(t *testing.T) {
 	// this is a hard coded value that is not given by the fixture helper and must be provided manually
 	img.Metadata.ManifestDigest = "sha256:2731251dc34951c0e50fcc643b4c5f74922dad1a5d98f302b504cf46cd5d9368"
 
-	s, err := source.NewFromImage(img, source.AllLayersScope, "user-image-input")
+	s, err := source.NewFromImage(img, source.SquashedScope, "user-image-input")
 	var d *distro.Distro
 	pres := NewPresenter(catalog, s.Metadata, d)
 

syft/presenter/json/test-fixtures/snapshot/TestJsonDirsPresenter.golden

@@ -50,6 +50,7 @@
     "package": "package-2",
     "source": "",
     "version": "2.0.1",
+    "sourceVersion": "",
     "architecture": "",
     "maintainer": "",
     "installedSize": 0,
@@ -71,7 +72,7 @@
   "version": "[not provided]"
  },
  "schema": {
-  "version": "1.0.0",
-  "url": "https://raw.githubusercontent.com/anchore/syft/main/schema/json/schema-1.0.0.json"
+  "version": "1.0.1",
+  "url": "https://raw.githubusercontent.com/anchore/syft/main/schema/json/schema-1.0.1.json"
  }
 }

syft/presenter/json/test-fixtures/snapshot/TestJsonImgsPresenter.golden

@@ -52,6 +52,7 @@
     "package": "package-2",
     "source": "",
     "version": "2.0.1",
+    "sourceVersion": "",
     "architecture": "",
     "maintainer": "",
     "installedSize": 0,
@@ -70,7 +71,7 @@
     "stereoscope-fixture-image-simple:04e16e44161c8888a1a963720fd0443cbf7eef8101434c431de8725cd98cc9f7"
    ],
    "imageSize": 65,
-   "scope": "AllLayers",
+   "scope": "Squashed",
    "layers": [
     {
      "mediaType": "application/vnd.docker.image.rootfs.diff.tar.gzip",
@@ -102,7 +103,7 @@
   "version": "[not provided]"
  },
  "schema": {
-  "version": "1.0.0",
-  "url": "https://raw.githubusercontent.com/anchore/syft/main/schema/json/schema-1.0.0.json"
+  "version": "1.0.1",
+  "url": "https://raw.githubusercontent.com/anchore/syft/main/schema/json/schema-1.0.1.json"
  }
 }

syft/presenter/table/presenter_test.go

@@ -5,6 +5,8 @@ import (
 	"flag"
 	"testing"
 
+	"github.com/anchore/stereoscope/pkg/filetree"
+
 	"github.com/go-test/deep"
 
 	"github.com/anchore/go-testutils"
@@ -25,12 +27,15 @@ func TestTablePresenter(t *testing.T) {
 	img, cleanup := imagetest.GetFixtureImage(t, "docker-archive", testImage)
 	defer cleanup()
 
+	_, ref1, _ := img.SquashedTree().File("/somefile-1.txt", filetree.FollowBasenameLinks)
+	_, ref2, _ := img.SquashedTree().File("/somefile-2.txt", filetree.FollowBasenameLinks)
+
 	// populate catalog with test data
 	catalog.Add(pkg.Package{
 		Name:    "package-1",
 		Version: "1.0.1",
 		Locations: []source.Location{
-			source.NewLocationFromImage(*img.SquashedTree().File("/somefile-1.txt"), img),
+			source.NewLocationFromImage(string(ref1.RealPath), *ref1, img),
 		},
 		Type: pkg.DebPkg,
 	})
@@ -38,7 +43,7 @@ func TestTablePresenter(t *testing.T) {
 		Name:    "package-2",
 		Version: "2.0.1",
 		Locations: []source.Location{
-			source.NewLocationFromImage(*img.SquashedTree().File("/somefile-2.txt"), img),
+			source.NewLocationFromImage(string(ref2.RealPath), *ref2, img),
 		},
 		Type: pkg.DebPkg,
 	})

syft/presenter/text/presenter_test.go

@@ -5,6 +5,8 @@ import (
 	"flag"
 	"testing"
 
+	"github.com/anchore/stereoscope/pkg/filetree"
+
 	"github.com/anchore/go-testutils"
 	"github.com/anchore/stereoscope/pkg/imagetest"
 	"github.com/anchore/syft/syft/pkg"
@@ -70,12 +72,15 @@ func TestTextImgPresenter(t *testing.T) {
 	img, cleanup := imagetest.GetFixtureImage(t, "docker-archive", "image-simple")
 	defer cleanup()
 
+	_, ref1, _ := img.SquashedTree().File("/somefile-1.txt", filetree.FollowBasenameLinks)
+	_, ref2, _ := img.SquashedTree().File("/somefile-2.txt", filetree.FollowBasenameLinks)
+
 	// populate catalog with test data
 	catalog.Add(pkg.Package{
 		Name:    "package-1",
 		Version: "1.0.1",
 		Locations: []source.Location{
-			source.NewLocationFromImage(*img.SquashedTree().File("/somefile-1.txt"), img),
+			source.NewLocationFromImage(string(ref1.RealPath), *ref1, img),
 		},
 		FoundBy: "dpkg",
 		Type:    pkg.DebPkg,
@@ -84,7 +89,7 @@ func TestTextImgPresenter(t *testing.T) {
 		Name:    "package-2",
 		Version: "2.0.1",
 		Locations: []source.Location{
-			source.NewLocationFromImage(*img.SquashedTree().File("/somefile-2.txt"), img),
+			source.NewLocationFromImage(string(ref2.RealPath), *ref2, img),
 		},
 		FoundBy:  "dpkg",
 		Metadata: PackageInfo{Name: "package-2", Version: "1.0.2"},
@@ -97,7 +102,7 @@ func TestTextImgPresenter(t *testing.T) {
 		l.Metadata.Digest = "sha256:ad8ecdc058976c07e7e347cb89fa9ad86a294b5ceaae6d09713fb035f84115abf3c4a2388a4af3aa60f13b94f4c6846930bdf53"
 	}
 
-	s, err := source.NewFromImage(img, source.AllLayersScope, "user-image-input")
+	s, err := source.NewFromImage(img, source.SquashedScope, "user-image-input")
 	if err != nil {
 		t.Fatal(err)
 	}

syft/source/all_layers_resolver.go

@@ -2,10 +2,15 @@ package source
 
 import (
 	"archive/tar"
+	"bytes"
 	"fmt"
+	"io"
+	"io/ioutil"
 
 	"github.com/anchore/stereoscope/pkg/file"
+	"github.com/anchore/stereoscope/pkg/filetree"
 	"github.com/anchore/stereoscope/pkg/image"
+	"github.com/anchore/syft/internal/log"
 )
 
 var _ Resolver = (*AllLayersResolver)(nil)
@@ -32,6 +37,18 @@ func NewAllLayersResolver(img *image.Image) (*AllLayersResolver, error) {
 	}, nil
 }
 
+// HasPath indicates if the given path exists in the underlying source.
+func (r *AllLayersResolver) HasPath(path string) bool {
+	p := file.Path(path)
+	for _, layerIdx := range r.layers {
+		tree := r.img.Layers[layerIdx].Tree
+		if tree.HasPath(p) {
+			return true
+		}
+	}
+	return false
+}
+
 func (r *AllLayersResolver) fileByRef(ref file.Reference, uniqueFileIDs file.ReferenceSet, layerIdx int) ([]file.Reference, error) {
 	uniqueFiles := make([]file.Reference, 0)
 
@@ -70,19 +87,22 @@ func (r *AllLayersResolver) FilesByPath(paths ...string) ([]Location, error) {
 	for _, path := range paths {
 		for idx, layerIdx := range r.layers {
 			tree := r.img.Layers[layerIdx].Tree
-			ref := tree.File(file.Path(path))
+			_, ref, err := tree.File(file.Path(path), filetree.FollowBasenameLinks, filetree.DoNotFollowDeadBasenameLinks)
+			if err != nil {
+				return nil, err
+			}
 			if ref == nil {
 				// no file found, keep looking through layers
 				continue
 			}
 
 			// don't consider directories (special case: there is no path information for /)
-			if ref.Path == "/" {
+			if ref.RealPath == "/" {
 				continue
 			} else if r.img.FileCatalog.Exists(*ref) {
 				metadata, err := r.img.FileCatalog.Get(*ref)
 				if err != nil {
-					return nil, fmt.Errorf("unable to get file metadata for path=%q: %w", ref.Path, err)
+					return nil, fmt.Errorf("unable to get file metadata for path=%q: %w", ref.RealPath, err)
 				}
 				if metadata.Metadata.IsDir {
 					continue
@@ -94,7 +114,7 @@ func (r *AllLayersResolver) FilesByPath(paths ...string) ([]Location, error) {
 				return nil, err
 			}
 			for _, result := range results {
-				uniqueLocations = append(uniqueLocations, NewLocationFromImage(result, r.img))
+				uniqueLocations = append(uniqueLocations, NewLocationFromImage(path, result, r.img))
 			}
 		}
 	}
@@ -109,31 +129,31 @@ func (r *AllLayersResolver) FilesByGlob(patterns ...string) ([]Location, error)
 
 	for _, pattern := range patterns {
 		for idx, layerIdx := range r.layers {
-			refs, err := r.img.Layers[layerIdx].Tree.FilesByGlob(pattern)
+			results, err := r.img.Layers[layerIdx].Tree.FilesByGlob(pattern, filetree.DoNotFollowDeadBasenameLinks)
 			if err != nil {
 				return nil, fmt.Errorf("failed to resolve files by glob (%s): %w", pattern, err)
 			}
 
-			for _, ref := range refs {
+			for _, result := range results {
 				// don't consider directories (special case: there is no path information for /)
-				if ref.Path == "/" {
+				if result.RealPath == "/" {
 					continue
-				} else if r.img.FileCatalog.Exists(ref) {
-					metadata, err := r.img.FileCatalog.Get(ref)
+				} else if r.img.FileCatalog.Exists(result.Reference) {
+					metadata, err := r.img.FileCatalog.Get(result.Reference)
 					if err != nil {
-						return nil, fmt.Errorf("unable to get file metadata for path=%q: %w", ref.Path, err)
+						return nil, fmt.Errorf("unable to get file metadata for path=%q: %w", result.MatchPath, err)
 					}
 					if metadata.Metadata.IsDir {
 						continue
 					}
 				}
 
-				results, err := r.fileByRef(ref, uniqueFileIDs, idx)
+				refResults, err := r.fileByRef(result.Reference, uniqueFileIDs, idx)
 				if err != nil {
 					return nil, err
 				}
-				for _, result := range results {
-					uniqueLocations = append(uniqueLocations, NewLocationFromImage(result, r.img))
+				for _, refResult := range refResults {
+					uniqueLocations = append(uniqueLocations, NewLocationFromImage(string(result.MatchPath), refResult, r.img))
 				}
 			}
 		}
@@ -150,37 +170,41 @@ func (r *AllLayersResolver) RelativeFileByPath(location Location, path string) *
 		return nil
 	}
 
-	relativeRef := entry.Source.SquashedTree.File(file.Path(path))
-	if relativeRef == nil {
+	exists, relativeRef, err := entry.Layer.SquashedTree.File(file.Path(path), filetree.FollowBasenameLinks)
+	if err != nil {
+		log.Errorf("failed to find path=%q in squash: %+w", path, err)
+		return nil
+	}
+	if !exists && relativeRef == nil {
 		return nil
 	}
 
-	relativeLocation := NewLocationFromImage(*relativeRef, r.img)
+	relativeLocation := NewLocationFromImage(path, *relativeRef, r.img)
 
 	return &relativeLocation
 }
 
 // MultipleFileContentsByLocation returns the file contents for all file.References relative to the image. Note that a
 // file.Reference is a path relative to a particular layer.
-func (r *AllLayersResolver) MultipleFileContentsByLocation(locations []Location) (map[Location]string, error) {
+func (r *AllLayersResolver) MultipleFileContentsByLocation(locations []Location) (map[Location]io.ReadCloser, error) {
 	return mapLocationRefs(r.img.MultipleFileContentsByRef, locations)
 }
 
 // FileContentsByLocation fetches file contents for a single file reference, irregardless of the source layer.
 // If the path does not exist an error is returned.
-func (r *AllLayersResolver) FileContentsByLocation(location Location) (string, error) {
+func (r *AllLayersResolver) FileContentsByLocation(location Location) (io.ReadCloser, error) {
 	return r.img.FileContentsByRef(location.ref)
 }
 
-type multiContentFetcher func(refs ...file.Reference) (map[file.Reference]string, error)
+type multiContentFetcher func(refs ...file.Reference) (map[file.Reference]io.ReadCloser, error)
 
-func mapLocationRefs(callback multiContentFetcher, locations []Location) (map[Location]string, error) {
+func mapLocationRefs(callback multiContentFetcher, locations []Location) (map[Location]io.ReadCloser, error) {
 	var fileRefs = make([]file.Reference, len(locations))
-	var locationByRefs = make(map[file.Reference]Location)
-	var results = make(map[Location]string)
+	var locationByRefs = make(map[file.Reference][]Location)
+	var results = make(map[Location]io.ReadCloser)
 
 	for i, location := range locations {
-		locationByRefs[location.ref] = location
+		locationByRefs[location.ref] = append(locationByRefs[location.ref], location)
 		fileRefs[i] = location.ref
 	}
 
@@ -189,8 +213,26 @@ func mapLocationRefs(callback multiContentFetcher, locations []Location) (map[Lo
 		return nil, err
 	}
 
+	// TODO: this is not tested, we need a test case that covers a mapLocationRefs which has multiple Locations with the same reference in the request. The io.Reader should be copied.
 	for ref, content := range contentsByRef {
-		results[locationByRefs[ref]] = content
+		mappedLocations := locationByRefs[ref]
+		switch {
+		case len(mappedLocations) > 1:
+			// TODO: fixme... this can lead to lots of unexpected memory usage in unusual circumstances (cache is not leveraged for large files).
+			// stereoscope wont duplicate content requests if the caller asks for the same file multiple times... thats up to the caller
+			contentsBytes, err := ioutil.ReadAll(content)
+			if err != nil {
+				return nil, fmt.Errorf("unable to read ref=%+v :%w", ref, err)
+			}
+			for _, loc := range mappedLocations {
+				results[loc] = ioutil.NopCloser(bytes.NewReader(contentsBytes))
+			}
+
+		case len(mappedLocations) == 1:
+			results[locationByRefs[ref][0]] = content
+		default:
+			return nil, fmt.Errorf("unexpected ref-location count=%d for ref=%v", len(mappedLocations), ref)
+		}
 	}
 	return results, nil
 }

syft/source/all_layers_resolver_test.go

@@ -13,9 +13,10 @@ type resolution struct {
 
 func TestAllLayersResolver_FilesByPath(t *testing.T) {
 	cases := []struct {
-		name        string
-		linkPath    string
-		resolutions []resolution
+		name                 string
+		linkPath             string
+		resolutions          []resolution
+		forcePositiveHasPath bool
 	}{
 		{
 			name:     "link with previous data",
@@ -41,10 +42,6 @@ func TestAllLayersResolver_FilesByPath(t *testing.T) {
 			name:     "link with overridden data",
 			linkPath: "/link-2",
 			resolutions: []resolution{
-				{
-					layer: 3,
-					path:  "/link-2",
-				},
 				{
 					layer: 4,
 					path:  "/file-2.txt",
@@ -70,19 +67,17 @@ func TestAllLayersResolver_FilesByPath(t *testing.T) {
 			},
 		},
 		{
-			name:     "dead link",
-			linkPath: "/link-dead",
-			resolutions: []resolution{
-				{
-					layer: 8,
-					path:  "/link-dead",
-				},
-			},
+			name:                 "dead link",
+			linkPath:             "/link-dead",
+			resolutions:          []resolution{},
+			forcePositiveHasPath: true,
 		},
 		{
 			name:        "ignore directories",
 			linkPath:    "/bin",
 			resolutions: []resolution{},
+			// directories don't resolve BUT do exist
+			forcePositiveHasPath: true,
 		},
 	}
 	for _, c := range cases {
@@ -95,6 +90,17 @@ func TestAllLayersResolver_FilesByPath(t *testing.T) {
 				t.Fatalf("could not create resolver: %+v", err)
 			}
 
+			hasPath := resolver.HasPath(c.linkPath)
+			if !c.forcePositiveHasPath {
+				if len(c.resolutions) > 0 && !hasPath {
+					t.Errorf("expected HasPath() to indicate existance, but did not")
+				} else if len(c.resolutions) == 0 && hasPath {
+					t.Errorf("expeced HasPath() to NOT indicate existance, but does")
+				}
+			} else if !hasPath {
+				t.Errorf("expected HasPath() to indicate existance, but did not (force path)")
+			}
+
 			refs, err := resolver.FilesByPath(c.linkPath)
 			if err != nil {
 				t.Fatalf("could not use resolver: %+v", err)
@@ -107,8 +113,12 @@ func TestAllLayersResolver_FilesByPath(t *testing.T) {
 			for idx, actual := range refs {
 				expected := c.resolutions[idx]
 
-				if actual.Path != expected.path {
-					t.Errorf("bad resolve path: '%s'!='%s'", actual.Path, expected.path)
+				if string(actual.ref.RealPath) != expected.path {
+					t.Errorf("bad resolve path: '%s'!='%s'", string(actual.ref.RealPath), expected.path)
+				}
+
+				if expected.path != "" && string(actual.ref.RealPath) != actual.RealPath {
+					t.Errorf("we should always prefer real paths over ones with links")
 				}
 
 				entry, err := img.FileCatalog.Get(actual.ref)
@@ -116,8 +126,8 @@ func TestAllLayersResolver_FilesByPath(t *testing.T) {
 					t.Fatalf("failed to get metadata: %+v", err)
 				}
 
-				if entry.Source.Metadata.Index != expected.layer {
-					t.Errorf("bad resolve layer: '%d'!='%d'", entry.Source.Metadata.Index, expected.layer)
+				if entry.Layer.Metadata.Index != expected.layer {
+					t.Errorf("bad resolve layer: '%d'!='%d'", entry.Layer.Metadata.Index, expected.layer)
 				}
 			}
 		})
@@ -132,7 +142,7 @@ func TestAllLayersResolver_FilesByGlob(t *testing.T) {
 	}{
 		{
 			name: "link with previous data",
-			glob: "**ink-1",
+			glob: "**/*ink-1",
 			resolutions: []resolution{
 				{
 					layer: 1,
@@ -142,7 +152,7 @@ func TestAllLayersResolver_FilesByGlob(t *testing.T) {
 		},
 		{
 			name: "link with in layer data",
-			glob: "**nk-within",
+			glob: "**/*nk-within",
 			resolutions: []resolution{
 				{
 					layer: 5,
@@ -152,12 +162,8 @@ func TestAllLayersResolver_FilesByGlob(t *testing.T) {
 		},
 		{
 			name: "link with overridden data",
-			glob: "**ink-2",
+			glob: "**/*ink-2",
 			resolutions: []resolution{
-				{
-					layer: 3,
-					path:  "/link-2",
-				},
 				{
 					layer: 4,
 					path:  "/file-2.txt",
@@ -170,7 +176,7 @@ func TestAllLayersResolver_FilesByGlob(t *testing.T) {
 		},
 		{
 			name: "indirect link (with overridden data)",
-			glob: "**nk-indirect",
+			glob: "**/*nk-indirect",
 			resolutions: []resolution{
 				{
 					layer: 4,
@@ -183,14 +189,9 @@ func TestAllLayersResolver_FilesByGlob(t *testing.T) {
 			},
 		},
 		{
-			name: "dead link",
-			glob: "**k-dead",
-			resolutions: []resolution{
-				{
-					layer: 8,
-					path:  "/link-dead",
-				},
-			},
+			name:        "dead link",
+			glob:        "**/*k-dead",
+			resolutions: []resolution{},
 		},
 		{
 			name:        "ignore directories",
@@ -220,8 +221,12 @@ func TestAllLayersResolver_FilesByGlob(t *testing.T) {
 			for idx, actual := range refs {
 				expected := c.resolutions[idx]
 
-				if actual.Path != expected.path {
-					t.Errorf("bad resolve path: '%s'!='%s'", actual.Path, expected.path)
+				if string(actual.ref.RealPath) != expected.path {
+					t.Errorf("bad resolve path: '%s'!='%s'", string(actual.ref.RealPath), expected.path)
+				}
+
+				if expected.path != "" && string(actual.ref.RealPath) != actual.RealPath {
+					t.Errorf("we should always prefer real paths over ones with links")
 				}
 
 				entry, err := img.FileCatalog.Get(actual.ref)
@@ -229,8 +234,8 @@ func TestAllLayersResolver_FilesByGlob(t *testing.T) {
 					t.Fatalf("failed to get metadata: %+v", err)
 				}
 
-				if entry.Source.Metadata.Index != expected.layer {
-					t.Errorf("bad resolve layer: '%d'!='%d'", entry.Source.Metadata.Index, expected.layer)
+				if entry.Layer.Metadata.Index != expected.layer {
+					t.Errorf("bad resolve layer: '%d'!='%d'", entry.Layer.Metadata.Index, expected.layer)
 				}
 			}
 		})

syft/source/content_requester.go

@@ -0,0 +1,56 @@
+package source
+
+import "sync"
+
+// ContentRequester is an object tailored for taking source.Location objects which file contents will be resolved
+// upon invoking Execute().
+type ContentRequester struct {
+	request map[Location][]*FileData
+	lock    sync.Mutex
+}
+
+// NewContentRequester creates a new ContentRequester object with the given initial request data.
+func NewContentRequester(data ...*FileData) *ContentRequester {
+	requester := &ContentRequester{
+		request: make(map[Location][]*FileData),
+	}
+	for _, d := range data {
+		requester.Add(d)
+	}
+	return requester
+}
+
+// Add appends a new single FileData containing a source.Location to later have the contents fetched and stored within
+// the given FileData object.
+func (r *ContentRequester) Add(data *FileData) {
+	r.lock.Lock()
+	defer r.lock.Unlock()
+
+	r.request[data.Location] = append(r.request[data.Location], data)
+}
+
+// Execute takes the previously provided source.Location's and resolves the file contents, storing the results within
+// the previously provided FileData objects.
+func (r *ContentRequester) Execute(resolver ContentResolver) error {
+	r.lock.Lock()
+	defer r.lock.Unlock()
+
+	var locations = make([]Location, len(r.request))
+	idx := 0
+	for l := range r.request {
+		locations[idx] = l
+		idx++
+	}
+
+	response, err := resolver.MultipleFileContentsByLocation(locations)
+	if err != nil {
+		return err
+	}
+
+	for l, contents := range response {
+		for i := range r.request[l] {
+			r.request[l][i].Contents = contents
+		}
+	}
+	return nil
+}

syft/source/content_requester_test.go

@@ -0,0 +1,74 @@
+package source
+
+import (
+	"io/ioutil"
+	"testing"
+
+	"github.com/anchore/stereoscope/pkg/imagetest"
+	"github.com/sergi/go-diff/diffmatchpatch"
+)
+
+func TestContentRequester(t *testing.T) {
+	tests := []struct {
+		fixture          string
+		expectedContents map[string]string
+	}{
+		{
+			fixture: "image-simple",
+			expectedContents: map[string]string{
+				"/somefile-1.txt":           "this file has contents",
+				"/somefile-2.txt":           "file-2 contents!",
+				"/really/nested/file-3.txt": "another file!\nwith lines...",
+			},
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.fixture, func(t *testing.T) {
+			img, cleanup := imagetest.GetFixtureImage(t, "docker-archive", "image-simple")
+			defer cleanup()
+
+			resolver, err := NewAllLayersResolver(img)
+			if err != nil {
+				t.Fatalf("could not create resolver: %+v", err)
+			}
+
+			var data []*FileData
+			for path := range test.expectedContents {
+
+				locations, err := resolver.FilesByPath(path)
+				if err != nil {
+					t.Fatalf("could not build request: %+v", err)
+				}
+				if len(locations) != 1 {
+					t.Fatalf("bad resolver paths: %+v", locations)
+				}
+
+				data = append(data, &FileData{
+					Location: locations[0],
+				})
+			}
+
+			if err := NewContentRequester(data...).Execute(resolver); err != nil {
+				t.Fatalf("could not execute request: %+v", err)
+			}
+
+			for _, entry := range data {
+				if expected, ok := test.expectedContents[entry.Location.RealPath]; ok {
+					actualBytes, err := ioutil.ReadAll(entry.Contents)
+					if err != nil {
+						t.Fatalf("could not read %q: %+v", entry.Location.RealPath, err)
+					}
+					for expected != string(actualBytes) {
+						t.Errorf("mismatched contents for %q", entry.Location.RealPath)
+						dmp := diffmatchpatch.New()
+						diffs := dmp.DiffMain(expected, string(actualBytes), true)
+						t.Errorf("diff: %s", dmp.DiffPrettyText(diffs))
+					}
+					continue
+				}
+				t.Errorf("could not find %q", entry.Location.RealPath)
+			}
+		})
+	}
+}

syft/source/directory_resolver.go

@@ -2,13 +2,14 @@ package source
 
 import (
 	"fmt"
-	"io/ioutil"
+	"io"
 	"os"
 	"path"
 	"path/filepath"
 
+	"github.com/anchore/stereoscope/pkg/file"
 	"github.com/anchore/syft/internal/log"
-	"github.com/bmatcuk/doublestar"
+	"github.com/bmatcuk/doublestar/v2"
 )
 
 var _ Resolver = (*DirectoryResolver)(nil)
@@ -18,27 +19,37 @@ type DirectoryResolver struct {
 	Path string
 }
 
+func (r DirectoryResolver) requestPath(userPath string) string {
+	fullPath := userPath
+	if filepath.IsAbs(fullPath) {
+		// a path relative to root should be prefixed with the resolvers directory path, otherwise it should be left as is
+		fullPath = path.Join(r.Path, fullPath)
+	}
+	return fullPath
+}
+
+// HasPath indicates if the given path exists in the underlying source.
+func (r *DirectoryResolver) HasPath(userPath string) bool {
+	_, err := os.Stat(r.requestPath(userPath))
+	return !os.IsNotExist(err)
+}
+
 // Stringer to represent a directory path data source
-func (s DirectoryResolver) String() string {
-	return fmt.Sprintf("dir:%s", s.Path)
+func (r DirectoryResolver) String() string {
+	return fmt.Sprintf("dir:%s", r.Path)
 }
 
 // FilesByPath returns all file.References that match the given paths from the directory.
-func (s DirectoryResolver) FilesByPath(userPaths ...string) ([]Location, error) {
+func (r DirectoryResolver) FilesByPath(userPaths ...string) ([]Location, error) {
 	var references = make([]Location, 0)
 
 	for _, userPath := range userPaths {
-		userStrPath := userPath
-
-		if filepath.IsAbs(userStrPath) {
-			// a path relative to root should be prefixed with the resolvers directory path, otherwise it should be left as is
-			userStrPath = path.Join(s.Path, userStrPath)
-		}
+		userStrPath := r.requestPath(userPath)
 		fileMeta, err := os.Stat(userStrPath)
 		if os.IsNotExist(err) {
 			continue
 		} else if err != nil {
-			log.Errorf("path (%s) is not valid: %v", userStrPath, err)
+			log.Errorf("path (%r) is not valid: %v", userStrPath, err)
 		}
 
 		// don't consider directories
@@ -53,11 +64,11 @@ func (s DirectoryResolver) FilesByPath(userPaths ...string) ([]Location, error)
 }
 
 // FilesByGlob returns all file.References that match the given path glob pattern from any layer in the image.
-func (s DirectoryResolver) FilesByGlob(patterns ...string) ([]Location, error) {
+func (r DirectoryResolver) FilesByGlob(patterns ...string) ([]Location, error) {
 	result := make([]Location, 0)
 
 	for _, pattern := range patterns {
-		pathPattern := path.Join(s.Path, pattern)
+		pathPattern := path.Join(r.Path, pattern)
 		pathMatches, err := doublestar.Glob(pathPattern)
 		if err != nil {
 			return nil, err
@@ -83,8 +94,8 @@ func (s DirectoryResolver) FilesByGlob(patterns ...string) ([]Location, error) {
 // RelativeFileByPath fetches a single file at the given path relative to the layer squash of the given reference.
 // This is helpful when attempting to find a file that is in the same layer or lower as another file. For the
 // DirectoryResolver, this is a simple path lookup.
-func (s *DirectoryResolver) RelativeFileByPath(_ Location, path string) *Location {
-	paths, err := s.FilesByPath(path)
+func (r *DirectoryResolver) RelativeFileByPath(_ Location, path string) *Location {
+	paths, err := r.FilesByPath(path)
 	if err != nil {
 		return nil
 	}
@@ -96,35 +107,16 @@ func (s *DirectoryResolver) RelativeFileByPath(_ Location, path string) *Locatio
 }
 
 // MultipleFileContentsByLocation returns the file contents for all file.References relative a directory.
-func (s DirectoryResolver) MultipleFileContentsByLocation(locations []Location) (map[Location]string, error) {
-	refContents := make(map[Location]string)
+func (r DirectoryResolver) MultipleFileContentsByLocation(locations []Location) (map[Location]io.ReadCloser, error) {
+	refContents := make(map[Location]io.ReadCloser)
 	for _, location := range locations {
-		contents, err := fileContents(location.Path)
-
-		if err != nil {
-			return nil, fmt.Errorf("could not read contents of file: %s", location.Path)
-		}
-		refContents[location] = string(contents)
+		refContents[location] = file.NewDeferredReadCloser(location.RealPath)
 	}
 	return refContents, nil
 }
 
 // FileContentsByLocation fetches file contents for a single file reference relative to a directory.
 // If the path does not exist an error is returned.
-func (s DirectoryResolver) FileContentsByLocation(location Location) (string, error) {
-	contents, err := fileContents(location.Path)
-	if err != nil {
-		return "", fmt.Errorf("could not read contents of file: %s", location.Path)
-	}
-
-	return string(contents), nil
-}
-
-func fileContents(path string) ([]byte, error) {
-	contents, err := ioutil.ReadFile(path)
-
-	if err != nil {
-		return nil, err
-	}
-	return contents, nil
+func (r DirectoryResolver) FileContentsByLocation(location Location) (io.ReadCloser, error) {
+	return file.NewDeferredReadCloser(location.RealPath), nil
 }

syft/source/directory_resolver_test.go

@@ -6,11 +6,12 @@ import (
 
 func TestDirectoryResolver_FilesByPath(t *testing.T) {
 	cases := []struct {
-		name     string
-		root     string
-		input    string
-		expected string
-		refCount int
+		name                 string
+		root                 string
+		input                string
+		expected             string
+		refCount             int
+		forcePositiveHasPath bool
 	}{
 		{
 			name:     "finds a file (relative)",
@@ -47,15 +48,28 @@ func TestDirectoryResolver_FilesByPath(t *testing.T) {
 			refCount: 1,
 		},
 		{
-			name:     "directories ignored",
-			root:     "./test-fixtures/",
-			input:    "/image-symlinks",
-			refCount: 0,
+			name:                 "directories ignored",
+			root:                 "./test-fixtures/",
+			input:                "/image-symlinks",
+			refCount:             0,
+			forcePositiveHasPath: true,
 		},
 	}
 	for _, c := range cases {
 		t.Run(c.name, func(t *testing.T) {
 			resolver := DirectoryResolver{c.root}
+
+			hasPath := resolver.HasPath(c.input)
+			if !c.forcePositiveHasPath {
+				if c.refCount != 0 && !hasPath {
+					t.Errorf("expected HasPath() to indicate existance, but did not")
+				} else if c.refCount == 0 && hasPath {
+					t.Errorf("expeced HasPath() to NOT indicate existance, but does")
+				}
+			} else if !hasPath {
+				t.Errorf("expected HasPath() to indicate existance, but did not (force path)")
+			}
+
 			refs, err := resolver.FilesByPath(c.input)
 			if err != nil {
 				t.Fatalf("could not use resolver: %+v, %+v", err, refs)
@@ -66,8 +80,8 @@ func TestDirectoryResolver_FilesByPath(t *testing.T) {
 			}
 
 			for _, actual := range refs {
-				if actual.Path != c.expected {
-					t.Errorf("bad resolve path: '%s'!='%s'", actual.Path, c.expected)
+				if actual.RealPath != c.expected {
+					t.Errorf("bad resolve path: '%s'!='%s'", actual.RealPath, c.expected)
 				}
 			}
 		})
@@ -171,8 +185,9 @@ func TestDirectoryResolver_FilesByGlobMultiple(t *testing.T) {
 			t.Fatalf("could not use resolver: %+v, %+v", err, refs)
 		}
 
-		if len(refs) != 2 {
-			t.Errorf("unexpected number of refs: %d != 2", len(refs))
+		expected := 2
+		if len(refs) != expected {
+			t.Errorf("unexpected number of refs: %d != %d", len(refs), expected)
 		}
 
 	})
@@ -180,15 +195,16 @@ func TestDirectoryResolver_FilesByGlobMultiple(t *testing.T) {
 
 func TestDirectoryResolver_FilesByGlobRecursive(t *testing.T) {
 	t.Run("finds multiple matching files", func(t *testing.T) {
-		resolver := DirectoryResolver{"test-fixtures"}
+		resolver := DirectoryResolver{"test-fixtures/image-symlinks"}
 		refs, err := resolver.FilesByGlob("**/*.txt")
 
 		if err != nil {
 			t.Fatalf("could not use resolver: %+v, %+v", err, refs)
 		}
 
-		if len(refs) != 4 {
-			t.Errorf("unexpected number of refs: %d != 4", len(refs))
+		expected := 6
+		if len(refs) != expected {
+			t.Errorf("unexpected number of refs: %d != %d", len(refs), expected)
 		}
 
 	})
@@ -202,8 +218,9 @@ func TestDirectoryResolver_FilesByGlobSingle(t *testing.T) {
 			t.Fatalf("could not use resolver: %+v, %+v", err, refs)
 		}
 
-		if len(refs) != 1 {
-			t.Errorf("unexpected number of refs: %d != 1", len(refs))
+		expected := 1
+		if len(refs) != expected {
+			t.Errorf("unexpected number of refs: %d != %d", len(refs), expected)
 		}
 
 	})

syft/source/file_data.go

@@ -0,0 +1,8 @@
+package source
+
+import "io"
+
+type FileData struct {
+	Location Location
+	Contents io.ReadCloser
+}

syft/source/image_squash_resolver.go

@@ -2,8 +2,10 @@ package source
 
 import (
 	"fmt"
+	"io"
 
 	"github.com/anchore/stereoscope/pkg/file"
+	"github.com/anchore/stereoscope/pkg/filetree"
 	"github.com/anchore/stereoscope/pkg/image"
 )
 
@@ -22,6 +24,11 @@ func NewImageSquashResolver(img *image.Image) (*ImageSquashResolver, error) {
 	return &ImageSquashResolver{img: img}, nil
 }
 
+// HasPath indicates if the given path exists in the underlying source.
+func (r *ImageSquashResolver) HasPath(path string) bool {
+	return r.img.SquashedTree().HasPath(file.Path(path))
+}
+
 // FilesByPath returns all file.References that match the given paths within the squashed representation of the image.
 func (r *ImageSquashResolver) FilesByPath(paths ...string) ([]Location, error) {
 	uniqueFileIDs := file.NewFileReferenceSet()
@@ -29,19 +36,22 @@ func (r *ImageSquashResolver) FilesByPath(paths ...string) ([]Location, error) {
 
 	for _, path := range paths {
 		tree := r.img.SquashedTree()
-		ref := tree.File(file.Path(path))
+		_, ref, err := tree.File(file.Path(path), filetree.FollowBasenameLinks)
+		if err != nil {
+			return nil, err
+		}
 		if ref == nil {
 			// no file found, keep looking through layers
 			continue
 		}
 
 		// don't consider directories (special case: there is no path information for /)
-		if ref.Path == "/" {
+		if ref.RealPath == "/" {
 			continue
 		} else if r.img.FileCatalog.Exists(*ref) {
 			metadata, err := r.img.FileCatalog.Get(*ref)
 			if err != nil {
-				return nil, fmt.Errorf("unable to get file metadata for path=%q: %w", ref.Path, err)
+				return nil, fmt.Errorf("unable to get file metadata for path=%q: %w", ref.RealPath, err)
 			}
 			if metadata.Metadata.IsDir {
 				continue
@@ -56,7 +66,7 @@ func (r *ImageSquashResolver) FilesByPath(paths ...string) ([]Location, error) {
 
 		if resolvedRef != nil && !uniqueFileIDs.Contains(*resolvedRef) {
 			uniqueFileIDs.Add(*resolvedRef)
-			uniqueLocations = append(uniqueLocations, NewLocationFromImage(*resolvedRef, r.img))
+			uniqueLocations = append(uniqueLocations, NewLocationFromImage(path, *resolvedRef, r.img))
 		}
 	}
 
@@ -69,28 +79,28 @@ func (r *ImageSquashResolver) FilesByGlob(patterns ...string) ([]Location, error
 	uniqueLocations := make([]Location, 0)
 
 	for _, pattern := range patterns {
-		refs, err := r.img.SquashedTree().FilesByGlob(pattern)
+		results, err := r.img.SquashedTree().FilesByGlob(pattern)
 		if err != nil {
 			return nil, fmt.Errorf("failed to resolve files by glob (%s): %w", pattern, err)
 		}
 
-		for _, ref := range refs {
+		for _, result := range results {
 			// don't consider directories (special case: there is no path information for /)
-			if ref.Path == "/" {
+			if result.MatchPath == "/" {
 				continue
-			} else if r.img.FileCatalog.Exists(ref) {
-				metadata, err := r.img.FileCatalog.Get(ref)
+			} else if r.img.FileCatalog.Exists(result.Reference) {
+				metadata, err := r.img.FileCatalog.Get(result.Reference)
 				if err != nil {
-					return nil, fmt.Errorf("unable to get file metadata for path=%q: %w", ref.Path, err)
+					return nil, fmt.Errorf("unable to get file metadata for path=%q: %w", result.MatchPath, err)
 				}
 				if metadata.Metadata.IsDir {
 					continue
 				}
 			}
 
-			resolvedLocations, err := r.FilesByPath(string(ref.Path))
+			resolvedLocations, err := r.FilesByPath(string(result.MatchPath))
 			if err != nil {
-				return nil, fmt.Errorf("failed to find files by path (ref=%+v): %w", ref, err)
+				return nil, fmt.Errorf("failed to find files by path (result=%+v): %w", result, err)
 			}
 			for _, resolvedLocation := range resolvedLocations {
 				if !uniqueFileIDs.Contains(resolvedLocation.ref) {
@@ -121,12 +131,12 @@ func (r *ImageSquashResolver) RelativeFileByPath(_ Location, path string) *Locat
 
 // MultipleFileContentsByLocation returns the file contents for all file.References relative to the image. Note that a
 // file.Reference is a path relative to a particular layer, in this case only from the squashed representation.
-func (r *ImageSquashResolver) MultipleFileContentsByLocation(locations []Location) (map[Location]string, error) {
+func (r *ImageSquashResolver) MultipleFileContentsByLocation(locations []Location) (map[Location]io.ReadCloser, error) {
 	return mapLocationRefs(r.img.MultipleFileContentsByRef, locations)
 }
 
 // FileContentsByLocation fetches file contents for a single file reference, irregardless of the source layer.
 // If the path does not exist an error is returned.
-func (r *ImageSquashResolver) FileContentsByLocation(location Location) (string, error) {
+func (r *ImageSquashResolver) FileContentsByLocation(location Location) (io.ReadCloser, error) {
 	return r.img.FileContentsByRef(location.ref)
 }

syft/source/image_squash_resolver_test.go

@@ -8,10 +8,11 @@ import (
 
 func TestImageSquashResolver_FilesByPath(t *testing.T) {
 	cases := []struct {
-		name         string
-		linkPath     string
-		resolveLayer uint
-		resolvePath  string
+		name                 string
+		linkPath             string
+		resolveLayer         uint
+		resolvePath          string
+		forcePositiveHasPath bool
 	}{
 		{
 			name:         "link with previous data",
@@ -41,12 +42,22 @@ func TestImageSquashResolver_FilesByPath(t *testing.T) {
 			name:         "dead link",
 			linkPath:     "/link-dead",
 			resolveLayer: 8,
-			resolvePath:  "/link-dead",
+			resolvePath:  "",
+			// the path should exist, even if the link is dead
+			forcePositiveHasPath: true,
 		},
 		{
 			name:        "ignore directories",
 			linkPath:    "/bin",
 			resolvePath: "",
+			// the path should exist, even if we ignore it
+			forcePositiveHasPath: true,
+		},
+		{
+			name:         "parent is a link (with overridden data)",
+			linkPath:     "/parent-link/file-4.txt",
+			resolveLayer: 11,
+			resolvePath:  "/parent/file-4.txt",
 		},
 	}
 	for _, c := range cases {
@@ -59,6 +70,17 @@ func TestImageSquashResolver_FilesByPath(t *testing.T) {
 				t.Fatalf("could not create resolver: %+v", err)
 			}
 
+			hasPath := resolver.HasPath(c.linkPath)
+			if !c.forcePositiveHasPath {
+				if c.resolvePath != "" && !hasPath {
+					t.Errorf("expected HasPath() to indicate existance, but did not")
+				} else if c.resolvePath == "" && hasPath {
+					t.Errorf("expeced HasPath() to NOT indicate existance, but does")
+				}
+			} else if !hasPath {
+				t.Errorf("expected HasPath() to indicate existance, but did not (force path)")
+			}
+
 			refs, err := resolver.FilesByPath(c.linkPath)
 			if err != nil {
 				t.Fatalf("could not use resolver: %+v", err)
@@ -80,8 +102,12 @@ func TestImageSquashResolver_FilesByPath(t *testing.T) {
 
 			actual := refs[0]
 
-			if actual.Path != c.resolvePath {
-				t.Errorf("bad resolve path: '%s'!='%s'", actual.Path, c.resolvePath)
+			if string(actual.ref.RealPath) != c.resolvePath {
+				t.Errorf("bad resolve path: '%s'!='%s'", string(actual.ref.RealPath), c.resolvePath)
+			}
+
+			if c.resolvePath != "" && string(actual.ref.RealPath) != actual.RealPath {
+				t.Errorf("we should always prefer real paths over ones with links")
 			}
 
 			entry, err := img.FileCatalog.Get(actual.ref)
@@ -89,8 +115,8 @@ func TestImageSquashResolver_FilesByPath(t *testing.T) {
 				t.Fatalf("failed to get metadata: %+v", err)
 			}
 
-			if entry.Source.Metadata.Index != c.resolveLayer {
-				t.Errorf("bad resolve layer: '%d'!='%d'", entry.Source.Metadata.Index, c.resolveLayer)
+			if entry.Layer.Metadata.Index != c.resolveLayer {
+				t.Errorf("bad resolve layer: '%d'!='%d'", entry.Layer.Metadata.Index, c.resolveLayer)
 			}
 		})
 	}
@@ -105,39 +131,51 @@ func TestImageSquashResolver_FilesByGlob(t *testing.T) {
 	}{
 		{
 			name:         "link with previous data",
-			glob:         "**link-1",
+			glob:         "**/link-1",
 			resolveLayer: 1,
 			resolvePath:  "/file-1.txt",
 		},
 		{
 			name:         "link with in layer data",
-			glob:         "**link-within",
+			glob:         "**/link-within",
 			resolveLayer: 5,
 			resolvePath:  "/file-3.txt",
 		},
 		{
 			name:         "link with overridden data",
-			glob:         "**link-2",
+			glob:         "**/link-2",
 			resolveLayer: 7,
 			resolvePath:  "/file-2.txt",
 		},
 		{
 			name:         "indirect link (with overridden data)",
-			glob:         "**link-indirect",
+			glob:         "**/link-indirect",
 			resolveLayer: 7,
 			resolvePath:  "/file-2.txt",
 		},
 		{
-			name:         "dead link",
-			glob:         "**link-dead",
-			resolveLayer: 8,
-			resolvePath:  "/link-dead",
+			name: "dead link",
+			glob: "**/link-dead",
+			// dead links are dead! they shouldn't match on globs
+			resolvePath: "",
 		},
 		{
 			name:        "ignore directories",
 			glob:        "**/bin",
 			resolvePath: "",
 		},
+		{
+			name:         "parent without link",
+			glob:         "**/parent/*.txt",
+			resolveLayer: 11,
+			resolvePath:  "/parent/file-4.txt",
+		},
+		{
+			name:         "parent is a link (override)",
+			glob:         "**/parent-link/file-4.txt",
+			resolveLayer: 11,
+			resolvePath:  "/parent/file-4.txt",
+		},
 	}
 	for _, c := range cases {
 		t.Run(c.name, func(t *testing.T) {
@@ -170,8 +208,12 @@ func TestImageSquashResolver_FilesByGlob(t *testing.T) {
 
 			actual := refs[0]
 
-			if actual.Path != c.resolvePath {
-				t.Errorf("bad resolve path: '%s'!='%s'", actual.Path, c.resolvePath)
+			if string(actual.ref.RealPath) != c.resolvePath {
+				t.Errorf("bad resolve path: '%s'!='%s'", string(actual.ref.RealPath), c.resolvePath)
+			}
+
+			if c.resolvePath != "" && string(actual.ref.RealPath) != actual.RealPath {
+				t.Errorf("we should always prefer real paths over ones with links")
 			}
 
 			entry, err := img.FileCatalog.Get(actual.ref)
@@ -179,8 +221,8 @@ func TestImageSquashResolver_FilesByGlob(t *testing.T) {
 				t.Fatalf("failed to get metadata: %+v", err)
 			}
 
-			if entry.Source.Metadata.Index != c.resolveLayer {
-				t.Errorf("bad resolve layer: '%d'!='%d'", entry.Source.Metadata.Index, c.resolveLayer)
+			if entry.Layer.Metadata.Index != c.resolveLayer {
+				t.Errorf("bad resolve layer: '%d'!='%d'", entry.Layer.Metadata.Index, c.resolveLayer)
 			}
 		})
 	}

syft/source/location.go

@@ -7,9 +7,11 @@ import (
 	"github.com/anchore/stereoscope/pkg/image"
 )
 
-// Location represents a path relative to a particular filesystem.
+// Location represents a path relative to a particular filesystem resolved to a specific file.Reference. This struct is used as a key
+// in content fetching to uniquely identify a file relative to a request (the VirtualPath).
 type Location struct {
-	Path         string         `json:"path"`              // The string path of the location (e.g. /etc/hosts)
+	RealPath     string         `json:"path"`              // The path where all path ancestors have no hardlinks / symlinks
+	VirtualPath  string         `json:"-"`                 // The path to the file which may or may not have hardlinks / symlinks
 	FileSystemID string         `json:"layerID,omitempty"` // An ID representing the filesystem. For container images this is a layer digest, directories or root filesystem this is blank.
 	ref          file.Reference // The file reference relative to the stereoscope.FileCatalog that has more information about this location.
 }
@@ -17,24 +19,26 @@ type Location struct {
 // NewLocation creates a new Location representing a path without denoting a filesystem or FileCatalog reference.
 func NewLocation(path string) Location {
 	return Location{
-		Path: path,
+		RealPath: path,
 	}
 }
 
 // NewLocationFromImage creates a new Location representing the given path (extracted from the ref) relative to the given image.
-func NewLocationFromImage(ref file.Reference, img *image.Image) Location {
+func NewLocationFromImage(virtualPath string, ref file.Reference, img *image.Image) Location {
 	entry, err := img.FileCatalog.Get(ref)
 	if err != nil {
 		log.Warnf("unable to find file catalog entry for ref=%+v", ref)
 		return Location{
-			Path: string(ref.Path),
-			ref:  ref,
+			VirtualPath: virtualPath,
+			RealPath:    string(ref.RealPath),
+			ref:         ref,
 		}
 	}
 
 	return Location{
-		Path:         string(ref.Path),
-		FileSystemID: entry.Source.Metadata.Digest,
+		VirtualPath:  virtualPath,
+		RealPath:     string(ref.RealPath),
+		FileSystemID: entry.Layer.Metadata.Digest,
 		ref:          ref,
 	}
 }

syft/source/mock_resolver.go

@@ -0,0 +1,112 @@
+package source
+
+import (
+	"fmt"
+	"io"
+	"os"
+
+	"github.com/anchore/syft/internal/file"
+)
+
+var _ Resolver = (*MockResolver)(nil)
+
+// MockResolver implements the Resolver interface and is intended for use *only in test code*.
+// It provides an implementation that can resolve local filesystem paths using only a provided discrete list of file
+// paths, which are typically paths to test fixtures.
+type MockResolver struct {
+	Locations []Location
+}
+
+// NewMockResolverForPaths creates a new MockResolver, where the only resolvable
+// files are those specified by the supplied paths.
+func NewMockResolverForPaths(paths ...string) *MockResolver {
+	var locations []Location
+	for _, p := range paths {
+		locations = append(locations, NewLocation(p))
+	}
+
+	return &MockResolver{Locations: locations}
+}
+
+// HasPath indicates if the given path exists in the underlying source.
+func (r MockResolver) HasPath(path string) bool {
+	for _, l := range r.Locations {
+		if l.RealPath == path {
+			return true
+		}
+	}
+	return false
+}
+
+// String returns the string representation of the MockResolver.
+func (r MockResolver) String() string {
+	return fmt.Sprintf("mock:(%s,...)", r.Locations[0].RealPath)
+}
+
+// FileContentsByLocation fetches file contents for a single location. If the
+// path does not exist, an error is returned.
+func (r MockResolver) FileContentsByLocation(location Location) (io.ReadCloser, error) {
+	for _, l := range r.Locations {
+		if l == location {
+			return os.Open(location.RealPath)
+		}
+	}
+
+	return nil, fmt.Errorf("no file for location: %v", location)
+}
+
+// MultipleFileContentsByLocation returns the file contents for all specified Locations.
+func (r MockResolver) MultipleFileContentsByLocation(locations []Location) (map[Location]io.ReadCloser, error) {
+	results := make(map[Location]io.ReadCloser)
+	for _, l := range locations {
+		contents, err := r.FileContentsByLocation(l)
+		if err != nil {
+			return nil, err
+		}
+		results[l] = contents
+	}
+
+	return results, nil
+}
+
+// FilesByPath returns all Locations that match the given paths.
+func (r MockResolver) FilesByPath(paths ...string) ([]Location, error) {
+	var results []Location
+	for _, p := range paths {
+		for _, location := range r.Locations {
+			if p == location.RealPath {
+				results = append(results, NewLocation(p))
+			}
+		}
+	}
+
+	return results, nil
+}
+
+// FilesByGlob returns all Locations that match the given path glob pattern.
+func (r MockResolver) FilesByGlob(patterns ...string) ([]Location, error) {
+	var results []Location
+	for _, pattern := range patterns {
+		for _, location := range r.Locations {
+			if file.GlobMatch(pattern, location.RealPath) {
+				results = append(results, location)
+			}
+		}
+	}
+
+	return results, nil
+}
+
+// RelativeFileByPath returns a single Location for the given path.
+func (r MockResolver) RelativeFileByPath(_ Location, path string) *Location {
+	paths, err := r.FilesByPath(path)
+	if err != nil {
+		return nil
+	}
+
+	if len(paths) < 1 {
+		return nil
+	}
+
+	return &paths[0]
+}

syft/source/resolver.go

@@ -2,6 +2,7 @@ package source
 
 import (
 	"fmt"
+	"io"
 
 	"github.com/anchore/stereoscope/pkg/image"
 )
@@ -14,13 +15,15 @@ type Resolver interface {
 
 // ContentResolver knows how to get file content for given file.References
 type ContentResolver interface {
-	FileContentsByLocation(Location) (string, error)
-	MultipleFileContentsByLocation([]Location) (map[Location]string, error)
-	// TODO: we should consider refactoring to return a set of io.Readers or file.Openers instead of the full contents themselves (allow for optional buffering).
+	FileContentsByLocation(Location) (io.ReadCloser, error)
+	// TODO: it is possible to be given duplicate locations that will be overridden in the map (key), a subtle problem that coule easily be misued.
+	MultipleFileContentsByLocation([]Location) (map[Location]io.ReadCloser, error)
 }
 
-// FileResolver knows how to get file.References for given string paths and globs
+// FileResolver knows how to get a Location for given string paths and globs
 type FileResolver interface {
+	// HasPath indicates if the given path exists in the underlying source.
+	HasPath(path string) bool
 	// FilesByPath fetches a set of file references which have the given path (for an image, there may be multiple matches)
 	FilesByPath(paths ...string) ([]Location, error)
 	// FilesByGlob fetches a set of file references which the given glob matches

syft/source/source_test.go

@@ -1,6 +1,7 @@
 package source
 
 import (
+	"io/ioutil"
 	"os"
 	"testing"
 
@@ -137,9 +138,14 @@ func TestMultipleFileContentsByLocation(t *testing.T) {
 			location := locations[0]
 
 			contents, err := p.Resolver.MultipleFileContentsByLocation([]Location{location})
-			content := contents[location]
+			contentReader := contents[location]
 
-			if content != test.expected {
+			content, err := ioutil.ReadAll(contentReader)
+			if err != nil {
+				t.Fatalf("cannot read contents: %+v", err)
+			}
+
+			if string(content) != test.expected {
 				t.Errorf("unexpected contents from file: '%s' != '%s'", content, test.expected)
 			}
 

syft/source/test-fixtures/image-simple/Dockerfile

@@ -0,0 +1,6 @@
+# Note: changes to this file will result in updating several test values. Consider making a new image fixture instead of editing this one.
+FROM scratch
+ADD file-1.txt /somefile-1.txt
+ADD file-2.txt /somefile-2.txt
+# note: adding a directory will behave differently on docker engine v18 vs v19
+ADD target /

syft/source/test-fixtures/image-simple/file-1.txt

@@ -0,0 +1 @@
+this file has contents

syft/source/test-fixtures/image-simple/file-2.txt

@@ -0,0 +1 @@
+file-2 contents!

syft/source/test-fixtures/image-simple/target/really/nested/file-3.txt

@@ -0,0 +1,2 @@
+another file!
+with lines...

syft/source/test-fixtures/image-symlinks/Dockerfile

@@ -21,4 +21,12 @@ RUN ln -s ./link-2 link-indirect
 ADD new-file-2.txt file-2.txt
 
 # LAYER 8: dead link
-RUN ln -s ./i-dont-exist.txt link-dead
+RUN ln -s ./i-dont-exist.txt link-dead
+
+# LAYER 9: add the parent dir
+ADD parent /parent
+# LAYER 10: parent is a symlink
+RUN ln -s /parent parent-link
+
+# LAYER 11: parent is a symlink and the child target is overridden
+COPY new-file-4.txt /parent-link/file-4.txt

syft/source/test-fixtures/image-symlinks/new-file-4.txt

@@ -0,0 +1 @@
+override file 4!

syft/source/test-fixtures/image-symlinks/parent/file-4.txt

@@ -0,0 +1 @@
+the 4th file!

test/integration/distro_test.go

@@ -16,7 +16,7 @@ func TestDistroImage(t *testing.T) {
 	tarPath := imagetest.GetFixtureImageTarPath(t, fixtureImageName)
 	defer cleanup()
 
-	_, _, actualDistro, err := syft.Catalog("docker-archive:"+tarPath, source.AllLayersScope)
+	_, _, actualDistro, err := syft.Catalog("docker-archive:"+tarPath, source.SquashedScope)
 	if err != nil {
 		t.Fatalf("failed to catalog image: %+v", err)
 	}

test/integration/document_import_test.go

@@ -31,7 +31,7 @@ func TestCatalogFromJSON(t *testing.T) {
 			tarPath := imagetest.GetFixtureImageTarPath(t, test.fixture)
 			defer cleanup()
 
-			expectedSource, expectedCatalog, expectedDistro, err := syft.Catalog("docker-archive:"+tarPath, source.AllLayersScope)
+			expectedSource, expectedCatalog, expectedDistro, err := syft.Catalog("docker-archive:"+tarPath, source.SquashedScope)
 			if err != nil {
 				t.Fatalf("failed to catalog image: %+v", err)
 			}
@@ -81,6 +81,12 @@ func TestCatalogFromJSON(t *testing.T) {
 					e.Metadata = metadata
 				}
 
+				// ignore the virtual path on the location for now
+				for l := range a.Locations {
+					a.Locations[l].VirtualPath = ""
+					e.Locations[l].VirtualPath = ""
+				}
+
 				for _, d := range deep.Equal(a, e) {
 					// ignore errors for empty collections vs nil for select fields
 					// TODO: this is brittle, but not dangerously so. We should still find a better way to do this.

test/integration/java_no_main_package_test.go

@@ -0,0 +1,20 @@
+package integration
+
+import (
+	"github.com/anchore/stereoscope/pkg/imagetest"
+	"github.com/anchore/syft/syft"
+	"github.com/anchore/syft/syft/source"
+	"testing"
+)
+
+func TestJavaNoMainPackage(t *testing.T) { // Regression: https://github.com/anchore/syft/issues/252
+	fixtureImageName := "image-java-no-main-package"
+	_, cleanup := imagetest.GetFixtureImage(t, "docker-archive", fixtureImageName)
+	tarPath := imagetest.GetFixtureImageTarPath(t, fixtureImageName)
+	defer cleanup()
+
+	_, _, _, err := syft.Catalog("docker-archive:"+tarPath, source.SquashedScope)
+	if err != nil {
+		t.Fatalf("failed to catalog image: %+v", err)
+	}
+}

test/integration/json_schema_test.go

@@ -60,7 +60,7 @@ func TestJsonSchemaImg(t *testing.T) {
 	tarPath := imagetest.GetFixtureImageTarPath(t, fixtureImageName)
 	defer cleanup()
 
-	src, catalog, _, err := syft.Catalog("docker-archive:"+tarPath, source.AllLayersScope)
+	src, catalog, _, err := syft.Catalog("docker-archive:"+tarPath, source.SquashedScope)
 	if err != nil {
 		t.Fatalf("failed to catalog image: %+v", err)
 	}
@@ -87,7 +87,7 @@ func TestJsonSchemaImg(t *testing.T) {
 }
 
 func TestJsonSchemaDirs(t *testing.T) {
-	src, catalog, _, err := syft.Catalog("dir:test-fixtures/image-pkg-coverage", source.AllLayersScope)
+	src, catalog, _, err := syft.Catalog("dir:test-fixtures/image-pkg-coverage", source.SquashedScope)
 	if err != nil {
 		t.Errorf("unable to create source from dir: %+v", err)
 	}

test/integration/pkg_cases_test.go

@@ -15,7 +15,10 @@ var imageOnlyTestCases = []testCase{
 		pkgType:     pkg.GemPkg,
 		pkgLanguage: pkg.Ruby,
 		pkgInfo: map[string]string{
+			// specifications in the root specification directory
 			"bundler": "2.1.4",
+			// specifications in named directories
+			"unbundler": "3.1.4",
 		},
 	},
 	{

test/integration/pkg_coverage_test.go

@@ -18,7 +18,7 @@ func TestPkgCoverageImage(t *testing.T) {
 	tarPath := imagetest.GetFixtureImageTarPath(t, fixtureImageName)
 	defer cleanup()
 
-	_, catalog, _, err := syft.Catalog("docker-archive:"+tarPath, source.AllLayersScope)
+	_, catalog, _, err := syft.Catalog("docker-archive:"+tarPath, source.SquashedScope)
 	if err != nil {
 		t.Fatalf("failed to catalog image: %+v", err)
 	}
@@ -100,7 +100,7 @@ func TestPkgCoverageImage(t *testing.T) {
 }
 
 func TestPkgCoverageDirectory(t *testing.T) {
-	_, catalog, _, err := syft.Catalog("dir:test-fixtures/image-pkg-coverage", source.AllLayersScope)
+	_, catalog, _, err := syft.Catalog("dir:test-fixtures/image-pkg-coverage", source.SquashedScope)
 
 	if err != nil {
 		t.Errorf("unable to create source from dir: %+v", err)

test/integration/regression_test.go

@@ -24,7 +24,7 @@ func TestRegression212ApkBufferSize(t *testing.T) {
 		t.Fatalf("failed to catalog image: %+v", err)
 	}
 
-	expectedPkgs := 57
+	expectedPkgs := 58
 	actualPkgs := 0
 	for range catalog.Enumerate(pkg.ApkPkg) {
 		actualPkgs += 1

test/integration/test-fixtures/image-java-no-main-package/Dockerfile

@@ -0,0 +1,23 @@
+FROM jenkins:2.60.3
+
+USER root
+
+WORKDIR /usr/share/jenkins
+
+RUN mkdir tmp
+
+WORKDIR /usr/share/jenkins/tmp
+
+RUN apt-get update 2>&1 > /dev/null && apt-get install -y less zip 2>&1 > /dev/null
+
+RUN unzip ../jenkins.war 2>&1 > /dev/null
+
+RUN rm -f ./META-INF/MANIFEST.MF
+
+WORKDIR /usr/share/jenkins
+
+RUN rm -rf jenkins.war
+
+RUN cd ./tmp && zip -r ../jenkins.war . && cd ..
+
+RUN rm -rf ./tmp

test/integration/test-fixtures/image-large-apk-data/Dockerfile

@@ -1,2 +1,5 @@
-FROM alpine:latest
-RUN apk add tzdata vim alpine-sdk
+FROM alpine@sha256:d9a7354e3845ea8466bb00b22224d9116b183e594527fb5b6c3d30bc01a20378
+RUN apk add --no-cache \
+            tzdata=2021a-r0 \
+            vim=8.2.2320-r0 \
+            alpine-sdk=1.0-r0

test/integration/test-fixtures/image-pkg-coverage/ruby/specifications/default/unbundler.gemspec

@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+# -*- encoding: utf-8 -*-                                                                                                                                         
+# stub: unbundler 2.1.4 ruby lib                                                                                                                                    
+                                                                                                                                                                  
+Gem::Specification.new do |s|                                                                                                                                     
+    s.name = "unbundler".freeze
+    s.version = "3.1.4"
+                                                                                                                                                                    
+    s.required_rubygems_version = Gem::Requirement.new(">= 2.5.2".freeze) if s.respond_to? :required_rubygems_version=                                              
+    s.require_paths = ["lib".freeze]                                                                                                                                
+    s.authors = ["Andr\u00E9 Arko".freeze, "Samuel Giddins".freeze, "Colby Swandale".freeze, "Hiroshi Shibata".freeze, "David Rodr\u00EDguez".freeze, "Grey Baker".f
+    s.bindir = "exe".freeze                                                                                                                                         
+    s.date = "2020-01-05"                                                                                                                                           
+    s.description = "Bundler manages an application's dependencies through its entire life, across many machines, systematically and repeatably".freeze             
+    s.email = ["team@unbundler.io".freeze]                                                                                                                            
+    s.executables = ["unbundle".freeze, "unbundler".freeze]                                                                                                             
+    s.files = ["exe/unbundle".freeze, "exe/unbundler".freeze]                                                                                                           
+    s.homepage = "https://unbundler.io".freeze                                                                                                                        
+    s.licenses = ["MIT".freeze]                                                                                                                                     
+    s.required_ruby_version = Gem::Requirement.new(">= 2.3.0".freeze)                                                                                               
+    s.rubygems_version = "3.1.2".freeze                                                                                                                             
+    s.summary = "The best way to manage your application's dependencies".freeze                                                                                     
+                                                                                                                                                                    
+    s.installed_by_version = "3.1.2" if s.respond_to? :installed_by_version                                                                                         
+  end 

ui/event_handlers.go

@@ -312,7 +312,7 @@ func CatalogerStartedHandler(ctx context.Context, fr *frame.Frame, event partybu
 // ImportStartedHandler shows the intermittent upload progress to Anchore Enterprise.
 // nolint:dupl
 func ImportStartedHandler(ctx context.Context, fr *frame.Frame, event partybus.Event, wg *sync.WaitGroup) error {
-	_, prog, err := syftEventParsers.ParseImportStarted(event)
+	host, prog, err := syftEventParsers.ParseImportStarted(event)
 	if err != nil {
 		return fmt.Errorf("bad %s event: %w", event.Type, err)
 	}
@@ -348,7 +348,8 @@ func ImportStartedHandler(ctx context.Context, fr *frame.Frame, event partybus.E
 
 		spin := color.Green.Sprint(completedStatus)
 		title = tileFormat.Sprint("Uploaded image")
-		_, _ = io.WriteString(line, fmt.Sprintf(statusTitleTemplate, spin, title))
+		auxInfo := auxInfoFormat.Sprintf("[%s]", host)
+		_, _ = io.WriteString(line, fmt.Sprintf(statusTitleTemplate+"%s", spin, title, auxInfo))
 	}()
 	return err
 }