Merge pull request #308 from anchore/prefer-real-paths-on-glob-results

Prefer real paths over those with links for glob results

.gitignore

@@ -3,6 +3,7 @@ CHANGELOG.md
 /snapshot
 .server/
 .vscode/
+.history/
 *.fingerprint
 *.tar
 *.jar
@@ -32,3 +33,5 @@ coverage.txt
 
 # macOS Finder metadata
 .DS_STORE
+
+*.profile

.golangci.yaml

@@ -31,7 +31,6 @@ linters:
     - misspell
     - nakedret
     - nolintlint
-    - prealloc
     - rowserrcheck
     - scopelint
     - staticcheck
@@ -55,5 +54,6 @@ linters:
 #    - lll         # without a way to specify per-line exception cases, this is not usable
 #    - maligned    # this is an excellent linter, but tricky to optimize and we are not sensitive to memory layout optimizations
 #    - nestif
+#    - prealloc    # following this rule isn't consistently a good idea, as it sometimes forces unnecessary allocations that result in less idiomatic code
 #    - testpackage
-#    - wsl
+#    - wsl

Makefile

@@ -15,7 +15,10 @@ RESET := $(shell tput -T linux sgr0)
 TITLE := $(BOLD)$(PURPLE)
 SUCCESS := $(BOLD)$(GREEN)
 # the quality gate lower threshold for unit test total % coverage (by function statements)
-COVERAGE_THRESHOLD := 70
+COVERAGE_THRESHOLD := 68
+# CI cache busting values; change these if you want CI to not use previous stored cache
+COMPARE_CACHE_BUSTER="f7e689d76a9"
+INTEGRATION_CACHE_BUSTER="789bacdf"
 
 ## Build variables
 DISTDIR=./dist
@@ -140,7 +143,7 @@ integration: ## Run integration tests
 
 # note: this is used by CI to determine if the integration test fixture cache (docker image tars) should be busted
 integration-fingerprint:
-	find test/integration/test-fixtures/image-* -type f -exec md5sum {} + | awk '{print $1}' | sort | md5sum | tee test/integration/test-fixtures/cache.fingerprint
+	find test/integration/test-fixtures/image-* -type f -exec md5sum {} + | awk '{print $1}' | sort | md5sum | tee test/integration/test-fixtures/cache.fingerprint && echo "$(INTEGRATION_CACHE_BUSTER)" >> test/integration/test-fixtures/cache.fingerprint
 
 .PHONY: java-packages-fingerprint
 java-packages-fingerprint:
@@ -154,8 +157,7 @@ fixtures:
 
 .PHONY: generate-json-schema
 generate-json-schema:  ## Generate a new json schema
-	cd schema/json
-	go run generate.go
+	cd schema/json && go run generate.go
 
 .PHONY: clear-test-cache
 clear-test-cache: ## Delete all test cache (built docker image tars)
@@ -174,6 +176,7 @@ $(SNAPSHOTDIR): ## Build snapshot release binaries and packages
 	BUILD_GIT_TREE_STATE=$(GITTREESTATE) \
 	$(TEMPDIR)/goreleaser release --skip-publish --rm-dist --snapshot --config $(TEMPDIR)/goreleaser.yaml
 
+# note: we cannot clean the snapshot directory since the pipeline builds the snapshot separately
 .PHONY: acceptance-mac
 acceptance-mac: $(SNAPSHOTDIR) ## Run acceptance tests on build snapshot binaries and packages (Mac)
 	$(call title,Running acceptance test: Run on Mac)
@@ -183,13 +186,14 @@ acceptance-mac: $(SNAPSHOTDIR) ## Run acceptance tests on build snapshot binarie
 			$(ACC_TEST_IMAGE) \
 			$(RESULTSDIR)
 
+# note: we cannot clean the snapshot directory since the pipeline builds the snapshot separately
 .PHONY: acceptance-linux
 acceptance-linux: acceptance-test-deb-package-install acceptance-test-rpm-package-install ## Run acceptance tests on build snapshot binaries and packages (Linux)
 
 # note: this is used by CI to determine if the inline-scan report cache should be busted for the inline-compare tests
 .PHONY: compare-fingerprint
 compare-fingerprint:
-	find test/inline-compare/* -type f -exec md5sum {} + | grep -v '\-reports' | grep -v 'fingerprint' | awk '{print $1}' | sort | md5sum | tee test/inline-compare/inline-compare.fingerprint
+	find test/inline-compare/* -type f -exec md5sum {} + | grep -v '\-reports' | grep -v 'fingerprint' | awk '{print $1}' | sort | md5sum | tee test/inline-compare/inline-compare.fingerprint && echo "$(COMPARE_CACHE_BUSTER)" >> test/inline-compare/inline-compare.fingerprint
 
 .PHONY: compare-snapshot
 compare-snapshot: $(SNAPSHOTDIR) ## Compare the reports of a run of a snapshot build of syft against inline-scan

README.md

@@ -81,30 +81,58 @@ Configuration search paths:
 Configuration options (example values are the default):
 
 ```yaml
-# same as -o ; the output format of the SBOM report (options: table, text, json)
+# the output format of the SBOM report (options: table, text, json)
+# same as -o ; SYFT_OUTPUT env var
 output: "table"
 
-# same as -s ; the search space to look for packages (options: all-layers, squashed)
+# the search space to look for packages (options: all-layers, squashed)
+# same as -s ; SYFT_SCOPE env var
 scope: "squashed"
 
-# same as -q ; suppress all output (except for the SBOM report)
+# suppress all output (except for the SBOM report)
+# same as -q ; SYFT_QUIET env var
 quiet: false
 
+# enable/disable checking for application updates on startup
+# same as SYFT_CHECK_FOR_APP_UPDATE env var
+check-for-app-update: true
+
 log:
   # use structured logging
+  # same as SYFT_LOG_STRUCTURED env var
   structured: false
 
   # the log level; note: detailed logging suppress the ETUI
+  # same as SYFT_LOG_LEVEL env var
   level: "error"
 
   # location to write the log file (default is not to have a log file)
+  # same as SYFT_LOG_FILE env var
   file: ""
 
-# enable/disable checking for application updates on startup
-check-for-app-update: true
-```
+anchore:
+  # (feature-preview) enable uploading of results to Anchore Enterprise automatically (supported on Enterprise 3.0+)
+  # same as SYFT_ANCHORE_UPLOAD_ENABLED env var
+  upload-enabled: false
 
-## Future plans
+  # (feature-preview) the Anchore Enterprise Host or URL to upload results to (supported on Enterprise 3.0+)
+  # same as -H ; SYFT_ANCHORE_HOST env var
+  host: ""
 
-The following areas of potential development are currently being investigated:
-- Establish a stable interchange format w/Grype
+  # (feature-preview) the path after the host to the Anchore External API (supported on Enterprise 3.0+)
+  # same as SYFT_ANCHORE_PATH env var
+  path: ""
+
+  # (feature-preview) the username to authenticate against Anchore Enterprise (supported on Enterprise 3.0+)
+  # same as -u ; SYFT_ANCHORE_USERNAME env var
+  username: ""
+
+  # (feature-preview) the password to authenticate against Anchore Enterprise (supported on Enterprise 3.0+)
+  # same as -p ; SYFT_ANCHORE_PASSWORD env var
+  password: ""
+
+  # (feature-preview) path to dockerfile to be uploaded with the syft results to Anchore Enterprise (supported on Enterprise 3.0+)
+  # same as -d ; SYFT_ANCHORE_DOCKERFILE env var
+  dockerfile: ""
+
+```

cmd/cmd.go

@@ -4,21 +4,17 @@ import (
 	"fmt"
 	"os"
 
-	"github.com/gookit/color"
-
-	"github.com/spf13/cobra"
-
-	"github.com/anchore/syft/syft/presenter"
-	"github.com/anchore/syft/syft/source"
-
 	"github.com/anchore/stereoscope"
 	"github.com/anchore/syft/internal/config"
 	"github.com/anchore/syft/internal/log"
 	"github.com/anchore/syft/internal/logger"
 	"github.com/anchore/syft/syft"
+	"github.com/anchore/syft/syft/presenter"
+	"github.com/anchore/syft/syft/source"
+	"github.com/gookit/color"
+	"github.com/spf13/cobra"
 	"github.com/spf13/viper"
 	"github.com/wagoodman/go-partybus"
-	"gopkg.in/yaml.v2"
 )
 
 var appConfig *config.Application
@@ -57,8 +53,13 @@ func setGlobalCliOptions() {
 		os.Exit(1)
 	}
 
+	setGlobalFormatOptions()
+	setGlobalUploadOptions()
+}
+
+func setGlobalFormatOptions() {
 	// output & formatting options
-	flag = "output"
+	flag := "output"
 	rootCmd.Flags().StringP(
 		flag, "o", string(presenter.TablePresenter),
 		fmt.Sprintf("report output formatter, options=%v", presenter.Options),
@@ -81,12 +82,67 @@ func setGlobalCliOptions() {
 	rootCmd.Flags().CountVarP(&cliOpts.Verbosity, "verbose", "v", "increase verbosity (-v = info, -vv = debug)")
 }
 
+func setGlobalUploadOptions() {
+	flag := "host"
+	rootCmd.Flags().StringP(
+		flag, "H", "",
+		"the hostname or URL of the Anchore Enterprise instance to upload to",
+	)
+	if err := viper.BindPFlag("anchore.host", rootCmd.Flags().Lookup(flag)); err != nil {
+		fmt.Printf("unable to bind flag '%s': %+v", flag, err)
+		os.Exit(1)
+	}
+
+	flag = "username"
+	rootCmd.Flags().StringP(
+		flag, "u", "",
+		"the username to authenticate against Anchore Enterprise",
+	)
+	if err := viper.BindPFlag("anchore.username", rootCmd.Flags().Lookup(flag)); err != nil {
+		fmt.Printf("unable to bind flag '%s': %+v", flag, err)
+		os.Exit(1)
+	}
+
+	flag = "password"
+	rootCmd.Flags().StringP(
+		flag, "p", "",
+		"the password to authenticate against Anchore Enterprise",
+	)
+	if err := viper.BindPFlag("anchore.password", rootCmd.Flags().Lookup(flag)); err != nil {
+		fmt.Printf("unable to bind flag '%s': %+v", flag, err)
+		os.Exit(1)
+	}
+
+	flag = "dockerfile"
+	rootCmd.Flags().StringP(
+		flag, "d", "",
+		"include dockerfile for upload to Anchore Enterprise",
+	)
+	if err := viper.BindPFlag("anchore.dockerfile", rootCmd.Flags().Lookup(flag)); err != nil {
+		fmt.Printf("unable to bind flag '#{flag}': #{err}")
+		os.Exit(1)
+	}
+
+	flag = "overwrite-existing-image"
+	rootCmd.Flags().Bool(
+		flag, false,
+		"overwrite an existing image during the upload to Anchore Enterprise",
+	)
+	if err := viper.BindPFlag("anchore.overwrite-existing-image", rootCmd.Flags().Lookup(flag)); err != nil {
+		fmt.Printf("unable to bind flag '#{flag}': #{err}")
+		os.Exit(1)
+	}
+}
+
 func initAppConfig() {
-	cfg, err := config.LoadConfigFromFile(viper.GetViper(), &cliOpts)
+	cfgVehicle := viper.GetViper()
+	wasHostnameSet := rootCmd.Flags().Changed("host")
+	cfg, err := config.LoadApplicationConfig(cfgVehicle, cliOpts, wasHostnameSet)
 	if err != nil {
 		fmt.Printf("failed to load application config: \n\t%+v\n", err)
 		os.Exit(1)
 	}
+
 	appConfig = cfg
 }
 
@@ -107,13 +163,7 @@ func initLogging() {
 }
 
 func logAppConfig() {
-	appCfgStr, err := yaml.Marshal(&appConfig)
-
-	if err != nil {
-		log.Debugf("Could not display application config: %+v", err)
-	} else {
-		log.Debugf("Application config:\n%+v", color.Magenta.Sprint(string(appCfgStr)))
-	}
+	log.Debugf("Application config:\n%+v", color.Magenta.Sprint(appConfig.String()))
 }
 
 func initEventBus() {

cmd/root.go

@@ -3,17 +3,24 @@ package cmd
 import (
 	"context"
 	"fmt"
+	"io/ioutil"
 	"os"
 	"strings"
 
+	"github.com/pkg/profile"
+
 	"github.com/anchore/syft/internal"
+	"github.com/anchore/syft/internal/anchore"
 	"github.com/anchore/syft/internal/bus"
 	"github.com/anchore/syft/internal/log"
 	"github.com/anchore/syft/internal/ui"
 	"github.com/anchore/syft/internal/version"
 	"github.com/anchore/syft/syft"
+	"github.com/anchore/syft/syft/distro"
 	"github.com/anchore/syft/syft/event"
+	"github.com/anchore/syft/syft/pkg"
 	"github.com/anchore/syft/syft/presenter"
+	"github.com/anchore/syft/syft/source"
 	"github.com/docker/docker/api/types"
 	"github.com/docker/docker/api/types/filters"
 	"github.com/docker/docker/client"
@@ -23,7 +30,7 @@ import (
 
 var rootCmd = &cobra.Command{
 	Use:   fmt.Sprintf("%s [SOURCE]", internal.ApplicationName),
-	Short: "A tool for generating a Software Bill Of Materials (SBOM) from container images and filesystems",
+	Short: "A tool for generating a Software Bill Of Materials (PackageSBOM) from container images and filesystems",
 	Long: internal.Tprintf(`
 Supports the following image sources:
     {{.appName}} yourrepo/yourimage:tag     defaults to using images from a Docker daemon
@@ -48,7 +55,20 @@ You can also explicitly specify the scheme to use:
 			}
 			os.Exit(1)
 		}
+
+		if appConfig.Dev.ProfileCPU && appConfig.Dev.ProfileMem {
+			log.Errorf("cannot profile CPU and memory simultaneously")
+			os.Exit(1)
+		}
+
+		if appConfig.Dev.ProfileCPU {
+			defer profile.Start(profile.CPUProfile).Stop()
+		} else if appConfig.Dev.ProfileMem {
+			defer profile.Start(profile.MemProfile).Stop()
+		}
+
 		err := doRunCmd(cmd, args)
+
 		if err != nil {
 			log.Errorf(err.Error())
 			os.Exit(1)
@@ -98,6 +118,13 @@ func startWorker(userInput string) <-chan error {
 			return
 		}
 
+		if appConfig.Anchore.UploadEnabled {
+			if err := doImport(src, src.Metadata, catalog, distro); err != nil {
+				errs <- err
+				return
+			}
+		}
+
 		bus.Publish(partybus.Event{
 			Type:  event.CatalogerFinished,
 			Value: presenter.GetPresenter(appConfig.PresenterOpt, src.Metadata, catalog, distro),
@@ -106,6 +133,64 @@ func startWorker(userInput string) <-chan error {
 	return errs
 }
 
+func doImport(src source.Source, s source.Metadata, catalog *pkg.Catalog, d *distro.Distro) error {
+	// TODO: ETUI element for this
+	log.Infof("uploading results to %s", appConfig.Anchore.Host)
+
+	if src.Metadata.Scheme != source.ImageScheme {
+		return fmt.Errorf("unable to upload results: only images are supported")
+	}
+
+	var dockerfileContents []byte
+	if appConfig.Anchore.Dockerfile != "" {
+		if _, err := os.Stat(appConfig.Anchore.Dockerfile); os.IsNotExist(err) {
+			return fmt.Errorf("unable to read dockerfile=%q: %w", appConfig.Anchore.Dockerfile, err)
+		}
+
+		fh, err := os.Open(appConfig.Anchore.Dockerfile)
+		if err != nil {
+			return fmt.Errorf("unable to open dockerfile=%q: %w", appConfig.Anchore.Dockerfile, err)
+		}
+
+		dockerfileContents, err = ioutil.ReadAll(fh)
+		if err != nil {
+			return fmt.Errorf("unable to read dockerfile=%q: %w", appConfig.Anchore.Dockerfile, err)
+		}
+	}
+
+	var scheme string
+	var hostname = appConfig.Anchore.Host
+	urlFields := strings.Split(hostname, "://")
+	if len(urlFields) > 1 {
+		scheme = urlFields[0]
+		hostname = urlFields[1]
+	}
+
+	c, err := anchore.NewClient(anchore.Configuration{
+		Hostname: hostname,
+		Username: appConfig.Anchore.Username,
+		Password: appConfig.Anchore.Password,
+		Scheme:   scheme,
+	})
+	if err != nil {
+		return fmt.Errorf("failed to create anchore client: %+v", err)
+	}
+
+	importCfg := anchore.ImportConfig{
+		ImageMetadata:           src.Image.Metadata,
+		SourceMetadata:          s,
+		Catalog:                 catalog,
+		Distro:                  d,
+		Dockerfile:              dockerfileContents,
+		OverwriteExistingUpload: appConfig.Anchore.OverwriteExistingImage,
+	}
+
+	if err := c.Import(context.Background(), importCfg); err != nil {
+		return fmt.Errorf("failed to upload results to host=%s: %+v", appConfig.Anchore.Host, err)
+	}
+	return nil
+}
+
 func doRunCmd(_ *cobra.Command, args []string) error {
 	userInput := args[0]
 	errs := startWorker(userInput)

go.mod

@@ -5,11 +5,13 @@ go 1.14
 require (
 	github.com/adrg/xdg v0.2.1
 	github.com/alecthomas/jsonschema v0.0.0-20200530073317-71f438968921
+	github.com/anchore/client-go v0.0.0-20201216213038-a486b838e238
 	github.com/anchore/go-rpmdb v0.0.0-20201106153645-0043963c2e12
 	github.com/anchore/go-testutils v0.0.0-20200925183923-d5f45b0d3c04
 	github.com/anchore/go-version v1.2.2-0.20200701162849-18adb9c92b9b
-	github.com/anchore/stereoscope v0.0.0-20201203153145-3f9a05a624d7
-	github.com/bmatcuk/doublestar v1.3.3
+	github.com/anchore/stereoscope v0.0.0-20210105001222-7beea73cb7e5
+	github.com/antihax/optional v1.0.0
+	github.com/bmatcuk/doublestar/v2 v2.0.4
 	github.com/docker/docker v17.12.0-ce-rc1.0.20200309214505-aa6a9891b09c+incompatible
 	github.com/dustin/go-humanize v1.0.0
 	github.com/facebookincubator/nvdtools v0.1.4
@@ -23,6 +25,7 @@ require (
 	github.com/olekukonko/tablewriter v0.0.4
 	github.com/package-url/packageurl-go v0.1.0
 	github.com/pelletier/go-toml v1.8.0
+	github.com/pkg/profile v1.5.0
 	github.com/scylladb/go-set v1.0.2
 	github.com/sergi/go-diff v1.1.0
 	github.com/sirupsen/logrus v1.6.0

go.sum

@@ -126,19 +126,23 @@ github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuy
 github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
 github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
 github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho=
+github.com/anchore/client-go v0.0.0-20201216213038-a486b838e238 h1:/iI+1cj1a27ow0wj378pPJIm8sCSy6I21Tz6oLbLDQY=
+github.com/anchore/client-go v0.0.0-20201216213038-a486b838e238/go.mod h1:FaODhIA06mxO1E6R32JE0TL1JWZZkmjRIAd4ULvHUKk=
 github.com/anchore/go-rpmdb v0.0.0-20201106153645-0043963c2e12 h1:xbeIbn5F52JVx3RUIajxCj8b0y+9lywspql4sFhcxWQ=
 github.com/anchore/go-rpmdb v0.0.0-20201106153645-0043963c2e12/go.mod h1:juoyWXIj7sJ1IDl4E/KIfyLtovbs5XQVSIdaQifFQT8=
 github.com/anchore/go-testutils v0.0.0-20200925183923-d5f45b0d3c04 h1:VzprUTpc0vW0nnNKJfJieyH/TZ9UYAnTZs5/gHTdAe8=
 github.com/anchore/go-testutils v0.0.0-20200925183923-d5f45b0d3c04/go.mod h1:6dK64g27Qi1qGQZ67gFmBFvEHScy0/C8qhQhNe5B5pQ=
 github.com/anchore/go-version v1.2.2-0.20200701162849-18adb9c92b9b h1:e1bmaoJfZVsCYMrIZBpFxwV26CbsuoEh5muXD5I1Ods=
 github.com/anchore/go-version v1.2.2-0.20200701162849-18adb9c92b9b/go.mod h1:Bkc+JYWjMCF8OyZ340IMSIi2Ebf3uwByOk6ho4wne1E=
-github.com/anchore/stereoscope v0.0.0-20201106140100-12e75c48f409 h1:xKSpDRjmYrEFrdMeDh4AuSUAFc99pdro6YFBKxy2um0=
-github.com/anchore/stereoscope v0.0.0-20201106140100-12e75c48f409/go.mod h1:2Jja/4l0zYggW52og+nn0rut4i+OYjCf9vTyrM8RT4E=
-github.com/anchore/stereoscope v0.0.0-20201130153727-b3f1fad856b0 h1:wa0hdnvBeCpI+rmzDbPG7k5SKlsGkot7aZ8Az1i/vws=
-github.com/anchore/stereoscope v0.0.0-20201130153727-b3f1fad856b0/go.mod h1:2Jja/4l0zYggW52og+nn0rut4i+OYjCf9vTyrM8RT4E=
-github.com/anchore/stereoscope v0.0.0-20201203153145-3f9a05a624d7 h1:G3LnRqHL/IIeQZTAMtDOJNYfSYsXLNCZX4DCiS0R0FY=
-github.com/anchore/stereoscope v0.0.0-20201203153145-3f9a05a624d7/go.mod h1:2Jja/4l0zYggW52og+nn0rut4i+OYjCf9vTyrM8RT4E=
+github.com/anchore/stereoscope v0.0.0-20210104203718-4c1d1bd9a255 h1:Ng7BDr9PQTCztANogjfEdEjjWUylhlPyZPhtarIGo00=
+github.com/anchore/stereoscope v0.0.0-20210104203718-4c1d1bd9a255/go.mod h1:BMdPL0QEIYfpjQ3M7sHYZvuh6+vcomqF3TMHL8gr6Vw=
+github.com/anchore/stereoscope v0.0.0-20210105000809-428eda0b2ec6 h1:JWpsV/8x1fuCYjJmNjT43cVFblLTpO/ISDnePukiTNw=
+github.com/anchore/stereoscope v0.0.0-20210105000809-428eda0b2ec6/go.mod h1:BMdPL0QEIYfpjQ3M7sHYZvuh6+vcomqF3TMHL8gr6Vw=
+github.com/anchore/stereoscope v0.0.0-20210105001222-7beea73cb7e5 h1:NGRfS6BZKElgiMbqdoH9iQn+6oxT7CJdZYrqgwvGkWY=
+github.com/anchore/stereoscope v0.0.0-20210105001222-7beea73cb7e5/go.mod h1:BMdPL0QEIYfpjQ3M7sHYZvuh6+vcomqF3TMHL8gr6Vw=
 github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883/go.mod h1:rCTlJbsFo29Kk6CurOXKm700vrz8f0KW0JNfpkRJY/8=
+github.com/antihax/optional v1.0.0 h1:xK2lYat7ZLaVVcIuj82J8kIro4V6kDe0AUDFboUCwcg=
+github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY=
 github.com/apex/log v1.1.4/go.mod h1:AlpoD9aScyQfJDVHmLMEcx4oU6LqzkWp4Mg9GdAcEvQ=
 github.com/apex/log v1.3.0 h1:1fyfbPvUwD10nMoh3hY6MXzvZShJQn9/ck7ATgAt5pA=
 github.com/apex/log v1.3.0/go.mod h1:jd8Vpsr46WAe3EZSQ/IUMs2qQD/GOycT5rPWCO1yGcs=
@@ -166,8 +170,8 @@ github.com/bketelsen/crypt v0.0.3-0.20200106085610-5cbc8cc4026c/go.mod h1:MKsuJm
 github.com/blakesmith/ar v0.0.0-20190502131153-809d4375e1fb h1:m935MPodAbYS46DG4pJSv7WO+VECIWUQ7OJYSoTrMh4=
 github.com/blakesmith/ar v0.0.0-20190502131153-809d4375e1fb/go.mod h1:PkYb9DJNAwrSvRx5DYA+gUcOIgTGVMNkfSCbZM8cWpI=
 github.com/blang/semver v3.5.0+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk=
-github.com/bmatcuk/doublestar v1.3.3 h1:pVP1d49CcQQaNOl+PI6sPybIrIOD/6sux31PFdmhTH0=
-github.com/bmatcuk/doublestar v1.3.3/go.mod h1:wiQtGV+rzVYxB7WIlirSN++5HPtPlXEo9MEoZQC/PmE=
+github.com/bmatcuk/doublestar/v2 v2.0.4 h1:6I6oUiT/sU27eE2OFcWqBhL1SwjyvQuOssxT4a1yidI=
+github.com/bmatcuk/doublestar/v2 v2.0.4/go.mod h1:QMmcs3H2AUQICWhfzLXz+IYln8lRQmTZRptLie8RgRw=
 github.com/bombsimon/wsl/v2 v2.0.0/go.mod h1:mf25kr/SqFEPhhcxW1+7pxzGlW+hIl/hYTKY95VwV8U=
 github.com/bombsimon/wsl/v2 v2.2.0/go.mod h1:Azh8c3XGEJl9LyX0/sFC+CKMc7Ssgua0g+6abzXN4Pg=
 github.com/bombsimon/wsl/v3 v3.0.0/go.mod h1:st10JtZYLE4D5sC7b8xV4zTKZwAQjCH/Hy2Pm1FNZIc=
@@ -265,6 +269,7 @@ github.com/facebookincubator/nvdtools v0.1.4/go.mod h1:0/FIVnSEl9YHXLq3tKBPpKaI0
 github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4=
 github.com/fatih/color v1.9.0 h1:8xPHl4/q1VyqGIPif1F+1V3Y3lSmrq01EabUW3CoW5s=
 github.com/fatih/color v1.9.0/go.mod h1:eQcE1qtQxscV5RaZvpXrrb8Drkc3/DdQ+uUYCNjL+zU=
+github.com/fatih/set v0.2.1 h1:nn2CaJyknWE/6txyUDGwysr3G5QC6xWB/PtVjPBbeaA=
 github.com/fatih/set v0.2.1/go.mod h1:+RKtMCH+favT2+3YecHGxcc0b4KyVWA1QWWJUs4E0CI=
 github.com/fortytw2/leaktest v1.2.0/go.mod h1:jDsjWgpAGjm2CA7WthBh/CdZYEPF31XHquHwclZch5g=
 github.com/fortytw2/leaktest v1.3.0/go.mod h1:jDsjWgpAGjm2CA7WthBh/CdZYEPF31XHquHwclZch5g=
@@ -691,6 +696,8 @@ github.com/pkg/errors v0.8.1-0.20171018195549-f15c970de5b7/go.mod h1:bwawxfHBFNV
 github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
 github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pkg/profile v1.5.0 h1:042Buzk+NhDI+DeSAA62RwJL8VAuZUMQZUjCsRz1Mug=
+github.com/pkg/profile v1.5.0/go.mod h1:qBsxPvzyUincmltOk6iyRVxHYg4adc0OFOv72ZdLa18=
 github.com/pmezard/go-difflib v0.0.0-20151028094244-d8ed2627bdf0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
@@ -815,6 +822,7 @@ github.com/tetafro/godot v0.4.2/go.mod h1:/7NLHhv08H1+8DNj0MElpAACw1ajsCuf3TKNQx
 github.com/timakin/bodyclose v0.0.0-20190930140734-f7f2e9bca95e/go.mod h1:Qimiffbc6q9tBWlVV6x0P9sat/ao1xEkREYPPj9hphk=
 github.com/timakin/bodyclose v0.0.0-20200424151742-cb6215831a94 h1:ig99OeTyDwQWhPe2iw9lwfQVF1KB3Q4fpP3X7/2VBG8=
 github.com/timakin/bodyclose v0.0.0-20200424151742-cb6215831a94/go.mod h1:Qimiffbc6q9tBWlVV6x0P9sat/ao1xEkREYPPj9hphk=
+github.com/tj/assert v0.0.0-20171129193455-018094318fb0 h1:Rw8kxzWo1mr6FSaYXjQELRe88y2KdfynXdnK72rdjtA=
 github.com/tj/assert v0.0.0-20171129193455-018094318fb0/go.mod h1:mZ9/Rh9oLWpLLDRpvE+3b7gP/C2YyLFYxNmcLnPTMe0=
 github.com/tj/go-elastic v0.0.0-20171221160941-36157cbbebc2/go.mod h1:WjeM0Oo1eNAjXGDx2yma7uG2XoyRZTq1uv3M/o7imD0=
 github.com/tj/go-kinesis v0.0.0-20171128231115-08b17f58cb1b/go.mod h1:/yhzCV0xPfx6jb1bBgRFjl5lytqVqZXEaeqWP8lTEao=

internal/anchore/client.go

@@ -0,0 +1,58 @@
+package anchore
+
+import (
+	"context"
+	"fmt"
+
+	"github.com/anchore/client-go/pkg/external"
+	"github.com/anchore/syft/internal"
+	"github.com/anchore/syft/internal/version"
+)
+
+type Configuration struct {
+	Hostname  string
+	Username  string
+	Password  string
+	UserAgent string
+	Scheme    string
+}
+
+type Client struct {
+	config Configuration
+	client *external.APIClient
+}
+
+func NewClient(cfg Configuration) (*Client, error) {
+	if cfg.UserAgent == "" {
+		versionInfo := version.FromBuild()
+		// format: product / product-version comment
+		cfg.UserAgent = fmt.Sprintf("%s / %s %s", internal.ApplicationName, versionInfo.Version, versionInfo.Platform)
+	}
+
+	if cfg.Scheme == "" {
+		cfg.Scheme = "https"
+	}
+
+	return &Client{
+		config: cfg,
+		client: external.NewAPIClient(&external.Configuration{
+			Host:      cfg.Hostname,
+			UserAgent: cfg.UserAgent,
+			Scheme:    cfg.Scheme,
+		}),
+	}, nil
+}
+
+func (c *Client) newRequestContext(parentContext context.Context) context.Context {
+	if parentContext == nil {
+		parentContext = context.Background()
+	}
+	return context.WithValue(
+		parentContext,
+		external.ContextBasicAuth,
+		external.BasicAuth{
+			UserName: c.config.Username,
+			Password: c.config.Password,
+		},
+	)
+}

internal/anchore/import.go

@@ -0,0 +1,143 @@
+package anchore
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"time"
+
+	"github.com/antihax/optional"
+
+	"github.com/anchore/client-go/pkg/external"
+	"github.com/anchore/stereoscope/pkg/image"
+	"github.com/anchore/syft/internal/bus"
+	"github.com/anchore/syft/syft/distro"
+	"github.com/anchore/syft/syft/event"
+	"github.com/anchore/syft/syft/pkg"
+	"github.com/anchore/syft/syft/source"
+	"github.com/wagoodman/go-partybus"
+	"github.com/wagoodman/go-progress"
+)
+
+type ImportConfig struct {
+	ImageMetadata           image.Metadata
+	SourceMetadata          source.Metadata
+	Catalog                 *pkg.Catalog
+	Distro                  *distro.Distro
+	Dockerfile              []byte
+	OverwriteExistingUpload bool
+}
+
+func importProgress(source string) (*progress.Stage, *progress.Manual) {
+	stage := &progress.Stage{}
+	prog := &progress.Manual{
+		// this is the number of stages to expect; start + individual endpoints + stop
+		Total: 6,
+	}
+	bus.Publish(partybus.Event{
+		Type:   event.ImportStarted,
+		Source: source,
+		Value: progress.StagedProgressable(&struct {
+			progress.Stager
+			progress.Progressable
+		}{
+			Stager:       progress.Stager(stage),
+			Progressable: prog,
+		}),
+	})
+
+	return stage, prog
+}
+
+// nolint:funlen
+func (c *Client) Import(ctx context.Context, cfg ImportConfig) error {
+	stage, prog := importProgress(c.config.Hostname)
+
+	ctxWithTimeout, cancel := context.WithTimeout(ctx, time.Second*30)
+	defer cancel()
+
+	authedCtx := c.newRequestContext(ctxWithTimeout)
+
+	stage.Current = "starting session"
+	startOperation, _, err := c.client.ImportsApi.CreateOperation(authedCtx)
+	if err != nil {
+		var detail = "no details given"
+		var openAPIErr external.GenericOpenAPIError
+		if errors.As(err, &openAPIErr) {
+			detail = string(openAPIErr.Body())
+		}
+		return fmt.Errorf("unable to start import session: %w: %s", err, detail)
+	}
+	prog.N++
+	sessionID := startOperation.Uuid
+
+	packageDigest, err := importPackageSBOM(authedCtx, c.client.ImportsApi, sessionID, cfg.SourceMetadata, cfg.Catalog, cfg.Distro, stage)
+	if err != nil {
+		return fmt.Errorf("failed to import Package SBOM: %w", err)
+	}
+	prog.N++
+
+	manifestDigest, err := importManifest(authedCtx, c.client.ImportsApi, sessionID, cfg.ImageMetadata.RawManifest, stage)
+	if err != nil {
+		return fmt.Errorf("failed to import Manifest: %w", err)
+	}
+	prog.N++
+
+	configDigest, err := importConfig(authedCtx, c.client.ImportsApi, sessionID, cfg.ImageMetadata.RawConfig, stage)
+	if err != nil {
+		return fmt.Errorf("failed to import Config: %w", err)
+	}
+	prog.N++
+
+	dockerfileDigest, err := importDockerfile(authedCtx, c.client.ImportsApi, sessionID, cfg.Dockerfile, stage)
+	if err != nil {
+		return fmt.Errorf("failed to import Dockerfile: %w", err)
+	}
+	prog.N++
+
+	stage.Current = "finalizing"
+	imageModel := addImageModel(cfg.ImageMetadata, packageDigest, manifestDigest, dockerfileDigest, configDigest, sessionID)
+	opts := external.AddImageOpts{
+		Force: optional.NewBool(cfg.OverwriteExistingUpload),
+	}
+
+	_, _, err = c.client.ImagesApi.AddImage(authedCtx, imageModel, &opts)
+	if err != nil {
+		var detail = "no details given"
+		var openAPIErr external.GenericOpenAPIError
+		if errors.As(err, &openAPIErr) {
+			detail = string(openAPIErr.Body())
+		}
+		return fmt.Errorf("unable to complete import session=%q: %w: %s", sessionID, err, detail)
+	}
+	prog.N++
+
+	stage.Current = ""
+	prog.SetCompleted()
+
+	return nil
+}
+
+func addImageModel(imageMetadata image.Metadata, packageDigest, manifestDigest, dockerfileDigest, configDigest, sessionID string) external.ImageAnalysisRequest {
+	var tags = make([]string, len(imageMetadata.Tags))
+	for i, t := range imageMetadata.Tags {
+		tags[i] = t.String()
+	}
+
+	return external.ImageAnalysisRequest{
+		Source: external.ImageSource{
+			Import: &external.ImageImportManifest{
+				Contents: external.ImportContentDigests{
+					Packages:    packageDigest,
+					Manifest:    manifestDigest,
+					Dockerfile:  dockerfileDigest,
+					ImageConfig: configDigest,
+				},
+				Tags:          tags,
+				Digest:        imageMetadata.ManifestDigest,
+				LocalImageId:  imageMetadata.ID,
+				OperationUuid: sessionID,
+			},
+		},
+	}
+}

internal/anchore/import_config.go

@@ -0,0 +1,50 @@
+// nolint:dupl
+package anchore
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"net/http"
+
+	"github.com/wagoodman/go-progress"
+
+	"github.com/anchore/client-go/pkg/external"
+	"github.com/anchore/syft/internal/log"
+)
+
+type configImportAPI interface {
+	ImportImageConfig(ctx context.Context, sessionID string, contents interface{}) (external.ImageImportContentResponse, *http.Response, error)
+}
+
+func importConfig(ctx context.Context, api configImportAPI, sessionID string, config []byte, stage *progress.Stage) (string, error) {
+	if len(config) > 0 {
+		log.Debug("importing image config")
+		stage.Current = "image config"
+
+		// API requires an object, but we do not verify the shape of this object locally
+		var sender map[string]interface{}
+		if err := json.Unmarshal(config, &sender); err != nil {
+			return "", err
+		}
+
+		response, httpResponse, err := api.ImportImageConfig(ctx, sessionID, sender)
+		if err != nil {
+			var openAPIErr external.GenericOpenAPIError
+			if errors.As(err, &openAPIErr) {
+				log.Errorf("api response: %+v", string(openAPIErr.Body()))
+			}
+			return "", fmt.Errorf("unable to import Config: %w", err)
+		}
+
+		defer httpResponse.Body.Close()
+
+		if httpResponse.StatusCode != 200 {
+			return "", fmt.Errorf("unable to import Config: %s", httpResponse.Status)
+		}
+
+		return response.Digest, nil
+	}
+	return "", nil
+}

internal/anchore/import_config_test.go

@@ -0,0 +1,129 @@
+package anchore
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"net/http"
+	"strings"
+	"testing"
+
+	"github.com/wagoodman/go-progress"
+
+	"github.com/anchore/client-go/pkg/external"
+	"github.com/docker/docker/pkg/ioutils"
+	"github.com/go-test/deep"
+)
+
+type mockConfigImportAPI struct {
+	sessionID      string
+	model          interface{}
+	httpResponse   *http.Response
+	err            error
+	ctx            context.Context
+	responseDigest string
+	wasCalled      bool
+}
+
+func (m *mockConfigImportAPI) ImportImageConfig(ctx context.Context, sessionID string, contents interface{}) (external.ImageImportContentResponse, *http.Response, error) {
+	m.wasCalled = true
+	m.model = contents
+	m.sessionID = sessionID
+	m.ctx = ctx
+	if m.httpResponse == nil {
+		m.httpResponse = &http.Response{}
+	}
+	m.httpResponse.Body = ioutils.NewReadCloserWrapper(strings.NewReader(""), func() error { return nil })
+	return external.ImageImportContentResponse{Digest: m.responseDigest}, m.httpResponse, m.err
+}
+
+func TestConfigImport(t *testing.T) {
+
+	sessionID := "my-session"
+
+	tests := []struct {
+		name            string
+		manifestJSONStr string
+		api             *mockConfigImportAPI
+		expectsError    bool
+		expectsCall     bool
+	}{
+
+		{
+			name:            "Go case: import works",
+			manifestJSONStr: `{ "key": "the-manifest-contents!" }`,
+			api: &mockConfigImportAPI{
+				httpResponse:   &http.Response{StatusCode: 200},
+				responseDigest: "digest!",
+			},
+			expectsCall: true,
+		},
+		{
+			name:            "No manifest provided",
+			manifestJSONStr: "",
+			api:             &mockConfigImportAPI{},
+			expectsCall:     false,
+		},
+		{
+			name:            "API returns an error",
+			manifestJSONStr: `{ "key": "the-manifest-contents!" }`,
+			api: &mockConfigImportAPI{
+				err: fmt.Errorf("api error, something went wrong"),
+			},
+			expectsError: true,
+			expectsCall:  true,
+		},
+		{
+			name:            "API HTTP-level error",
+			manifestJSONStr: `{ "key": "the-manifest-contents!" }`,
+			api: &mockConfigImportAPI{
+				httpResponse: &http.Response{StatusCode: 404},
+			},
+			expectsError: true,
+			expectsCall:  true,
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+
+			digest, err := importConfig(context.TODO(), test.api, sessionID, []byte(test.manifestJSONStr), &progress.Stage{})
+
+			// validate error handling
+			if err != nil && !test.expectsError {
+				t.Fatalf("did not expect an error, but got: %+v", err)
+			} else if err == nil && test.expectsError {
+				t.Fatalf("did expect an error, but got none")
+			}
+
+			if !test.api.wasCalled && test.expectsCall {
+				t.Fatalf("was not called!")
+			} else if test.api.wasCalled && !test.expectsCall {
+				t.Fatalf("should not have been called")
+			}
+
+			if !test.expectsCall {
+				return
+			}
+
+			if digest != test.api.responseDigest {
+				t.Errorf("unexpected content digest: %q != %q", digest, test.api.responseDigest)
+			}
+
+			// validating that the mock got the right parameters
+			if test.api.sessionID != sessionID {
+				t.Errorf("different session ID: %s != %s", test.api.sessionID, sessionID)
+			}
+
+			var expected map[string]interface{}
+			if err := json.Unmarshal([]byte(test.manifestJSONStr), &expected); err != nil {
+				t.Fatalf("could not unmarshal expected results")
+			}
+
+			for _, d := range deep.Equal(test.api.model, expected) {
+				t.Errorf("model difference: %s", d)
+			}
+
+		})
+	}
+}

internal/anchore/import_dockerfile.go

@@ -0,0 +1,43 @@
+package anchore
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"net/http"
+
+	"github.com/wagoodman/go-progress"
+
+	"github.com/anchore/syft/internal/log"
+
+	"github.com/anchore/client-go/pkg/external"
+)
+
+type dockerfileImportAPI interface {
+	ImportImageDockerfile(ctx context.Context, sessionID string, contents string) (external.ImageImportContentResponse, *http.Response, error)
+}
+
+func importDockerfile(ctx context.Context, api dockerfileImportAPI, sessionID string, dockerfile []byte, stage *progress.Stage) (string, error) {
+	if len(dockerfile) > 0 {
+		log.Debug("importing dockerfile")
+		stage.Current = "dockerfile"
+
+		response, httpResponse, err := api.ImportImageDockerfile(ctx, sessionID, string(dockerfile))
+		if err != nil {
+			var openAPIErr external.GenericOpenAPIError
+			if errors.As(err, &openAPIErr) {
+				log.Errorf("api response: %+v", string(openAPIErr.Body()))
+			}
+			return "", fmt.Errorf("unable to import Dockerfile: %w", err)
+		}
+
+		defer httpResponse.Body.Close()
+
+		if httpResponse.StatusCode != 200 {
+			return "", fmt.Errorf("unable to import Dockerfile: %s", httpResponse.Status)
+		}
+
+		return response.Digest, nil
+	}
+	return "", nil
+}

internal/anchore/import_dockerfile_test.go

@@ -0,0 +1,124 @@
+package anchore
+
+import (
+	"context"
+	"fmt"
+	"net/http"
+	"strings"
+	"testing"
+
+	"github.com/wagoodman/go-progress"
+
+	"github.com/docker/docker/pkg/ioutils"
+
+	"github.com/anchore/client-go/pkg/external"
+	"github.com/go-test/deep"
+)
+
+type mockDockerfileImportAPI struct {
+	sessionID      string
+	model          string
+	httpResponse   *http.Response
+	err            error
+	ctx            context.Context
+	responseDigest string
+	wasCalled      bool
+}
+
+func (m *mockDockerfileImportAPI) ImportImageDockerfile(ctx context.Context, sessionID string, contents string) (external.ImageImportContentResponse, *http.Response, error) {
+	m.wasCalled = true
+	m.model = contents
+	m.sessionID = sessionID
+	m.ctx = ctx
+	if m.httpResponse == nil {
+		m.httpResponse = &http.Response{}
+	}
+	m.httpResponse.Body = ioutils.NewReadCloserWrapper(strings.NewReader(""), func() error { return nil })
+	return external.ImageImportContentResponse{Digest: m.responseDigest}, m.httpResponse, m.err
+}
+
+func TestDockerfileImport(t *testing.T) {
+
+	sessionID := "my-session"
+
+	tests := []struct {
+		name         string
+		dockerfile   string
+		api          *mockDockerfileImportAPI
+		expectsError bool
+		expectsCall  bool
+	}{
+
+		{
+			name:       "Go case: import works",
+			dockerfile: "the-manifest-contents!",
+			api: &mockDockerfileImportAPI{
+				httpResponse:   &http.Response{StatusCode: 200},
+				responseDigest: "digest!",
+			},
+			expectsCall: true,
+		},
+		{
+			name:        "No manifest provided",
+			dockerfile:  "",
+			api:         &mockDockerfileImportAPI{},
+			expectsCall: false,
+		},
+		{
+			name:       "API returns an error",
+			dockerfile: "the-manifest-contents!",
+			api: &mockDockerfileImportAPI{
+				err: fmt.Errorf("api error, something went wrong"),
+			},
+			expectsError: true,
+			expectsCall:  true,
+		},
+		{
+			name:       "API HTTP-level error",
+			dockerfile: "the-manifest-contents!",
+			api: &mockDockerfileImportAPI{
+				httpResponse: &http.Response{StatusCode: 404},
+			},
+			expectsError: true,
+			expectsCall:  true,
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+
+			digest, err := importDockerfile(context.TODO(), test.api, sessionID, []byte(test.dockerfile), &progress.Stage{})
+
+			// validate error handling
+			if err != nil && !test.expectsError {
+				t.Fatalf("did not expect an error, but got: %+v", err)
+			} else if err == nil && test.expectsError {
+				t.Fatalf("did expect an error, but got none")
+			}
+
+			if !test.api.wasCalled && test.expectsCall {
+				t.Fatalf("was not called!")
+			} else if test.api.wasCalled && !test.expectsCall {
+				t.Fatalf("should not have been called")
+			}
+
+			if !test.expectsCall {
+				return
+			}
+
+			if digest != test.api.responseDigest {
+				t.Errorf("unexpected content digest: %q != %q", digest, test.api.responseDigest)
+			}
+
+			// validating that the mock got the right parameters
+			if test.api.sessionID != sessionID {
+				t.Errorf("different session ID: %s != %s", test.api.sessionID, sessionID)
+			}
+
+			for _, d := range deep.Equal(test.api.model, test.dockerfile) {
+				t.Errorf("model difference: %s", d)
+			}
+
+		})
+	}
+}

internal/anchore/import_manifest.go

@@ -0,0 +1,50 @@
+// nolint: dupl
+package anchore
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"net/http"
+
+	"github.com/wagoodman/go-progress"
+
+	"github.com/anchore/client-go/pkg/external"
+	"github.com/anchore/syft/internal/log"
+)
+
+type manifestImportAPI interface {
+	ImportImageManifest(ctx context.Context, sessionID string, contents interface{}) (external.ImageImportContentResponse, *http.Response, error)
+}
+
+func importManifest(ctx context.Context, api manifestImportAPI, sessionID string, manifest []byte, stage *progress.Stage) (string, error) {
+	if len(manifest) > 0 {
+		log.Debug("importing image manifest")
+		stage.Current = "image manifest"
+
+		// API requires an object, but we do not verify the shape of this object locally
+		var sender map[string]interface{}
+		if err := json.Unmarshal(manifest, &sender); err != nil {
+			return "", err
+		}
+
+		response, httpResponse, err := api.ImportImageManifest(ctx, sessionID, sender)
+		if err != nil {
+			var openAPIErr external.GenericOpenAPIError
+			if errors.As(err, &openAPIErr) {
+				log.Errorf("api response: %+v", string(openAPIErr.Body()))
+			}
+			return "", fmt.Errorf("unable to import Manifest: %w", err)
+		}
+
+		defer httpResponse.Body.Close()
+
+		if httpResponse.StatusCode != 200 {
+			return "", fmt.Errorf("unable to import Manifest: %s", httpResponse.Status)
+		}
+
+		return response.Digest, nil
+	}
+	return "", nil
+}

internal/anchore/import_manifest_test.go

@@ -0,0 +1,129 @@
+package anchore
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"net/http"
+	"strings"
+	"testing"
+
+	"github.com/wagoodman/go-progress"
+
+	"github.com/anchore/client-go/pkg/external"
+	"github.com/docker/docker/pkg/ioutils"
+	"github.com/go-test/deep"
+)
+
+type mockManifestImportAPI struct {
+	sessionID      string
+	model          interface{}
+	httpResponse   *http.Response
+	err            error
+	ctx            context.Context
+	responseDigest string
+	wasCalled      bool
+}
+
+func (m *mockManifestImportAPI) ImportImageManifest(ctx context.Context, sessionID string, contents interface{}) (external.ImageImportContentResponse, *http.Response, error) {
+	m.wasCalled = true
+	m.model = contents
+	m.sessionID = sessionID
+	m.ctx = ctx
+	if m.httpResponse == nil {
+		m.httpResponse = &http.Response{}
+	}
+	m.httpResponse.Body = ioutils.NewReadCloserWrapper(strings.NewReader(""), func() error { return nil })
+	return external.ImageImportContentResponse{Digest: m.responseDigest}, m.httpResponse, m.err
+}
+
+func TestManifestImport(t *testing.T) {
+
+	sessionID := "my-session"
+
+	tests := []struct {
+		name         string
+		manifest     string
+		api          *mockManifestImportAPI
+		expectsError bool
+		expectsCall  bool
+	}{
+
+		{
+			name:     "Go case: import works",
+			manifest: `{ "key": "the-config-contents!" }`,
+			api: &mockManifestImportAPI{
+				httpResponse:   &http.Response{StatusCode: 200},
+				responseDigest: "digest!",
+			},
+			expectsCall: true,
+		},
+		{
+			name:        "No manifest provided",
+			manifest:    "",
+			api:         &mockManifestImportAPI{},
+			expectsCall: false,
+		},
+		{
+			name:     "API returns an error",
+			manifest: `{ "key": "the-config-contents!" }`,
+			api: &mockManifestImportAPI{
+				err: fmt.Errorf("api error, something went wrong"),
+			},
+			expectsError: true,
+			expectsCall:  true,
+		},
+		{
+			name:     "API HTTP-level error",
+			manifest: `{ "key": "the-config-contents!" }`,
+			api: &mockManifestImportAPI{
+				httpResponse: &http.Response{StatusCode: 404},
+			},
+			expectsError: true,
+			expectsCall:  true,
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+
+			digest, err := importManifest(context.TODO(), test.api, sessionID, []byte(test.manifest), &progress.Stage{})
+
+			// validate error handling
+			if err != nil && !test.expectsError {
+				t.Fatalf("did not expect an error, but got: %+v", err)
+			} else if err == nil && test.expectsError {
+				t.Fatalf("did expect an error, but got none")
+			}
+
+			if !test.api.wasCalled && test.expectsCall {
+				t.Fatalf("was not called!")
+			} else if test.api.wasCalled && !test.expectsCall {
+				t.Fatalf("should not have been called")
+			}
+
+			if !test.expectsCall {
+				return
+			}
+
+			if digest != test.api.responseDigest {
+				t.Errorf("unexpected content digest: %q != %q", digest, test.api.responseDigest)
+			}
+
+			// validating that the mock got the right parameters
+			if test.api.sessionID != sessionID {
+				t.Errorf("different session ID: %s != %s", test.api.sessionID, sessionID)
+			}
+
+			var expected map[string]interface{}
+			if err := json.Unmarshal([]byte(test.manifest), &expected); err != nil {
+				t.Fatalf("could not unmarshal expected results")
+			}
+
+			for _, d := range deep.Equal(test.api.model, expected) {
+				t.Errorf("model difference: %s", d)
+			}
+
+		})
+	}
+}

internal/anchore/import_package_sbom.go

@@ -0,0 +1,69 @@
+package anchore
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"net/http"
+
+	"github.com/wagoodman/go-progress"
+
+	jsonPresenter "github.com/anchore/syft/syft/presenter/json"
+
+	"github.com/anchore/syft/syft/distro"
+	"github.com/anchore/syft/syft/source"
+
+	"github.com/anchore/client-go/pkg/external"
+	"github.com/anchore/syft/internal/log"
+	"github.com/anchore/syft/syft/pkg"
+)
+
+type packageSBOMImportAPI interface {
+	ImportImagePackages(context.Context, string, external.ImagePackageManifest) (external.ImageImportContentResponse, *http.Response, error)
+}
+
+func packageSbomModel(s source.Metadata, catalog *pkg.Catalog, d *distro.Distro) (*external.ImagePackageManifest, error) {
+	var buf bytes.Buffer
+	pres := jsonPresenter.NewPresenter(catalog, s, d)
+	err := pres.Present(&buf)
+	if err != nil {
+		return nil, fmt.Errorf("unable to serialize results: %w", err)
+	}
+
+	// the model is 1:1 the JSON output of today. As the schema changes, this will need to be converted into individual mappings.
+	var model external.ImagePackageManifest
+	if err = json.Unmarshal(buf.Bytes(), &model); err != nil {
+		return nil, fmt.Errorf("unable to convert JSON presenter output to import model: %w", err)
+	}
+
+	return &model, nil
+}
+
+func importPackageSBOM(ctx context.Context, api packageSBOMImportAPI, sessionID string, s source.Metadata, catalog *pkg.Catalog, d *distro.Distro, stage *progress.Stage) (string, error) {
+	log.Debug("importing package SBOM")
+	stage.Current = "package SBOM"
+
+	model, err := packageSbomModel(s, catalog, d)
+	if err != nil {
+		return "", fmt.Errorf("unable to create PackageSBOM model: %w", err)
+	}
+
+	response, httpResponse, err := api.ImportImagePackages(ctx, sessionID, *model)
+	if err != nil {
+		var openAPIErr external.GenericOpenAPIError
+		if errors.As(err, &openAPIErr) {
+			log.Errorf("api response: %+v", string(openAPIErr.Body()))
+		}
+		return "", fmt.Errorf("unable to import PackageSBOM: %w", err)
+	}
+
+	defer httpResponse.Body.Close()
+
+	if httpResponse.StatusCode != 200 {
+		return "", fmt.Errorf("unable to import PackageSBOM: %s", httpResponse.Status)
+	}
+
+	return response.Digest, nil
+}

internal/anchore/import_package_sbom_test.go

@@ -0,0 +1,254 @@
+package anchore
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"fmt"
+	"net/http"
+	"strings"
+	"testing"
+
+	"github.com/wagoodman/go-progress"
+
+	jsonPresenter "github.com/anchore/syft/syft/presenter/json"
+
+	"github.com/anchore/syft/syft/distro"
+
+	"github.com/docker/docker/pkg/ioutils"
+
+	"github.com/anchore/client-go/pkg/external"
+	"github.com/anchore/syft/syft/pkg"
+	"github.com/anchore/syft/syft/source"
+	"github.com/go-test/deep"
+)
+
+func must(c pkg.CPE, e error) pkg.CPE {
+	if e != nil {
+		panic(e)
+	}
+	return c
+}
+
+// this test is tailored towards the assumption that the import doc shape and the syft json shape are the same.
+// TODO: replace this as the document shapes diverge.
+func TestPackageSbomToModel(t *testing.T) {
+
+	m := source.Metadata{
+		Scheme: source.ImageScheme,
+		ImageMetadata: source.ImageMetadata{
+			UserInput: "user-in",
+			Scope:     "scope!",
+			Layers: []source.LayerMetadata{
+				{
+					MediaType: "layer-metadata-type!",
+					Digest:    "layer-digest",
+					Size:      20,
+				},
+			},
+			Size:           10,
+			ManifestDigest: "sha256:digest!",
+			MediaType:      "mediatype!",
+			Tags:           nil,
+		},
+	}
+
+	d, _ := distro.NewDistro(distro.CentOS, "8.0", "")
+
+	p := pkg.Package{
+		Name:    "name",
+		Version: "version",
+		FoundBy: "foundBy",
+		Locations: []source.Location{
+			{
+				RealPath:     "path",
+				FileSystemID: "layerID",
+			},
+		},
+		Licenses: []string{"license"},
+		Language: pkg.Python,
+		Type:     pkg.PythonPkg,
+		CPEs: []pkg.CPE{
+			must(pkg.NewCPE("cpe:2.3:*:some:package:1:*:*:*:*:*:*:*")),
+		},
+		PURL: "purl",
+	}
+
+	c := pkg.NewCatalog(p)
+
+	model, err := packageSbomModel(m, c, &d)
+	if err != nil {
+		t.Fatalf("unable to generate model from source material: %+v", err)
+	}
+
+	var modelJSON []byte
+
+	modelJSON, err = json.Marshal(&model)
+	if err != nil {
+		t.Fatalf("unable to marshal model: %+v", err)
+	}
+
+	var buf bytes.Buffer
+	pres := jsonPresenter.NewPresenter(c, m, &d)
+	if err := pres.Present(&buf); err != nil {
+		t.Fatalf("unable to get expected json: %+v", err)
+	}
+
+	// unmarshal expected result
+	var expectedDoc jsonPresenter.Document
+	if err := json.Unmarshal(buf.Bytes(), &expectedDoc); err != nil {
+		t.Fatalf("unable to parse json doc: %+v", err)
+	}
+
+	// unmarshal actual result
+	var actualDoc jsonPresenter.Document
+	if err := json.Unmarshal(modelJSON, &actualDoc); err != nil {
+		t.Fatalf("unable to parse json doc: %+v", err)
+	}
+
+	for _, d := range deep.Equal(actualDoc, expectedDoc) {
+		t.Errorf("diff: %+v", d)
+	}
+}
+
+type mockPackageSBOMImportAPI struct {
+	sessionID      string
+	model          external.ImagePackageManifest
+	httpResponse   *http.Response
+	err            error
+	ctx            context.Context
+	responseDigest string
+}
+
+func (m *mockPackageSBOMImportAPI) ImportImagePackages(ctx context.Context, sessionID string, model external.ImagePackageManifest) (external.ImageImportContentResponse, *http.Response, error) {
+	m.model = model
+	m.sessionID = sessionID
+	m.ctx = ctx
+	if m.httpResponse == nil {
+		m.httpResponse = &http.Response{}
+	}
+	m.httpResponse.Body = ioutils.NewReadCloserWrapper(strings.NewReader(""), func() error { return nil })
+	return external.ImageImportContentResponse{Digest: m.responseDigest}, m.httpResponse, m.err
+}
+
+func TestPackageSbomImport(t *testing.T) {
+
+	catalog := pkg.NewCatalog(pkg.Package{
+		Name:    "name",
+		Version: "version",
+		FoundBy: "foundBy",
+		Locations: []source.Location{
+			{
+				RealPath:     "path",
+				FileSystemID: "layerID",
+			},
+		},
+		Licenses: []string{"license"},
+		Language: pkg.Python,
+		Type:     pkg.PythonPkg,
+		CPEs: []pkg.CPE{
+			must(pkg.NewCPE("cpe:2.3:*:some:package:1:*:*:*:*:*:*:*")),
+		},
+		PURL:         "purl",
+		MetadataType: pkg.PythonPackageMetadataType,
+		Metadata: pkg.PythonPackageMetadata{
+			Name:        "p-name",
+			Version:     "p-version",
+			License:     "p-license",
+			Author:      "p-author",
+			AuthorEmail: "p-email",
+			Platform:    "p-platform",
+			Files: []pkg.PythonFileRecord{
+				{
+					Path: "p-path",
+					Digest: &pkg.PythonFileDigest{
+						Algorithm: "p-alg",
+						Value:     "p-digest",
+					},
+					Size: "p-size",
+				},
+			},
+			SitePackagesRootPath: "p-site-packages-root",
+			TopLevelPackages:     []string{"top-level"},
+		},
+	})
+
+	m := source.Metadata{
+		Scheme: "a-schema",
+		ImageMetadata: source.ImageMetadata{
+			UserInput:      "user-in",
+			Scope:          "scope!",
+			Layers:         nil,
+			Size:           10,
+			ManifestDigest: "sha256:digest!",
+			MediaType:      "mediatype!",
+			Tags:           nil,
+		},
+	}
+
+	d, _ := distro.NewDistro(distro.CentOS, "8.0", "")
+
+	theModel, err := packageSbomModel(m, catalog, &d)
+	if err != nil {
+		t.Fatalf("could not get sbom model: %+v", err)
+	}
+
+	sessionID := "my-session"
+
+	tests := []struct {
+		name         string
+		api          *mockPackageSBOMImportAPI
+		expectsError bool
+	}{
+
+		{
+			name: "Go case: import works",
+			api: &mockPackageSBOMImportAPI{
+				httpResponse:   &http.Response{StatusCode: 200},
+				responseDigest: "digest!",
+			},
+		},
+		{
+			name: "API returns an error",
+			api: &mockPackageSBOMImportAPI{
+				err: fmt.Errorf("API error, something went wrong."),
+			},
+			expectsError: true,
+		},
+		{
+			name: "API HTTP-level error",
+			api: &mockPackageSBOMImportAPI{
+				httpResponse: &http.Response{StatusCode: 404},
+			},
+			expectsError: true,
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+
+			digest, err := importPackageSBOM(context.TODO(), test.api, sessionID, m, catalog, &d, &progress.Stage{})
+
+			// validate error handling
+			if err != nil && !test.expectsError {
+				t.Fatalf("did not expect an error, but got: %+v", err)
+			} else if err == nil && test.expectsError {
+				t.Fatalf("did expect an error, but got none")
+			}
+
+			if digest != test.api.responseDigest {
+				t.Errorf("unexpected content digest: %q != %q", digest, test.api.responseDigest)
+			}
+
+			// validating that the mock got the right parameters (api.ImportImagePackages)
+			if test.api.sessionID != sessionID {
+				t.Errorf("different session ID: %s != %s", test.api.sessionID, sessionID)
+			}
+
+			for _, d := range deep.Equal(&test.api.model, theModel) {
+				t.Errorf("model difference: %s", d)
+			}
+
+		})
+	}
+}

internal/config/config.go

@@ -12,66 +12,78 @@ import (
 	"github.com/mitchellh/go-homedir"
 	"github.com/sirupsen/logrus"
 	"github.com/spf13/viper"
+	"gopkg.in/yaml.v2"
 )
 
-type CliOnlyOptions struct {
-	ConfigPath string
-	Verbosity  int
-}
-
+// Application is the main syft application configuration.
 type Application struct {
-	ConfigPath        string
-	PresenterOpt      presenter.Option
-	Output            string `mapstructure:"output"`
-	ScopeOpt          source.Scope
-	Scope             string  `mapstructure:"scope"`
-	Quiet             bool    `mapstructure:"quiet"`
-	Log               Logging `mapstructure:"log"`
-	CliOptions        CliOnlyOptions
-	CheckForAppUpdate bool `mapstructure:"check-for-app-update"`
+	ConfigPath        string           `yaml:",omitempty"`                   // the location where the application config was read from (either from -c or discovered while loading)
+	PresenterOpt      presenter.Option `yaml:"-"`                            // -o, the native Presenter.Option to use for report formatting
+	Output            string           `yaml:"output" mapstructure:"output"` // -o, the Presenter hint string to use for report formatting
+	ScopeOpt          source.Scope     `yaml:"-"`                            // -s, the native source.Scope option to use for how to catalog the container image
+	Scope             string           `yaml:"scope" mapstructure:"scope"`   // -s, the source.Scope string hint for how to catalog the container image
+	Quiet             bool             `yaml:"quiet" mapstructure:"quiet"`   // -q, indicates to not show any status output to stderr (ETUI or logging UI)
+	Log               logging          `yaml:"log"  mapstructure:"log"`      // all logging-related options
+	CliOptions        CliOnlyOptions   `yaml:"-"`                            // all options only available through the CLI (not via env vars or config)
+	Dev               Development      `mapstructure:"dev"`
+	CheckForAppUpdate bool             `yaml:"check-for-app-update" mapstructure:"check-for-app-update"` // whether to check for an application update on start up or not
+	Anchore           anchore          `yaml:"anchore" mapstructure:"anchore"`                           // options for interacting with Anchore Engine/Enterprise
 }
 
-type Logging struct {
-	Structured   bool `mapstructure:"structured"`
-	LevelOpt     logrus.Level
-	Level        string `mapstructure:"level"`
-	FileLocation string `mapstructure:"file"`
+// CliOnlyOptions are options that are in the application config in memory, but are only exposed via CLI switches (not from unmarshaling a config file)
+type CliOnlyOptions struct {
+	ConfigPath string // -c. where the read config is on disk
+	Verbosity  int    // -v or -vv , controlling which UI (ETUI vs logging) and what the log level should be
 }
 
-func setNonCliDefaultValues(v *viper.Viper) {
-	v.SetDefault("log.level", "")
-	v.SetDefault("log.file", "")
-	v.SetDefault("log.structured", false)
-	v.SetDefault("check-for-app-update", true)
+// logging contains all logging-related configuration options available to the user via the application config.
+type logging struct {
+	Structured   bool         `yaml:"structured" mapstructure:"structured"` // show all log entries as JSON formatted strings
+	LevelOpt     logrus.Level `yaml:"level"`                                // the native log level object used by the logger
+	Level        string       `yaml:"-" mapstructure:"level"`               // the log level string hint
+	FileLocation string       `yaml:"file" mapstructure:"file"`             // the file path to write logs to
 }
 
-func LoadConfigFromFile(v *viper.Viper, cliOpts *CliOnlyOptions) (*Application, error) {
+type anchore struct {
+	// upload options
+	UploadEnabled          bool   `yaml:"upload-enabled"  mapstructure:"upload-enabled"`                    // whether to upload results to Anchore Engine/Enterprise (defaults to "false" unless there is the presence of -h CLI option)
+	Host                   string `yaml:"host" mapstructure:"host"`                                         // -H , hostname of the engine/enterprise instance to upload to
+	Path                   string `yaml:"path" mapstructure:"path"`                                         // override the engine/enterprise API upload path
+	Username               string `yaml:"username" mapstructure:"username"`                                 // -u , username to authenticate upload
+	Password               string `yaml:"password" mapstructure:"password"`                                 // -p , password to authenticate upload
+	Dockerfile             string `yaml:"dockerfile" mapstructure:"dockerfile"`                             // -d , dockerfile to attach for upload
+	OverwriteExistingImage bool   `yaml:"overwrite-existing-image" mapstructure:"overwrite-existing-image"` // --overwrite-existing-image , if any of the SBOM components have already been uploaded this flag will ensure they are overwritten with the current upload
+}
+
+type Development struct {
+	ProfileCPU bool `mapstructure:"profile-cpu"`
+	ProfileMem bool `mapstructure:"profile-mem"`
+}
+
+// LoadApplicationConfig populates the given viper object with application configuration discovered on disk
+func LoadApplicationConfig(v *viper.Viper, cliOpts CliOnlyOptions, wasHostnameSet bool) (*Application, error) {
 	// the user may not have a config, and this is OK, we can use the default config + default cobra cli values instead
 	setNonCliDefaultValues(v)
-	if cliOpts != nil {
-		_ = readConfig(v, cliOpts.ConfigPath)
-	} else {
-		_ = readConfig(v, "")
-	}
+	_ = readConfig(v, cliOpts.ConfigPath)
 
 	config := &Application{
-		CliOptions: *cliOpts,
+		CliOptions: cliOpts,
 	}
-	err := v.Unmarshal(config)
-	if err != nil {
+
+	if err := v.Unmarshal(config); err != nil {
 		return nil, fmt.Errorf("unable to parse config: %w", err)
 	}
 	config.ConfigPath = v.ConfigFileUsed()
 
-	err = config.Build()
-	if err != nil {
+	if err := config.build(v, wasHostnameSet); err != nil {
 		return nil, fmt.Errorf("invalid config: %w", err)
 	}
 
 	return config, nil
 }
 
-func (cfg *Application) Build() error {
+// build inflates simple config values into syft native objects (or other complex objects) after the config is fully read in.
+func (cfg *Application) build(v *viper.Viper, wasHostnameSet bool) error {
 	// set the presenter
 	presenterOption := presenter.ParseOption(cfg.Output)
 	if presenterOption == presenter.UnknownPresenter {
@@ -115,10 +127,40 @@ func (cfg *Application) Build() error {
 			}
 		}
 	}
+	// check if upload should be done relative to the CLI and config behavior
+	if !v.IsSet("anchore.upload-enabled") && wasHostnameSet {
+		// we know the user didn't specify to upload in the config file and a --hostname option was provided (so set upload)
+		cfg.Anchore.UploadEnabled = true
+	}
+
+	if !cfg.Anchore.UploadEnabled && cfg.Anchore.Dockerfile != "" {
+		return fmt.Errorf("cannot provide dockerfile option without enabling upload")
+	}
 
 	return nil
 }
 
+func (cfg Application) String() string {
+	// redact sensitive information
+	if cfg.Anchore.Username != "" {
+		cfg.Anchore.Username = "********"
+	}
+
+	if cfg.Anchore.Password != "" {
+		cfg.Anchore.Password = "********"
+	}
+
+	// yaml is pretty human friendly (at least when compared to json)
+	appCfgStr, err := yaml.Marshal(&cfg)
+
+	if err != nil {
+		return err.Error()
+	}
+
+	return string(appCfgStr)
+}
+
+// readConfig attempts to read the given config path from disk or discover an alternate store location
 func readConfig(v *viper.Viper, configPath string) error {
 	v.AutomaticEnv()
 	v.SetEnvPrefix(internal.ApplicationName)
@@ -174,3 +216,13 @@ func readConfig(v *viper.Viper, configPath string) error {
 
 	return fmt.Errorf("application config not found")
 }
+
+// setNonCliDefaultValues ensures that there are sane defaults for values that do not have CLI equivalent options (where there would already be a default value)
+func setNonCliDefaultValues(v *viper.Viper) {
+	v.SetDefault("log.level", "")
+	v.SetDefault("log.file", "")
+	v.SetDefault("log.structured", false)
+	v.SetDefault("check-for-app-update", true)
+	v.SetDefault("dev.profile-cpu", false)
+	v.SetDefault("dev.profile-mem", false)
+}

internal/constants.go

@@ -1,4 +1,10 @@
 package internal
 
-// ApplicationName is the non-capitalized name of the application (do not change this)
-const ApplicationName = "syft"
+const (
+	// ApplicationName is the non-capitalized name of the application (do not change this)
+	ApplicationName = "syft"
+
+	// JSONSchemaVersion is the current schema version output by the JSON presenter
+	// This is roughly following the "SchemaVer" guidelines for versioning the JSON schema. Please see schema/json/README.md for details on how to increment.
+	JSONSchemaVersion = "1.0.1"
+)

schema/cyclonedx/bd.xsd

@@ -1,183 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
-CycloneDX BOM Descriptor Extension
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
--->
-<xs:schema xmlns:xs="http://www.w3.org/2001/XMLSchema"
-           xmlns:vc="http://www.w3.org/2007/XMLSchema-versioning"
-           xmlns:bom="http://cyclonedx.org/schema/bom/1.1"
-           xmlns:bd="http://cyclonedx.org/schema/ext/bom-descriptor/1.0"
-           elementFormDefault="qualified"
-           targetNamespace="http://cyclonedx.org/schema/ext/bom-descriptor/1.0"
-           vc:minVersion="1.0"
-           vc:maxVersion="1.1"
-           version="1.0">
-
-  <xs:annotation>
-    <xs:documentation>
-      <name>CycloneDX BOM Descriptor Extension</name>
-      <url>https://cyclonedx.org/ext/bom-descriptor</url>
-      <license uri="http://www.apache.org/licenses/LICENSE-2.0"
-               version="2.0">Apache License, Version 2.0</license>
-      <authors>
-        <author>Steve Springett</author>
-      </authors>
-    </xs:documentation>
-  </xs:annotation>
-
-  <xs:import namespace="http://cyclonedx.org/schema/bom/1.1" schemaLocation="http://cyclonedx.org/schema/bom/1.1"/>
-
-  <xs:complexType name="metadata">
-    <xs:sequence minOccurs="0" maxOccurs="1">
-      <xs:element name="timestamp" type="xs:dateTime" minOccurs="0">
-        <xs:annotation>
-          <xs:documentation>The date and time (timestamp) when the document was created.</xs:documentation>
-        </xs:annotation>
-      </xs:element>
-      <xs:element name="tool" minOccurs="0" type="bd:toolType">
-        <xs:annotation>
-          <xs:documentation>The tool used to create the BOM.</xs:documentation>
-        </xs:annotation>
-      </xs:element>
-      <xs:element name="authors" minOccurs="0" maxOccurs="1">
-        <xs:annotation>
-          <xs:documentation>The person(s) who created the BOM. Authors are common in BOMs created through
-            manual processes. BOMs created through automated means may not have authors.</xs:documentation>
-        </xs:annotation>
-        <xs:complexType>
-          <xs:sequence minOccurs="0" maxOccurs="unbounded">
-            <xs:element name="author" type="bd:organizationalPerson"/>
-          </xs:sequence>
-        </xs:complexType>
-      </xs:element>
-      <xs:element name="component" type="bom:component" minOccurs="0">
-        <xs:annotation>
-          <xs:documentation>The component that the BOM describes.</xs:documentation>
-        </xs:annotation>
-      </xs:element>
-      <xs:element name="manufacture" type="bd:organizationalEntity" minOccurs="0" maxOccurs="unbounded">
-        <xs:annotation>
-          <xs:documentation>The organization that manufactured the component that the BOM describes.</xs:documentation>
-        </xs:annotation>
-      </xs:element>
-      <xs:element name="supplier" type="bd:organizationalEntity" minOccurs="0" maxOccurs="unbounded">
-        <xs:annotation>
-          <xs:documentation>The organization that supplied the component that the BOM describes. The
-            supplier may often be the manufacture, but may also be a distributor or repackager.</xs:documentation>
-        </xs:annotation>
-      </xs:element>
-    </xs:sequence>
-    <xs:anyAttribute namespace="##other" processContents="lax">
-      <xs:annotation>
-        <xs:documentation>User-defined attributes may be used on this element as long as they
-          do not have the same name as an existing attribute used by the schema.</xs:documentation>
-      </xs:annotation>
-    </xs:anyAttribute>
-  </xs:complexType>
-
-  <xs:complexType name="organizationalEntity">
-    <xs:sequence minOccurs="0" maxOccurs="1">
-      <xs:element name="name" type="xs:normalizedString" minOccurs="0" maxOccurs="1">
-        <xs:annotation>
-          <xs:documentation>The name of the organization</xs:documentation>
-        </xs:annotation>
-      </xs:element>
-      <xs:element name="url" type="xs:anyURI" minOccurs="0" maxOccurs="unbounded">
-        <xs:annotation>
-          <xs:documentation>The URL of the organization. Multiple URLs are allowed.</xs:documentation>
-        </xs:annotation>
-      </xs:element>
-      <xs:element name="contact" type="bd:organizationalPerson" minOccurs="0" maxOccurs="unbounded">
-        <xs:annotation>
-          <xs:documentation>A contact person at the organization. Multiple contacts are allowed.</xs:documentation>
-        </xs:annotation>
-      </xs:element>
-    </xs:sequence>
-    <xs:anyAttribute namespace="##other" processContents="lax">
-      <xs:annotation>
-        <xs:documentation>User-defined attributes may be used on this element as long as they
-          do not have the same name as an existing attribute used by the schema.</xs:documentation>
-      </xs:annotation>
-    </xs:anyAttribute>
-  </xs:complexType>
-
-  <xs:complexType name="toolType">
-    <xs:annotation>
-      <xs:documentation>Specifies a tool (manual or automated).</xs:documentation>
-    </xs:annotation>
-    <xs:sequence minOccurs="0" maxOccurs="1">
-      <xs:element name="vendor" minOccurs="0" maxOccurs="1" type="xs:normalizedString">
-        <xs:annotation>
-          <xs:documentation>The vendor of the tool used to create the BOM.</xs:documentation>
-        </xs:annotation>
-      </xs:element>
-      <xs:element name="name" minOccurs="0" maxOccurs="1" type="xs:normalizedString">
-        <xs:annotation>
-          <xs:documentation>The name of the tool used to create the BOM.</xs:documentation>
-        </xs:annotation>
-      </xs:element>
-      <xs:element name="version" minOccurs="0" maxOccurs="1" type="xs:normalizedString">
-        <xs:annotation>
-          <xs:documentation>The version of the tool used to create the BOM.</xs:documentation>
-        </xs:annotation>
-      </xs:element>
-      <xs:element name="hashes" minOccurs="0" maxOccurs="1">
-        <xs:complexType>
-          <xs:sequence minOccurs="0" maxOccurs="unbounded">
-            <xs:element name="hash" type="bom:hashType"/>
-          </xs:sequence>
-        </xs:complexType>
-      </xs:element>
-    </xs:sequence>
-    <xs:anyAttribute namespace="##other" processContents="lax">
-      <xs:annotation>
-        <xs:documentation>User-defined attributes may be used on this element as long as they
-          do not have the same name as an existing attribute used by the schema.</xs:documentation>
-      </xs:annotation>
-    </xs:anyAttribute>
-  </xs:complexType>
-
-  <xs:complexType name="organizationalPerson">
-    <xs:sequence minOccurs="0" maxOccurs="1">
-      <xs:element name="name" type="xs:normalizedString" minOccurs="0" maxOccurs="1">
-        <xs:annotation>
-          <xs:documentation>The name of the person</xs:documentation>
-        </xs:annotation>
-      </xs:element>
-      <xs:element name="email" type="xs:normalizedString" minOccurs="0" maxOccurs="unbounded">
-        <xs:annotation>
-          <xs:documentation>The email address of the person. Multiple email addresses are allowed.</xs:documentation>
-        </xs:annotation>
-      </xs:element>
-      <xs:element name="phone" type="xs:normalizedString" minOccurs="0" maxOccurs="unbounded">
-        <xs:annotation>
-          <xs:documentation>The phone number of the person. Multiple phone numbers are allowed.</xs:documentation>
-        </xs:annotation>
-      </xs:element>
-    </xs:sequence>
-    <xs:anyAttribute namespace="##other" processContents="lax">
-      <xs:annotation>
-        <xs:documentation>User-defined attributes may be used on this element as long as they
-          do not have the same name as an existing attribute used by the schema.</xs:documentation>
-      </xs:annotation>
-    </xs:anyAttribute>
-  </xs:complexType>
-
-  <xs:element name="metadata" type="bd:metadata">
-    <xs:annotation>
-      <xs:documentation>Provides additional information about a BOM.</xs:documentation>
-    </xs:annotation>
-  </xs:element>
-
-</xs:schema>

schema/json/README.md

@@ -0,0 +1,39 @@
+# JSON Schema
+
+This is the JSON schema for output from the JSON presenter (`syft <img> -o json`). The required inputs for defining the JSON schema are as follows:
+
+- the value of `internal.JSONSchemaVersion` that governs the schema filename
+- the `Document` struct definition within `syft/presenters/json/document.go` that governs the overall document shape
+- the `metadataContainer` struct definition within `schema/json/generate.go` that governs the allowable shapes of `pkg.Package.Metadata`
+
+With regard to testing the JSON schema, integration test cases provided by the developer are used as examples to validate that JSON output from Syft is always valid relative to the `schema/json/schema-$VERSION.json` file.
+
+## Versioning
+
+Versioning the JSON schema must be done manually by changing the `JSONSchemaVersion` constant within `internal/constants.go`.
+
+This schema is being versioned based off of the "SchemaVer" guidelines, which slightly diverges from Semantic Versioning to tailor for the purposes of data models. 
+
+Given a version number format `MODEL.REVISION.ADDITION`:
+
+- `MODEL`: increment when you make a breaking schema change which will prevent interaction with any historical data
+- `REVISION`: increment when you make a schema change which may prevent interaction with some historical data
+- `ADDITION`: increment when you make a schema change that is compatible with all historical data
+
+## Adding a New `pkg.*Metadata` Type
+
+When adding a new `pkg.*Metadata` that is assigned to the `pkg.Package.Metadata` struct field it is important that a few things
+are done:
+
+- a new integration test case is added to `test/integration/pkg_cases_test.go` that exercises the new package type with the new metadata
+- the new metadata struct is added to the `metadataContainer` struct within `schema/json/generate.go`
+
+## Generating a New Schema
+
+Create the new schema by running `cd schema/json && go run generate.go` (note you must be in the `schema/json` dir while running this):
+
+- If there is **not** an existing schema for the given version, then the new schema file will be written to `schema/json/schema-$VERSION.json`
+- If there is an existing schema for the given version and the new schema matches the existing schema, no action is taken
+- If there is an existing schema for the given version and the new schema **does not** match the existing schema, an error is shown indicating to increment the version appropriately (see the "Versioning" section)
+
+***Note: never delete a JSON schema and never change an existing JSON schema once it has been published in a release!*** Only add new schemas with a newly incremented version. All previous schema files must be stored in the `schema/json/` directory.

schema/json/generate.go

@@ -1,11 +1,17 @@
 package main
 
 import (
+	"bytes"
 	"encoding/json"
 	"fmt"
+	"io/ioutil"
 	"os"
+	"sort"
+	"strings"
 
 	"github.com/alecthomas/jsonschema"
+	"github.com/anchore/syft/internal"
+	"github.com/anchore/syft/syft/pkg"
 	jsonPresenter "github.com/anchore/syft/syft/presenter/json"
 )
 
@@ -16,20 +22,104 @@ are not captured (empty interfaces). This means that pkg.Package.Metadata is not
 can be extended to include specific package metadata struct shapes in the future.
 */
 
+// This should represent all possible metadatas represented in the pkg.Package.Metadata field (an interface{}).
+// When a new package metadata definition is created it will need to be manually added here. The variable name does
+// not matter as long as it is exported.
+type metadataContainer struct {
+	Apk    pkg.ApkMetadata
+	Dpkg   pkg.DpkgMetadata
+	Gem    pkg.GemMetadata
+	Java   pkg.JavaMetadata
+	Npm    pkg.NpmPackageJSONMetadata
+	Python pkg.PythonPackageMetadata
+	Rpm    pkg.RpmdbMetadata
+}
+
+// nolint:funlen
 func main() {
-	j := jsonschema.Reflect(&jsonPresenter.Document{})
-	filename := "schema.json"
-	fh, err := os.OpenFile("schema.json", os.O_RDWR|os.O_CREATE, 0755)
-	if err != nil {
-		panic(err)
+	metadataSchema := jsonschema.Reflect(&metadataContainer{})
+	documentSchema := jsonschema.Reflect(&jsonPresenter.Document{})
+
+	// TODO: inject source definitions
+
+	// inject the definitions of all metadatas into the schema definitions
+
+	var metadataNames []string
+	for name, definition := range metadataSchema.Definitions {
+		if name == "metadataContainer" {
+			// ignore the definition for the fake container
+			continue
+		}
+		documentSchema.Definitions[name] = definition
+		if strings.HasSuffix(name, "Metadata") {
+			metadataNames = append(metadataNames, name)
+		}
 	}
-	enc := json.NewEncoder(fh)
+
+	// ensure the generated list of names is stable between runs
+	sort.Strings(metadataNames)
+
+	var metadataTypes = []map[string]string{
+		// allow for no metadata to be provided
+		{"type": "null"},
+	}
+	for _, name := range metadataNames {
+		metadataTypes = append(metadataTypes, map[string]string{
+			"$ref": fmt.Sprintf("#/definitions/%s", name),
+		})
+	}
+
+	// set the "anyOf" field for Package.Metadata to be a conjunction of several types
+	documentSchema.Definitions["Package"].Properties.Set("metadata", map[string][]map[string]string{
+		"anyOf": metadataTypes,
+	})
+
+	filename := fmt.Sprintf("schema-%s.json", internal.JSONSchemaVersion)
+
+	var newSchemaBuffer = new(bytes.Buffer)
+	enc := json.NewEncoder(newSchemaBuffer)
 	// prevent > and < from being escaped in the payload
 	enc.SetEscapeHTML(false)
 	enc.SetIndent("", "  ")
-	err = enc.Encode(&j)
+	err := enc.Encode(&documentSchema)
 	if err != nil {
 		panic(err)
 	}
+
+	if _, err := os.Stat(filename); !os.IsNotExist(err) {
+		// check if the schema is the same...
+		existingFh, err := os.Open(filename)
+		if err != nil {
+			panic(err)
+		}
+
+		existingSchemaBytes, err := ioutil.ReadAll(existingFh)
+		if err != nil {
+			panic(err)
+		}
+
+		if bytes.Equal(existingSchemaBytes, newSchemaBuffer.Bytes()) {
+			// the generated schema is the same, bail with no error :)
+			fmt.Println("No change to the existing schema!")
+			os.Exit(0)
+		}
+
+		// the generated schema is different, bail with error :(
+		fmt.Printf("Cowardly refusing to overwrite existing schema (%s)!\nSee the scheam/json/README.md for how to increment\n", filename)
+		os.Exit(1)
+	}
+
+	fh, err := os.Create(filename)
+	if err != nil {
+		panic(err)
+	}
+
+	_, err = fh.Write(newSchemaBuffer.Bytes())
+	if err != nil {
+		panic(err)
+	}
+
+	defer fh.Close()
+
 	fmt.Printf("wrote new schema to %q\n", filename)
 }

schema/json/schema-1.0.0.json

@@ -0,0 +1,678 @@
+{
+  "$schema": "http://json-schema.org/draft-04/schema#",
+  "$ref": "#/definitions/Document",
+  "definitions": {
+    "ApkFileRecord": {
+      "required": [
+        "path"
+      ],
+      "properties": {
+        "path": {
+          "type": "string"
+        },
+        "ownerUid": {
+          "type": "string"
+        },
+        "ownerGid": {
+          "type": "string"
+        },
+        "permissions": {
+          "type": "string"
+        },
+        "checksum": {
+          "type": "string"
+        }
+      },
+      "additionalProperties": false,
+      "type": "object"
+    },
+    "ApkMetadata": {
+      "required": [
+        "package",
+        "originPackage",
+        "maintainer",
+        "version",
+        "license",
+        "architecture",
+        "url",
+        "description",
+        "size",
+        "installedSize",
+        "pullDependencies",
+        "pullChecksum",
+        "gitCommitOfApkPort",
+        "files"
+      ],
+      "properties": {
+        "package": {
+          "type": "string"
+        },
+        "originPackage": {
+          "type": "string"
+        },
+        "maintainer": {
+          "type": "string"
+        },
+        "version": {
+          "type": "string"
+        },
+        "license": {
+          "type": "string"
+        },
+        "architecture": {
+          "type": "string"
+        },
+        "url": {
+          "type": "string"
+        },
+        "description": {
+          "type": "string"
+        },
+        "size": {
+          "type": "integer"
+        },
+        "installedSize": {
+          "type": "integer"
+        },
+        "pullDependencies": {
+          "type": "string"
+        },
+        "pullChecksum": {
+          "type": "string"
+        },
+        "gitCommitOfApkPort": {
+          "type": "string"
+        },
+        "files": {
+          "items": {
+            "$schema": "http://json-schema.org/draft-04/schema#",
+            "$ref": "#/definitions/ApkFileRecord"
+          },
+          "type": "array"
+        }
+      },
+      "additionalProperties": false,
+      "type": "object"
+    },
+    "Descriptor": {
+      "required": [
+        "name",
+        "version"
+      ],
+      "properties": {
+        "name": {
+          "type": "string"
+        },
+        "version": {
+          "type": "string"
+        }
+      },
+      "additionalProperties": false,
+      "type": "object"
+    },
+    "Distribution": {
+      "required": [
+        "name",
+        "version",
+        "idLike"
+      ],
+      "properties": {
+        "name": {
+          "type": "string"
+        },
+        "version": {
+          "type": "string"
+        },
+        "idLike": {
+          "type": "string"
+        }
+      },
+      "additionalProperties": false,
+      "type": "object"
+    },
+    "Document": {
+      "required": [
+        "artifacts",
+        "source",
+        "distro",
+        "descriptor",
+        "schema"
+      ],
+      "properties": {
+        "artifacts": {
+          "items": {
+            "$schema": "http://json-schema.org/draft-04/schema#",
+            "$ref": "#/definitions/Package"
+          },
+          "type": "array"
+        },
+        "source": {
+          "$schema": "http://json-schema.org/draft-04/schema#",
+          "$ref": "#/definitions/Source"
+        },
+        "distro": {
+          "$schema": "http://json-schema.org/draft-04/schema#",
+          "$ref": "#/definitions/Distribution"
+        },
+        "descriptor": {
+          "$schema": "http://json-schema.org/draft-04/schema#",
+          "$ref": "#/definitions/Descriptor"
+        },
+        "schema": {
+          "$schema": "http://json-schema.org/draft-04/schema#",
+          "$ref": "#/definitions/Schema"
+        }
+      },
+      "additionalProperties": false,
+      "type": "object"
+    },
+    "DpkgFileRecord": {
+      "required": [
+        "path",
+        "md5"
+      ],
+      "properties": {
+        "path": {
+          "type": "string"
+        },
+        "md5": {
+          "type": "string"
+        }
+      },
+      "additionalProperties": false,
+      "type": "object"
+    },
+    "DpkgMetadata": {
+      "required": [
+        "package",
+        "source",
+        "version",
+        "architecture",
+        "maintainer",
+        "installedSize",
+        "files"
+      ],
+      "properties": {
+        "package": {
+          "type": "string"
+        },
+        "source": {
+          "type": "string"
+        },
+        "version": {
+          "type": "string"
+        },
+        "architecture": {
+          "type": "string"
+        },
+        "maintainer": {
+          "type": "string"
+        },
+        "installedSize": {
+          "type": "integer"
+        },
+        "files": {
+          "items": {
+            "$schema": "http://json-schema.org/draft-04/schema#",
+            "$ref": "#/definitions/DpkgFileRecord"
+          },
+          "type": "array"
+        }
+      },
+      "additionalProperties": false,
+      "type": "object"
+    },
+    "GemMetadata": {
+      "required": [
+        "name",
+        "version"
+      ],
+      "properties": {
+        "name": {
+          "type": "string"
+        },
+        "version": {
+          "type": "string"
+        },
+        "files": {
+          "items": {
+            "type": "string"
+          },
+          "type": "array"
+        },
+        "authors": {
+          "items": {
+            "type": "string"
+          },
+          "type": "array"
+        },
+        "licenses": {
+          "items": {
+            "type": "string"
+          },
+          "type": "array"
+        },
+        "homepage": {
+          "type": "string"
+        }
+      },
+      "additionalProperties": false,
+      "type": "object"
+    },
+    "JavaManifest": {
+      "properties": {
+        "main": {
+          "patternProperties": {
+            ".*": {
+              "type": "string"
+            }
+          },
+          "type": "object"
+        },
+        "namedSections": {
+          "patternProperties": {
+            ".*": {
+              "patternProperties": {
+                ".*": {
+                  "type": "string"
+                }
+              },
+              "type": "object"
+            }
+          },
+          "type": "object"
+        }
+      },
+      "additionalProperties": false,
+      "type": "object"
+    },
+    "JavaMetadata": {
+      "required": [
+        "virtualPath"
+      ],
+      "properties": {
+        "virtualPath": {
+          "type": "string"
+        },
+        "manifest": {
+          "$schema": "http://json-schema.org/draft-04/schema#",
+          "$ref": "#/definitions/JavaManifest"
+        },
+        "pomProperties": {
+          "$schema": "http://json-schema.org/draft-04/schema#",
+          "$ref": "#/definitions/PomProperties"
+        }
+      },
+      "additionalProperties": false,
+      "type": "object"
+    },
+    "Location": {
+      "required": [
+        "path"
+      ],
+      "properties": {
+        "path": {
+          "type": "string"
+        },
+        "layerID": {
+          "type": "string"
+        }
+      },
+      "additionalProperties": false,
+      "type": "object"
+    },
+    "NpmPackageJSONMetadata": {
+      "required": [
+        "author",
+        "licenses",
+        "homepage",
+        "description",
+        "url"
+      ],
+      "properties": {
+        "files": {
+          "items": {
+            "type": "string"
+          },
+          "type": "array"
+        },
+        "author": {
+          "type": "string"
+        },
+        "licenses": {
+          "items": {
+            "type": "string"
+          },
+          "type": "array"
+        },
+        "homepage": {
+          "type": "string"
+        },
+        "description": {
+          "type": "string"
+        },
+        "url": {
+          "type": "string"
+        }
+      },
+      "additionalProperties": false,
+      "type": "object"
+    },
+    "Package": {
+      "required": [
+        "name",
+        "version",
+        "type",
+        "foundBy",
+        "locations",
+        "licenses",
+        "language",
+        "cpes",
+        "purl",
+        "metadataType",
+        "metadata"
+      ],
+      "properties": {
+        "name": {
+          "type": "string"
+        },
+        "version": {
+          "type": "string"
+        },
+        "type": {
+          "type": "string"
+        },
+        "foundBy": {
+          "type": "string"
+        },
+        "locations": {
+          "items": {
+            "$schema": "http://json-schema.org/draft-04/schema#",
+            "$ref": "#/definitions/Location"
+          },
+          "type": "array"
+        },
+        "licenses": {
+          "items": {
+            "type": "string"
+          },
+          "type": "array"
+        },
+        "language": {
+          "type": "string"
+        },
+        "cpes": {
+          "items": {
+            "type": "string"
+          },
+          "type": "array"
+        },
+        "purl": {
+          "type": "string"
+        },
+        "metadataType": {
+          "type": "string"
+        },
+        "metadata": {
+          "anyOf": [
+            {
+              "type": "null"
+            },
+            {
+              "$ref": "#/definitions/ApkMetadata"
+            },
+            {
+              "$ref": "#/definitions/DpkgMetadata"
+            },
+            {
+              "$ref": "#/definitions/GemMetadata"
+            },
+            {
+              "$ref": "#/definitions/JavaMetadata"
+            },
+            {
+              "$ref": "#/definitions/NpmPackageJSONMetadata"
+            },
+            {
+              "$ref": "#/definitions/PythonPackageMetadata"
+            },
+            {
+              "$ref": "#/definitions/RpmdbMetadata"
+            }
+          ]
+        }
+      },
+      "additionalProperties": false,
+      "type": "object"
+    },
+    "PomProperties": {
+      "required": [
+        "path",
+        "name",
+        "groupId",
+        "artifactId",
+        "version",
+        "extraFields"
+      ],
+      "properties": {
+        "path": {
+          "type": "string"
+        },
+        "name": {
+          "type": "string"
+        },
+        "groupId": {
+          "type": "string"
+        },
+        "artifactId": {
+          "type": "string"
+        },
+        "version": {
+          "type": "string"
+        },
+        "extraFields": {
+          "patternProperties": {
+            ".*": {
+              "type": "string"
+            }
+          },
+          "type": "object"
+        }
+      },
+      "additionalProperties": false,
+      "type": "object"
+    },
+    "PythonFileDigest": {
+      "required": [
+        "algorithm",
+        "value"
+      ],
+      "properties": {
+        "algorithm": {
+          "type": "string"
+        },
+        "value": {
+          "type": "string"
+        }
+      },
+      "additionalProperties": false,
+      "type": "object"
+    },
+    "PythonFileRecord": {
+      "required": [
+        "path"
+      ],
+      "properties": {
+        "path": {
+          "type": "string"
+        },
+        "digest": {
+          "$schema": "http://json-schema.org/draft-04/schema#",
+          "$ref": "#/definitions/PythonFileDigest"
+        },
+        "size": {
+          "type": "string"
+        }
+      },
+      "additionalProperties": false,
+      "type": "object"
+    },
+    "PythonPackageMetadata": {
+      "required": [
+        "name",
+        "version",
+        "license",
+        "author",
+        "authorEmail",
+        "platform",
+        "sitePackagesRootPath"
+      ],
+      "properties": {
+        "name": {
+          "type": "string"
+        },
+        "version": {
+          "type": "string"
+        },
+        "license": {
+          "type": "string"
+        },
+        "author": {
+          "type": "string"
+        },
+        "authorEmail": {
+          "type": "string"
+        },
+        "platform": {
+          "type": "string"
+        },
+        "files": {
+          "items": {
+            "$schema": "http://json-schema.org/draft-04/schema#",
+            "$ref": "#/definitions/PythonFileRecord"
+          },
+          "type": "array"
+        },
+        "sitePackagesRootPath": {
+          "type": "string"
+        },
+        "topLevelPackages": {
+          "items": {
+            "type": "string"
+          },
+          "type": "array"
+        }
+      },
+      "additionalProperties": false,
+      "type": "object"
+    },
+    "RpmdbFileRecord": {
+      "required": [
+        "path",
+        "mode",
+        "size",
+        "sha256"
+      ],
+      "properties": {
+        "path": {
+          "type": "string"
+        },
+        "mode": {
+          "type": "integer"
+        },
+        "size": {
+          "type": "integer"
+        },
+        "sha256": {
+          "type": "string"
+        }
+      },
+      "additionalProperties": false,
+      "type": "object"
+    },
+    "RpmdbMetadata": {
+      "required": [
+        "name",
+        "version",
+        "epoch",
+        "architecture",
+        "release",
+        "sourceRpm",
+        "size",
+        "license",
+        "vendor",
+        "files"
+      ],
+      "properties": {
+        "name": {
+          "type": "string"
+        },
+        "version": {
+          "type": "string"
+        },
+        "epoch": {
+          "type": "integer"
+        },
+        "architecture": {
+          "type": "string"
+        },
+        "release": {
+          "type": "string"
+        },
+        "sourceRpm": {
+          "type": "string"
+        },
+        "size": {
+          "type": "integer"
+        },
+        "license": {
+          "type": "string"
+        },
+        "vendor": {
+          "type": "string"
+        },
+        "files": {
+          "items": {
+            "$schema": "http://json-schema.org/draft-04/schema#",
+            "$ref": "#/definitions/RpmdbFileRecord"
+          },
+          "type": "array"
+        }
+      },
+      "additionalProperties": false,
+      "type": "object"
+    },
+    "Schema": {
+      "required": [
+        "version",
+        "url"
+      ],
+      "properties": {
+        "version": {
+          "type": "string"
+        },
+        "url": {
+          "type": "string"
+        }
+      },
+      "additionalProperties": false,
+      "type": "object"
+    },
+    "Source": {
+      "required": [
+        "type",
+        "target"
+      ],
+      "properties": {
+        "type": {
+          "type": "string"
+        },
+        "target": {
+          "additionalProperties": true
+        }
+      },
+      "additionalProperties": false,
+      "type": "object"
+    }
+  }
+}

schema/json/schema-1.0.1.json

@@ -0,0 +1,682 @@
+{
+  "$schema": "http://json-schema.org/draft-04/schema#",
+  "$ref": "#/definitions/Document",
+  "definitions": {
+    "ApkFileRecord": {
+      "required": [
+        "path"
+      ],
+      "properties": {
+        "path": {
+          "type": "string"
+        },
+        "ownerUid": {
+          "type": "string"
+        },
+        "ownerGid": {
+          "type": "string"
+        },
+        "permissions": {
+          "type": "string"
+        },
+        "checksum": {
+          "type": "string"
+        }
+      },
+      "additionalProperties": false,
+      "type": "object"
+    },
+    "ApkMetadata": {
+      "required": [
+        "package",
+        "originPackage",
+        "maintainer",
+        "version",
+        "license",
+        "architecture",
+        "url",
+        "description",
+        "size",
+        "installedSize",
+        "pullDependencies",
+        "pullChecksum",
+        "gitCommitOfApkPort",
+        "files"
+      ],
+      "properties": {
+        "package": {
+          "type": "string"
+        },
+        "originPackage": {
+          "type": "string"
+        },
+        "maintainer": {
+          "type": "string"
+        },
+        "version": {
+          "type": "string"
+        },
+        "license": {
+          "type": "string"
+        },
+        "architecture": {
+          "type": "string"
+        },
+        "url": {
+          "type": "string"
+        },
+        "description": {
+          "type": "string"
+        },
+        "size": {
+          "type": "integer"
+        },
+        "installedSize": {
+          "type": "integer"
+        },
+        "pullDependencies": {
+          "type": "string"
+        },
+        "pullChecksum": {
+          "type": "string"
+        },
+        "gitCommitOfApkPort": {
+          "type": "string"
+        },
+        "files": {
+          "items": {
+            "$schema": "http://json-schema.org/draft-04/schema#",
+            "$ref": "#/definitions/ApkFileRecord"
+          },
+          "type": "array"
+        }
+      },
+      "additionalProperties": false,
+      "type": "object"
+    },
+    "Descriptor": {
+      "required": [
+        "name",
+        "version"
+      ],
+      "properties": {
+        "name": {
+          "type": "string"
+        },
+        "version": {
+          "type": "string"
+        }
+      },
+      "additionalProperties": false,
+      "type": "object"
+    },
+    "Distribution": {
+      "required": [
+        "name",
+        "version",
+        "idLike"
+      ],
+      "properties": {
+        "name": {
+          "type": "string"
+        },
+        "version": {
+          "type": "string"
+        },
+        "idLike": {
+          "type": "string"
+        }
+      },
+      "additionalProperties": false,
+      "type": "object"
+    },
+    "Document": {
+      "required": [
+        "artifacts",
+        "source",
+        "distro",
+        "descriptor",
+        "schema"
+      ],
+      "properties": {
+        "artifacts": {
+          "items": {
+            "$schema": "http://json-schema.org/draft-04/schema#",
+            "$ref": "#/definitions/Package"
+          },
+          "type": "array"
+        },
+        "source": {
+          "$schema": "http://json-schema.org/draft-04/schema#",
+          "$ref": "#/definitions/Source"
+        },
+        "distro": {
+          "$schema": "http://json-schema.org/draft-04/schema#",
+          "$ref": "#/definitions/Distribution"
+        },
+        "descriptor": {
+          "$schema": "http://json-schema.org/draft-04/schema#",
+          "$ref": "#/definitions/Descriptor"
+        },
+        "schema": {
+          "$schema": "http://json-schema.org/draft-04/schema#",
+          "$ref": "#/definitions/Schema"
+        }
+      },
+      "additionalProperties": false,
+      "type": "object"
+    },
+    "DpkgFileRecord": {
+      "required": [
+        "path",
+        "md5"
+      ],
+      "properties": {
+        "path": {
+          "type": "string"
+        },
+        "md5": {
+          "type": "string"
+        }
+      },
+      "additionalProperties": false,
+      "type": "object"
+    },
+    "DpkgMetadata": {
+      "required": [
+        "package",
+        "source",
+        "version",
+        "sourceVersion",
+        "architecture",
+        "maintainer",
+        "installedSize",
+        "files"
+      ],
+      "properties": {
+        "package": {
+          "type": "string"
+        },
+        "source": {
+          "type": "string"
+        },
+        "version": {
+          "type": "string"
+        },
+        "sourceVersion": {
+          "type": "string"
+        },
+        "architecture": {
+          "type": "string"
+        },
+        "maintainer": {
+          "type": "string"
+        },
+        "installedSize": {
+          "type": "integer"
+        },
+        "files": {
+          "items": {
+            "$schema": "http://json-schema.org/draft-04/schema#",
+            "$ref": "#/definitions/DpkgFileRecord"
+          },
+          "type": "array"
+        }
+      },
+      "additionalProperties": false,
+      "type": "object"
+    },
+    "GemMetadata": {
+      "required": [
+        "name",
+        "version"
+      ],
+      "properties": {
+        "name": {
+          "type": "string"
+        },
+        "version": {
+          "type": "string"
+        },
+        "files": {
+          "items": {
+            "type": "string"
+          },
+          "type": "array"
+        },
+        "authors": {
+          "items": {
+            "type": "string"
+          },
+          "type": "array"
+        },
+        "licenses": {
+          "items": {
+            "type": "string"
+          },
+          "type": "array"
+        },
+        "homepage": {
+          "type": "string"
+        }
+      },
+      "additionalProperties": false,
+      "type": "object"
+    },
+    "JavaManifest": {
+      "properties": {
+        "main": {
+          "patternProperties": {
+            ".*": {
+              "type": "string"
+            }
+          },
+          "type": "object"
+        },
+        "namedSections": {
+          "patternProperties": {
+            ".*": {
+              "patternProperties": {
+                ".*": {
+                  "type": "string"
+                }
+              },
+              "type": "object"
+            }
+          },
+          "type": "object"
+        }
+      },
+      "additionalProperties": false,
+      "type": "object"
+    },
+    "JavaMetadata": {
+      "required": [
+        "virtualPath"
+      ],
+      "properties": {
+        "virtualPath": {
+          "type": "string"
+        },
+        "manifest": {
+          "$schema": "http://json-schema.org/draft-04/schema#",
+          "$ref": "#/definitions/JavaManifest"
+        },
+        "pomProperties": {
+          "$schema": "http://json-schema.org/draft-04/schema#",
+          "$ref": "#/definitions/PomProperties"
+        }
+      },
+      "additionalProperties": false,
+      "type": "object"
+    },
+    "Location": {
+      "required": [
+        "path"
+      ],
+      "properties": {
+        "path": {
+          "type": "string"
+        },
+        "layerID": {
+          "type": "string"
+        }
+      },
+      "additionalProperties": false,
+      "type": "object"
+    },
+    "NpmPackageJSONMetadata": {
+      "required": [
+        "author",
+        "licenses",
+        "homepage",
+        "description",
+        "url"
+      ],
+      "properties": {
+        "files": {
+          "items": {
+            "type": "string"
+          },
+          "type": "array"
+        },
+        "author": {
+          "type": "string"
+        },
+        "licenses": {
+          "items": {
+            "type": "string"
+          },
+          "type": "array"
+        },
+        "homepage": {
+          "type": "string"
+        },
+        "description": {
+          "type": "string"
+        },
+        "url": {
+          "type": "string"
+        }
+      },
+      "additionalProperties": false,
+      "type": "object"
+    },
+    "Package": {
+      "required": [
+        "name",
+        "version",
+        "type",
+        "foundBy",
+        "locations",
+        "licenses",
+        "language",
+        "cpes",
+        "purl",
+        "metadataType",
+        "metadata"
+      ],
+      "properties": {
+        "name": {
+          "type": "string"
+        },
+        "version": {
+          "type": "string"
+        },
+        "type": {
+          "type": "string"
+        },
+        "foundBy": {
+          "type": "string"
+        },
+        "locations": {
+          "items": {
+            "$schema": "http://json-schema.org/draft-04/schema#",
+            "$ref": "#/definitions/Location"
+          },
+          "type": "array"
+        },
+        "licenses": {
+          "items": {
+            "type": "string"
+          },
+          "type": "array"
+        },
+        "language": {
+          "type": "string"
+        },
+        "cpes": {
+          "items": {
+            "type": "string"
+          },
+          "type": "array"
+        },
+        "purl": {
+          "type": "string"
+        },
+        "metadataType": {
+          "type": "string"
+        },
+        "metadata": {
+          "anyOf": [
+            {
+              "type": "null"
+            },
+            {
+              "$ref": "#/definitions/ApkMetadata"
+            },
+            {
+              "$ref": "#/definitions/DpkgMetadata"
+            },
+            {
+              "$ref": "#/definitions/GemMetadata"
+            },
+            {
+              "$ref": "#/definitions/JavaMetadata"
+            },
+            {
+              "$ref": "#/definitions/NpmPackageJSONMetadata"
+            },
+            {
+              "$ref": "#/definitions/PythonPackageMetadata"
+            },
+            {
+              "$ref": "#/definitions/RpmdbMetadata"
+            }
+          ]
+        }
+      },
+      "additionalProperties": false,
+      "type": "object"
+    },
+    "PomProperties": {
+      "required": [
+        "path",
+        "name",
+        "groupId",
+        "artifactId",
+        "version",
+        "extraFields"
+      ],
+      "properties": {
+        "path": {
+          "type": "string"
+        },
+        "name": {
+          "type": "string"
+        },
+        "groupId": {
+          "type": "string"
+        },
+        "artifactId": {
+          "type": "string"
+        },
+        "version": {
+          "type": "string"
+        },
+        "extraFields": {
+          "patternProperties": {
+            ".*": {
+              "type": "string"
+            }
+          },
+          "type": "object"
+        }
+      },
+      "additionalProperties": false,
+      "type": "object"
+    },
+    "PythonFileDigest": {
+      "required": [
+        "algorithm",
+        "value"
+      ],
+      "properties": {
+        "algorithm": {
+          "type": "string"
+        },
+        "value": {
+          "type": "string"
+        }
+      },
+      "additionalProperties": false,
+      "type": "object"
+    },
+    "PythonFileRecord": {
+      "required": [
+        "path"
+      ],
+      "properties": {
+        "path": {
+          "type": "string"
+        },
+        "digest": {
+          "$schema": "http://json-schema.org/draft-04/schema#",
+          "$ref": "#/definitions/PythonFileDigest"
+        },
+        "size": {
+          "type": "string"
+        }
+      },
+      "additionalProperties": false,
+      "type": "object"
+    },
+    "PythonPackageMetadata": {
+      "required": [
+        "name",
+        "version",
+        "license",
+        "author",
+        "authorEmail",
+        "platform",
+        "sitePackagesRootPath"
+      ],
+      "properties": {
+        "name": {
+          "type": "string"
+        },
+        "version": {
+          "type": "string"
+        },
+        "license": {
+          "type": "string"
+        },
+        "author": {
+          "type": "string"
+        },
+        "authorEmail": {
+          "type": "string"
+        },
+        "platform": {
+          "type": "string"
+        },
+        "files": {
+          "items": {
+            "$schema": "http://json-schema.org/draft-04/schema#",
+            "$ref": "#/definitions/PythonFileRecord"
+          },
+          "type": "array"
+        },
+        "sitePackagesRootPath": {
+          "type": "string"
+        },
+        "topLevelPackages": {
+          "items": {
+            "type": "string"
+          },
+          "type": "array"
+        }
+      },
+      "additionalProperties": false,
+      "type": "object"
+    },
+    "RpmdbFileRecord": {
+      "required": [
+        "path",
+        "mode",
+        "size",
+        "sha256"
+      ],
+      "properties": {
+        "path": {
+          "type": "string"
+        },
+        "mode": {
+          "type": "integer"
+        },
+        "size": {
+          "type": "integer"
+        },
+        "sha256": {
+          "type": "string"
+        }
+      },
+      "additionalProperties": false,
+      "type": "object"
+    },
+    "RpmdbMetadata": {
+      "required": [
+        "name",
+        "version",
+        "epoch",
+        "architecture",
+        "release",
+        "sourceRpm",
+        "size",
+        "license",
+        "vendor",
+        "files"
+      ],
+      "properties": {
+        "name": {
+          "type": "string"
+        },
+        "version": {
+          "type": "string"
+        },
+        "epoch": {
+          "type": "integer"
+        },
+        "architecture": {
+          "type": "string"
+        },
+        "release": {
+          "type": "string"
+        },
+        "sourceRpm": {
+          "type": "string"
+        },
+        "size": {
+          "type": "integer"
+        },
+        "license": {
+          "type": "string"
+        },
+        "vendor": {
+          "type": "string"
+        },
+        "files": {
+          "items": {
+            "$schema": "http://json-schema.org/draft-04/schema#",
+            "$ref": "#/definitions/RpmdbFileRecord"
+          },
+          "type": "array"
+        }
+      },
+      "additionalProperties": false,
+      "type": "object"
+    },
+    "Schema": {
+      "required": [
+        "version",
+        "url"
+      ],
+      "properties": {
+        "version": {
+          "type": "string"
+        },
+        "url": {
+          "type": "string"
+        }
+      },
+      "additionalProperties": false,
+      "type": "object"
+    },
+    "Source": {
+      "required": [
+        "type",
+        "target"
+      ],
+      "properties": {
+        "type": {
+          "type": "string"
+        },
+        "target": {
+          "additionalProperties": true
+        }
+      },
+      "additionalProperties": false,
+      "type": "object"
+    }
+  }
+}

schema/json/schema.json

@@ -1,165 +0,0 @@
-{
-  "$schema": "http://json-schema.org/draft-04/schema#",
-  "$ref": "#/definitions/Document",
-  "definitions": {
-    "Descriptor": {
-      "required": [
-        "name",
-        "version"
-      ],
-      "properties": {
-        "name": {
-          "type": "string"
-        },
-        "version": {
-          "type": "string"
-        }
-      },
-      "additionalProperties": false,
-      "type": "object"
-    },
-    "Distribution": {
-      "required": [
-        "name",
-        "version",
-        "idLike"
-      ],
-      "properties": {
-        "name": {
-          "type": "string"
-        },
-        "version": {
-          "type": "string"
-        },
-        "idLike": {
-          "type": "string"
-        }
-      },
-      "additionalProperties": false,
-      "type": "object"
-    },
-    "Document": {
-      "required": [
-        "artifacts",
-        "source",
-        "distro",
-        "descriptor"
-      ],
-      "properties": {
-        "artifacts": {
-          "items": {
-            "$schema": "http://json-schema.org/draft-04/schema#",
-            "$ref": "#/definitions/Package"
-          },
-          "type": "array"
-        },
-        "source": {
-          "$schema": "http://json-schema.org/draft-04/schema#",
-          "$ref": "#/definitions/Source"
-        },
-        "distro": {
-          "$schema": "http://json-schema.org/draft-04/schema#",
-          "$ref": "#/definitions/Distribution"
-        },
-        "descriptor": {
-          "$schema": "http://json-schema.org/draft-04/schema#",
-          "$ref": "#/definitions/Descriptor"
-        }
-      },
-      "additionalProperties": false,
-      "type": "object"
-    },
-    "Location": {
-      "required": [
-        "path"
-      ],
-      "properties": {
-        "path": {
-          "type": "string"
-        },
-        "layerID": {
-          "type": "string"
-        }
-      },
-      "additionalProperties": false,
-      "type": "object"
-    },
-    "Package": {
-      "required": [
-        "name",
-        "version",
-        "type",
-        "foundBy",
-        "locations",
-        "licenses",
-        "language",
-        "cpes",
-        "purl",
-        "metadataType"
-      ],
-      "properties": {
-        "name": {
-          "type": "string"
-        },
-        "version": {
-          "type": "string"
-        },
-        "type": {
-          "type": "string"
-        },
-        "foundBy": {
-          "type": "string"
-        },
-        "locations": {
-          "items": {
-            "$schema": "http://json-schema.org/draft-04/schema#",
-            "$ref": "#/definitions/Location"
-          },
-          "type": "array"
-        },
-        "licenses": {
-          "items": {
-            "type": "string"
-          },
-          "type": "array"
-        },
-        "language": {
-          "type": "string"
-        },
-        "cpes": {
-          "items": {
-            "type": "string"
-          },
-          "type": "array"
-        },
-        "purl": {
-          "type": "string"
-        },
-        "metadataType": {
-          "type": "string"
-        },
-        "metadata": {
-          "additionalProperties": true
-        }
-      },
-      "additionalProperties": false,
-      "type": "object"
-    },
-    "Source": {
-      "required": [
-        "type",
-        "target"
-      ],
-      "properties": {
-        "type": {
-          "type": "string"
-        },
-        "target": {
-          "additionalProperties": true
-        }
-      },
-      "additionalProperties": false,
-      "type": "object"
-    }
-  }
-}

syft/cataloger/common/generic_cataloger.go

@@ -4,7 +4,7 @@ Package common provides generic utilities used by multiple catalogers.
 package common
 
 import (
-	"strings"
+	"io"
 
 	"github.com/anchore/syft/internal/log"
 	"github.com/anchore/syft/syft/pkg"
@@ -89,7 +89,7 @@ func (c *GenericCataloger) selectFiles(resolver source.FileResolver) []source.Lo
 }
 
 // catalog takes a set of file contents and uses any configured parser functions to resolve and return discovered packages
-func (c *GenericCataloger) catalog(contents map[source.Location]string) ([]pkg.Package, error) {
+func (c *GenericCataloger) catalog(contents map[source.Location]io.ReadCloser) ([]pkg.Package, error) {
 	defer c.clear()
 
 	packages := make([]pkg.Package, 0)
@@ -101,7 +101,7 @@ func (c *GenericCataloger) catalog(contents map[source.Location]string) ([]pkg.P
 			continue
 		}
 
-		entries, err := parser(location.Path, strings.NewReader(content))
+		entries, err := parser(location.RealPath, content)
 		if err != nil {
 			// TODO: should we fail? or only log?
 			log.Warnf("cataloger '%s' failed to parse entries (location=%+v): %+v", c.upstreamCataloger, location, err)

syft/cataloger/common/generic_cataloger_test.go

@@ -4,28 +4,32 @@ import (
 	"fmt"
 	"io"
 	"io/ioutil"
+	"strings"
 	"testing"
 
-	"github.com/anchore/syft/syft/source"
-
 	"github.com/anchore/syft/syft/pkg"
+	"github.com/anchore/syft/syft/source"
 )
 
 type testResolverMock struct {
-	contents map[source.Location]string
+	contents map[source.Location]io.ReadCloser
 }
 
 func newTestResolver() *testResolverMock {
 	return &testResolverMock{
-		contents: make(map[source.Location]string),
+		contents: make(map[source.Location]io.ReadCloser),
 	}
 }
 
-func (r *testResolverMock) FileContentsByLocation(_ source.Location) (string, error) {
-	return "", fmt.Errorf("not implemented")
+func (r testResolverMock) HasPath(path string) bool {
+	panic("not implemented")
 }
 
-func (r *testResolverMock) MultipleFileContentsByLocation([]source.Location) (map[source.Location]string, error) {
+func (r *testResolverMock) FileContentsByLocation(_ source.Location) (io.ReadCloser, error) {
+	return nil, fmt.Errorf("not implemented")
+}
+
+func (r *testResolverMock) MultipleFileContentsByLocation([]source.Location) (map[source.Location]io.ReadCloser, error) {
 	return r.contents, nil
 }
 
@@ -34,7 +38,7 @@ func (r *testResolverMock) FilesByPath(paths ...string) ([]source.Location, erro
 
 	for idx, p := range paths {
 		results[idx] = source.NewLocation(p)
-		r.contents[results[idx]] = fmt.Sprintf("%s file contents!", p)
+		r.contents[results[idx]] = ioutil.NopCloser(strings.NewReader(fmt.Sprintf("%s file contents!", p)))
 	}
 
 	return results, nil
@@ -43,7 +47,7 @@ func (r *testResolverMock) FilesByPath(paths ...string) ([]source.Location, erro
 func (r *testResolverMock) FilesByGlob(_ ...string) ([]source.Location, error) {
 	path := "/a-path.txt"
 	location := source.NewLocation(path)
-	r.contents[location] = fmt.Sprintf("%s file contents!", path)
+	r.contents[location] = ioutil.NopCloser(strings.NewReader(fmt.Sprintf("%s file contents!", path)))
 	return []source.Location{location}, nil
 }
 
@@ -97,7 +101,7 @@ func TestGenericCataloger(t *testing.T) {
 
 	for _, p := range actualPkgs {
 		ref := p.Locations[0]
-		exP, ok := expectedPkgs[ref.Path]
+		exP, ok := expectedPkgs[ref.RealPath]
 		if !ok {
 			t.Errorf("missing expected pkg: ref=%+v", ref)
 			continue

syft/cataloger/cpe.go

@@ -2,6 +2,7 @@ package cataloger
 
 import (
 	"fmt"
+	"sort"
 	"strings"
 
 	"github.com/anchore/syft/internal"
@@ -12,6 +13,17 @@ import (
 // this is functionally equivalent to "*" and consistent with no input given (thus easier to test)
 const any = ""
 
+func newCPE(product, vendor, version, targetSW string) wfn.Attributes {
+	cpe := *(wfn.NewAttributesWithAny())
+	cpe.Part = "a"
+	cpe.Product = product
+	cpe.Vendor = vendor
+	cpe.Version = version
+	cpe.TargetSW = targetSW
+
+	return cpe
+}
+
 // generatePackageCPEs Create a list of CPEs, trying to guess the vendor, product tuple and setting TargetSoftware if possible
 func generatePackageCPEs(p pkg.Package) []pkg.CPE {
 	targetSws := candidateTargetSoftwareAttrs(p)
@@ -31,18 +43,14 @@ func generatePackageCPEs(p pkg.Package) []pkg.CPE {
 				keys.Add(key)
 
 				// add a new entry...
-				candidateCpe := wfn.NewAttributesWithAny()
-				candidateCpe.Part = "a"
-				candidateCpe.Product = product
-				candidateCpe.Vendor = vendor
-				candidateCpe.Version = p.Version
-				candidateCpe.TargetSW = targetSw
-
-				cpes = append(cpes, *candidateCpe)
+				c := newCPE(product, vendor, p.Version, targetSw)
+				cpes = append(cpes, c)
 			}
 		}
 	}
 
+	sort.Sort(ByCPESpecificity(cpes))
+
 	return cpes
 }
 

syft/cataloger/cpe_specificity.go

@@ -0,0 +1,31 @@
+package cataloger
+
+import "github.com/facebookincubator/nvdtools/wfn"
+
+type ByCPESpecificity []wfn.Attributes
+
+// Implementing sort.Interface
+func (c ByCPESpecificity) Len() int      { return len(c) }
+func (c ByCPESpecificity) Swap(i, j int) { c[i], c[j] = c[j], c[i] }
+func (c ByCPESpecificity) Less(i, j int) bool {
+	return countSpecifiedFields(c[i]) > countSpecifiedFields(c[j])
+}
+
+func countSpecifiedFields(cpe wfn.Attributes) int {
+	checksForSpecifiedField := []func(cpe wfn.Attributes) bool{
+		func(cpe wfn.Attributes) bool { return cpe.Part != "" },
+		func(cpe wfn.Attributes) bool { return cpe.Product != "" },
+		func(cpe wfn.Attributes) bool { return cpe.Vendor != "" },
+		func(cpe wfn.Attributes) bool { return cpe.Version != "" },
+		func(cpe wfn.Attributes) bool { return cpe.TargetSW != "" },
+	}
+
+	count := 0
+	for _, fieldIsSpecified := range checksForSpecifiedField {
+		if fieldIsSpecified(cpe) {
+			count++
+		}
+	}
+
+	return count
+}

syft/cataloger/deb/cataloger.go

@@ -8,7 +8,6 @@ import (
 	"io"
 	"path"
 	"path/filepath"
-	"strings"
 
 	"github.com/anchore/syft/syft/pkg"
 	"github.com/anchore/syft/syft/source"
@@ -33,6 +32,7 @@ func (c *Cataloger) Name() string {
 }
 
 // Catalog is given an object to resolve file references and content, this function returns any discovered Packages after analyzing dpkg support files.
+// nolint:funlen
 func (c *Cataloger) Catalog(resolver source.Resolver) ([]pkg.Package, error) {
 	dbFileMatches, err := resolver.FilesByGlob(dpkgStatusGlob)
 	if err != nil {
@@ -46,9 +46,9 @@ func (c *Cataloger) Catalog(resolver source.Resolver) ([]pkg.Package, error) {
 			return nil, err
 		}
 
-		pkgs, err = parseDpkgStatus(strings.NewReader(dbContents))
+		pkgs, err = parseDpkgStatus(dbContents)
 		if err != nil {
-			return nil, fmt.Errorf("unable to catalog dpkg package=%+v: %w", dbLocation.Path, err)
+			return nil, fmt.Errorf("unable to catalog dpkg package=%+v: %w", dbLocation.RealPath, err)
 		}
 
 		md5ContentsByName, md5RefsByName, err := fetchMd5Contents(resolver, dbLocation, pkgs)
@@ -56,7 +56,7 @@ func (c *Cataloger) Catalog(resolver source.Resolver) ([]pkg.Package, error) {
 			return nil, fmt.Errorf("unable to find dpkg md5 contents: %w", err)
 		}
 
-		copyrightContentsByName, copyrightRefsByName, err := fetchCopyrightContents(resolver, dbLocation, pkgs)
+		copyrightContentsByName, copyrightLocationByName, err := fetchCopyrightContents(resolver, dbLocation, pkgs)
 		if err != nil {
 			return nil, fmt.Errorf("unable to find dpkg copyright contents: %w", err)
 		}
@@ -66,25 +66,31 @@ func (c *Cataloger) Catalog(resolver source.Resolver) ([]pkg.Package, error) {
 			p.FoundBy = c.Name()
 			p.Locations = []source.Location{dbLocation}
 
+			metadata := p.Metadata.(pkg.DpkgMetadata)
+
 			if md5Reader, ok := md5ContentsByName[md5Key(*p)]; ok {
 				// attach the file list
-				metadata := p.Metadata.(pkg.DpkgMetadata)
 				metadata.Files = parseDpkgMD5Info(md5Reader)
-				p.Metadata = metadata
 
 				// keep a record of the file where this was discovered
 				if ref, ok := md5RefsByName[md5Key(*p)]; ok {
 					p.Locations = append(p.Locations, ref)
 				}
+			} else {
+				// ensure the file list is an empty collection (not nil)
+				metadata.Files = make([]pkg.DpkgFileRecord, 0)
 			}
 
+			// persist alterations
+			p.Metadata = metadata
+
 			copyrightReader, ok := copyrightContentsByName[p.Name]
 			if ok {
 				// attach the licenses
 				p.Licenses = parseLicensesFromCopyright(copyrightReader)
 
 				// keep a record of the file where this was discovered
-				if ref, ok := copyrightRefsByName[p.Name]; ok {
+				if ref, ok := copyrightLocationByName[p.Name]; ok {
 					p.Locations = append(p.Locations, ref)
 				}
 			}
@@ -98,20 +104,17 @@ func fetchMd5Contents(resolver source.Resolver, dbLocation source.Location, pkgs
 
 	var md5FileMatches []source.Location
 	var nameByRef = make(map[source.Location]string)
-	parentPath := filepath.Dir(dbLocation.Path)
+	parentPath := filepath.Dir(dbLocation.RealPath)
 
 	for _, p := range pkgs {
 		// look for /var/lib/dpkg/info/NAME:ARCH.md5sums
 		name := md5Key(p)
-		md5sumPath := path.Join(parentPath, "info", name+md5sumsExt)
-		md5SumLocation := resolver.RelativeFileByPath(dbLocation, md5sumPath)
+		md5SumLocation := resolver.RelativeFileByPath(dbLocation, path.Join(parentPath, "info", name+md5sumsExt))
 
 		if md5SumLocation == nil {
 			// the most specific key did not work, fallback to just the name
 			// look for /var/lib/dpkg/info/NAME.md5sums
-			name := p.Name
-			md5sumPath := path.Join(parentPath, "info", name+md5sumsExt)
-			md5SumLocation = resolver.RelativeFileByPath(dbLocation, md5sumPath)
+			md5SumLocation = resolver.RelativeFileByPath(dbLocation, path.Join(parentPath, "info", p.Name+md5sumsExt))
 		}
 		// we should have at least one reference
 		if md5SumLocation != nil {
@@ -128,14 +131,14 @@ func fetchMd5Contents(resolver source.Resolver, dbLocation source.Location, pkgs
 
 	// organize content results and refs by a combination of name and architecture
 	var contentsByName = make(map[string]io.Reader)
-	var refsByName = make(map[string]source.Location)
+	var locationByName = make(map[string]source.Location)
 	for location, contents := range md5ContentsByLocation {
 		name := nameByRef[location]
-		contentsByName[name] = strings.NewReader(contents)
-		refsByName[name] = location
+		contentsByName[name] = contents
+		locationByName[name] = location
 	}
 
-	return contentsByName, refsByName, nil
+	return contentsByName, locationByName, nil
 }
 
 func fetchCopyrightContents(resolver source.Resolver, dbLocation source.Location, pkgs []pkg.Package) (map[string]io.Reader, map[string]source.Location, error) {
@@ -167,7 +170,7 @@ func fetchCopyrightContents(resolver source.Resolver, dbLocation source.Location
 	var refsByName = make(map[string]source.Location)
 	for location, contents := range copyrightContentsByLocation {
 		name := nameByLocation[location]
-		contentsByName[name] = strings.NewReader(contents)
+		contentsByName[name] = contents
 		refsByName[name] = location
 	}
 

syft/cataloger/deb/cataloger_test.go

@@ -54,7 +54,7 @@ func TestDpkgCataloger(t *testing.T) {
 			img, cleanup := imagetest.GetFixtureImage(t, "docker-archive", "image-dpkg")
 			defer cleanup()
 
-			s, err := source.NewFromImage(img, source.AllLayersScope, "")
+			s, err := source.NewFromImage(img, source.SquashedScope, "")
 			if err != nil {
 				t.Fatal(err)
 			}
@@ -79,7 +79,7 @@ func TestDpkgCataloger(t *testing.T) {
 				// we will test the sources separately
 				var sourcesList = make([]string, len(a.Locations))
 				for i, s := range a.Locations {
-					sourcesList[i] = s.Path
+					sourcesList[i] = s.RealPath
 				}
 				a.Locations = nil
 

syft/cataloger/deb/parse_copyright.go

@@ -3,6 +3,7 @@ package deb
 import (
 	"bufio"
 	"io"
+	"regexp"
 	"sort"
 	"strings"
 
@@ -11,14 +12,22 @@ import (
 
 // For more information see: https://www.debian.org/doc/packaging-manuals/copyright-format/1.0/#license-syntax
 
+var licensePattern = regexp.MustCompile(`^License: (?P<license>\S*)`)
+
 func parseLicensesFromCopyright(reader io.Reader) []string {
 	findings := internal.NewStringSet()
 	scanner := bufio.NewScanner(reader)
 
 	for scanner.Scan() {
 		line := scanner.Text()
-		if strings.HasPrefix(line, "License:") {
-			candidate := strings.Replace(line, "License:", "", 1)
+
+		matchesByGroup := internal.MatchCaptureGroups(licensePattern, line)
+		if len(matchesByGroup) > 0 {
+			candidate, ok := matchesByGroup["license"]
+			if !ok {
+				continue
+			}
+
 			candidate = strings.TrimSpace(candidate)
 			if strings.Contains(candidate, " or ") || strings.Contains(candidate, " and ") {
 				// this is a multi-license summary, ignore this as other recurrent license lines should cover this

syft/cataloger/deb/parse_copyright_test.go

@@ -24,6 +24,11 @@ func TestParseLicensesFromCopyright(t *testing.T) {
 			fixture:  "test-fixtures/copyright/libaudit-common",
 			expected: []string{"GPL-2", "LGPL-2.1"},
 		},
+		{
+			fixture: "test-fixtures/copyright/python",
+			// note: this should not capture #, Permission, This, see ... however it's not clear how to fix this (this is probably good enough)
+			expected: []string{"#", "Apache", "Apache-2", "Apache-2.0", "Expat", "ISC", "LGPL-2.1+", "PSF-2", "Permission", "Python", "This", "see"},
+		},
 	}
 
 	for _, test := range tests {

syft/cataloger/deb/parse_dpkg_info_files.go

@@ -9,7 +9,8 @@ import (
 )
 
 func parseDpkgMD5Info(reader io.Reader) []pkg.DpkgFileRecord {
-	var findings []pkg.DpkgFileRecord
+	// we must preallocate to ensure the resulting struct does not have null
+	var findings = make([]pkg.DpkgFileRecord, 0)
 	scanner := bufio.NewScanner(reader)
 
 	for scanner.Scan() {

syft/cataloger/deb/parse_dpkg_status.go

@@ -5,9 +5,12 @@ import (
 	"errors"
 	"fmt"
 	"io"
+	"regexp"
 	"strconv"
 	"strings"
 
+	"github.com/anchore/syft/internal"
+
 	"github.com/anchore/syft/syft/pkg"
 	"github.com/mitchellh/mapstructure"
 )
@@ -45,6 +48,7 @@ func parseDpkgStatus(reader io.Reader) ([]pkg.Package, error) {
 }
 
 // parseDpkgStatusEntry returns an individual Dpkg entry, or returns errEndOfPackages if there are no more packages to parse from the reader.
+// nolint:funlen
 func parseDpkgStatusEntry(reader *bufio.Reader) (entry pkg.DpkgMetadata, err error) {
 	dpkgFields := make(map[string]interface{})
 	var retErr error
@@ -105,9 +109,29 @@ func parseDpkgStatusEntry(reader *bufio.Reader) (entry pkg.DpkgMetadata, err err
 		return pkg.DpkgMetadata{}, err
 	}
 
+	name, version := extractSourceVersion(entry.Source)
+	if version != "" {
+		entry.SourceVersion = version
+		entry.Source = name
+	}
+
 	return entry, retErr
 }
 
+// match examples:
+// "a-thing (1.2.3)"     name="a-thing" version="1.2.3"
+// "a-thing"             name="a-thing" version=""
+// ""                    name=""        version=""
+var sourceRegexp = regexp.MustCompile(`(?P<name>\S+)( \((?P<version>.*)\))?`)
+
+// If the source entry string is of the form "<name> (<version>)" then parse and return the components, if
+// of the "<name>" form, then return name and nil
+func extractSourceVersion(source string) (string, string) {
+	// special handling for the Source field since it has formatted data
+	match := internal.MatchCaptureGroups(sourceRegexp, source)
+	return match["name"], match["version"]
+}
+
 // handleNewKeyValue parse a new key-value pair from the given unprocessed line
 func handleNewKeyValue(line string) (string, interface{}, error) {
 	if i := strings.Index(line, ":"); i > 0 {

syft/cataloger/deb/parse_dpkg_status_test.go

@@ -115,3 +115,43 @@ func TestMultiplePackages(t *testing.T) {
 		})
 	}
 }
+
+func TestSourceVersionExtract(t *testing.T) {
+
+	tests := []struct {
+		name     string
+		input    string
+		expected []string
+	}{
+		{
+			name:     "name and version",
+			input:    "test (1.2.3)",
+			expected: []string{"test", "1.2.3"},
+		},
+		{
+			name:     "only name",
+			input:    "test",
+			expected: []string{"test", ""},
+		},
+		{
+			name:     "empty",
+			input:    "",
+			expected: []string{"", ""},
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			name, version := extractSourceVersion(test.input)
+
+			if name != test.expected[0] {
+				t.Errorf("mismatch name for %q : %q!=%q", test.input, name, test.expected[0])
+			}
+
+			if version != test.expected[1] {
+				t.Errorf("mismatch version for %q : %q!=%q", test.input, version, test.expected[1])
+			}
+
+		})
+	}
+}

syft/cataloger/deb/test-fixtures/copyright/python

@@ -0,0 +1,958 @@
+This package was put together by Klee Dienes <klee@debian.org> from 
+sources from ftp.python.org:/pub/python, based on the Debianization by 
+the previous maintainers Bernd S. Brentrup <bsb@uni-muenster.de> and 
+Bruce Perens. Current maintainer is Matthias Klose <doko@debian.org>. 
+
+It was downloaded from http://python.org/
+
+Copyright:
+
+Upstream Author: Guido van Rossum <guido@cwi.nl> and others.
+
+License:
+
+The following text includes the Python license and licenses and
+acknowledgements for incorporated software. The licenses can be read
+in the HTML and texinfo versions of the documentation as well, after
+installing the pythonx.y-doc package. Licenses for files not licensed
+under the Python Licenses are found at the end of this file.
+
+
+Python License
+==============
+
+A. HISTORY OF THE SOFTWARE
+==========================
+
+Python was created in the early 1990s by Guido van Rossum at Stichting
+Mathematisch Centrum (CWI, see http://www.cwi.nl) in the Netherlands
+as a successor of a language called ABC.  Guido remains Python's
+principal author, although it includes many contributions from others.
+
+In 1995, Guido continued his work on Python at the Corporation for
+National Research Initiatives (CNRI, see http://www.cnri.reston.va.us)
+in Reston, Virginia where he released several versions of the
+software.
+
+In May 2000, Guido and the Python core development team moved to
+BeOpen.com to form the BeOpen PythonLabs team.  In October of the same
+year, the PythonLabs team moved to Digital Creations (now Zope
+Corporation, see http://www.zope.com).  In 2001, the Python Software
+Foundation (PSF, see http://www.python.org/psf/) was formed, a
+non-profit organization created specifically to own Python-related
+Intellectual Property.  Zope Corporation is a sponsoring member of
+the PSF.
+
+All Python releases are Open Source (see http://www.opensource.org for
+the Open Source Definition).  Historically, most, but not all, Python
+releases have also been GPL-compatible; the table below summarizes
+the various releases.
+
+    Release         Derived     Year        Owner       GPL-
+                    from                                compatible? (1)
+
+    0.9.0 thru 1.2              1991-1995   CWI         yes
+    1.3 thru 1.5.2  1.2         1995-1999   CNRI        yes
+    1.6             1.5.2       2000        CNRI        no
+    2.0             1.6         2000        BeOpen.com  no
+    1.6.1           1.6         2001        CNRI        yes (2)
+    2.1             2.0+1.6.1   2001        PSF         no
+    2.0.1           2.0+1.6.1   2001        PSF         yes
+    2.1.1           2.1+2.0.1   2001        PSF         yes
+    2.2             2.1.1       2001        PSF         yes
+    2.1.2           2.1.1       2002        PSF         yes
+    2.1.3           2.1.2       2002        PSF         yes
+    2.2.1           2.2         2002        PSF         yes
+    2.2.2           2.2.1       2002        PSF         yes
+    2.2.3           2.2.2       2003        PSF         yes
+    2.3             2.2.2       2002-2003   PSF         yes
+    2.3.1           2.3         2002-2003   PSF         yes
+    2.3.2           2.3.1       2002-2003   PSF         yes
+    2.3.3           2.3.2       2002-2003   PSF         yes
+    2.3.4           2.3.3       2004        PSF         yes
+    2.3.5           2.3.4       2005        PSF         yes
+    2.4             2.3         2004        PSF         yes
+    2.4.1           2.4         2005        PSF         yes
+    2.4.2           2.4.1       2005        PSF         yes
+    2.4.3           2.4.2       2006        PSF         yes
+    2.5             2.4         2006        PSF         yes
+    2.5.1           2.5         2007        PSF         yes
+    2.5.2           2.5.1       2008        PSF         yes
+    2.5.3           2.5.2       2008        PSF         yes
+    2.6             2.5         2008        PSF         yes
+    2.6.1           2.6         2008        PSF         yes
+    2.6.2           2.6.1       2009        PSF         yes
+    2.6.3           2.6.2       2009        PSF         yes
+    2.6.4           2.6.3       2009        PSF         yes
+    2.6.5           2.6.4       2010        PSF         yes
+    2.7             2.6         2010        PSF         yes
+
+Footnotes:
+
+(1) GPL-compatible doesn't mean that we're distributing Python under
+    the GPL.  All Python licenses, unlike the GPL, let you distribute
+    a modified version without making your changes open source.  The
+    GPL-compatible licenses make it possible to combine Python with
+    other software that is released under the GPL; the others don't.
+
+(2) According to Richard Stallman, 1.6.1 is not GPL-compatible,
+    because its license has a choice of law clause.  According to
+    CNRI, however, Stallman's lawyer has told CNRI's lawyer that 1.6.1
+    is "not incompatible" with the GPL.
+
+Thanks to the many outside volunteers who have worked under Guido's
+direction to make these releases possible.
+
+
+B. TERMS AND CONDITIONS FOR ACCESSING OR OTHERWISE USING PYTHON
+===============================================================
+
+PYTHON SOFTWARE FOUNDATION LICENSE VERSION 2
+--------------------------------------------
+
+1. This LICENSE AGREEMENT is between the Python Software Foundation
+("PSF"), and the Individual or Organization ("Licensee") accessing and
+otherwise using this software ("Python") in source or binary form and
+its associated documentation.
+
+2. Subject to the terms and conditions of this License Agreement, PSF
+hereby grants Licensee a nonexclusive, royalty-free, world-wide
+license to reproduce, analyze, test, perform and/or display publicly,
+prepare derivative works, distribute, and otherwise use Python
+alone or in any derivative version, provided, however, that PSF's
+License Agreement and PSF's notice of copyright, i.e., "Copyright (c)
+2001, 2002, 2003, 2004, 2005, 2006, 2007 Python Software Foundation; 
+All Rights Reserved" are retained in Python alone or in any derivative 
+version prepared by Licensee.
+
+3. In the event Licensee prepares a derivative work that is based on
+or incorporates Python or any part thereof, and wants to make
+the derivative work available to others as provided herein, then
+Licensee hereby agrees to include in any such work a brief summary of
+the changes made to Python.
+
+4. PSF is making Python available to Licensee on an "AS IS"
+basis.  PSF MAKES NO REPRESENTATIONS OR WARRANTIES, EXPRESS OR
+IMPLIED.  BY WAY OF EXAMPLE, BUT NOT LIMITATION, PSF MAKES NO AND
+DISCLAIMS ANY REPRESENTATION OR WARRANTY OF MERCHANTABILITY OR FITNESS
+FOR ANY PARTICULAR PURPOSE OR THAT THE USE OF PYTHON WILL NOT
+INFRINGE ANY THIRD PARTY RIGHTS.
+
+5. PSF SHALL NOT BE LIABLE TO LICENSEE OR ANY OTHER USERS OF PYTHON
+FOR ANY INCIDENTAL, SPECIAL, OR CONSEQUENTIAL DAMAGES OR LOSS AS
+A RESULT OF MODIFYING, DISTRIBUTING, OR OTHERWISE USING PYTHON,
+OR ANY DERIVATIVE THEREOF, EVEN IF ADVISED OF THE POSSIBILITY THEREOF.
+
+6. This License Agreement will automatically terminate upon a material
+breach of its terms and conditions.
+
+7. Nothing in this License Agreement shall be deemed to create any
+relationship of agency, partnership, or joint venture between PSF and
+Licensee.  This License Agreement does not grant permission to use PSF
+trademarks or trade name in a trademark sense to endorse or promote
+products or services of Licensee, or any third party.
+
+8. By copying, installing or otherwise using Python, Licensee
+agrees to be bound by the terms and conditions of this License
+Agreement.
+
+
+BEOPEN.COM LICENSE AGREEMENT FOR PYTHON 2.0
+-------------------------------------------
+
+BEOPEN PYTHON OPEN SOURCE LICENSE AGREEMENT VERSION 1
+
+1. This LICENSE AGREEMENT is between BeOpen.com ("BeOpen"), having an
+office at 160 Saratoga Avenue, Santa Clara, CA 95051, and the
+Individual or Organization ("Licensee") accessing and otherwise using
+this software in source or binary form and its associated
+documentation ("the Software").
+
+2. Subject to the terms and conditions of this BeOpen Python License
+Agreement, BeOpen hereby grants Licensee a non-exclusive,
+royalty-free, world-wide license to reproduce, analyze, test, perform
+and/or display publicly, prepare derivative works, distribute, and
+otherwise use the Software alone or in any derivative version,
+provided, however, that the BeOpen Python License is retained in the
+Software, alone or in any derivative version prepared by Licensee.
+
+3. BeOpen is making the Software available to Licensee on an "AS IS"
+basis.  BEOPEN MAKES NO REPRESENTATIONS OR WARRANTIES, EXPRESS OR
+IMPLIED.  BY WAY OF EXAMPLE, BUT NOT LIMITATION, BEOPEN MAKES NO AND
+DISCLAIMS ANY REPRESENTATION OR WARRANTY OF MERCHANTABILITY OR FITNESS
+FOR ANY PARTICULAR PURPOSE OR THAT THE USE OF THE SOFTWARE WILL NOT
+INFRINGE ANY THIRD PARTY RIGHTS.
+
+4. BEOPEN SHALL NOT BE LIABLE TO LICENSEE OR ANY OTHER USERS OF THE
+SOFTWARE FOR ANY INCIDENTAL, SPECIAL, OR CONSEQUENTIAL DAMAGES OR LOSS
+AS A RESULT OF USING, MODIFYING OR DISTRIBUTING THE SOFTWARE, OR ANY
+DERIVATIVE THEREOF, EVEN IF ADVISED OF THE POSSIBILITY THEREOF.
+
+5. This License Agreement will automatically terminate upon a material
+breach of its terms and conditions.
+
+6. This License Agreement shall be governed by and interpreted in all
+respects by the law of the State of California, excluding conflict of
+law provisions.  Nothing in this License Agreement shall be deemed to
+create any relationship of agency, partnership, or joint venture
+between BeOpen and Licensee.  This License Agreement does not grant
+permission to use BeOpen trademarks or trade names in a trademark
+sense to endorse or promote products or services of Licensee, or any
+third party.  As an exception, the "BeOpen Python" logos available at
+http://www.pythonlabs.com/logos.html may be used according to the
+permissions granted on that web page.
+
+7. By copying, installing or otherwise using the software, Licensee
+agrees to be bound by the terms and conditions of this License
+Agreement.
+
+
+CNRI LICENSE AGREEMENT FOR PYTHON 1.6.1
+---------------------------------------
+
+1. This LICENSE AGREEMENT is between the Corporation for National
+Research Initiatives, having an office at 1895 Preston White Drive,
+Reston, VA 20191 ("CNRI"), and the Individual or Organization
+("Licensee") accessing and otherwise using Python 1.6.1 software in
+source or binary form and its associated documentation.
+
+2. Subject to the terms and conditions of this License Agreement, CNRI
+hereby grants Licensee a nonexclusive, royalty-free, world-wide
+license to reproduce, analyze, test, perform and/or display publicly,
+prepare derivative works, distribute, and otherwise use Python 1.6.1
+alone or in any derivative version, provided, however, that CNRI's
+License Agreement and CNRI's notice of copyright, i.e., "Copyright (c)
+1995-2001 Corporation for National Research Initiatives; All Rights
+Reserved" are retained in Python 1.6.1 alone or in any derivative
+version prepared by Licensee.  Alternately, in lieu of CNRI's License
+Agreement, Licensee may substitute the following text (omitting the
+quotes): "Python 1.6.1 is made available subject to the terms and
+conditions in CNRI's License Agreement.  This Agreement together with
+Python 1.6.1 may be located on the Internet using the following
+unique, persistent identifier (known as a handle): 1895.22/1013.  This
+Agreement may also be obtained from a proxy server on the Internet
+using the following URL: http://hdl.handle.net/1895.22/1013".
+
+3. In the event Licensee prepares a derivative work that is based on
+or incorporates Python 1.6.1 or any part thereof, and wants to make
+the derivative work available to others as provided herein, then
+Licensee hereby agrees to include in any such work a brief summary of
+the changes made to Python 1.6.1.
+
+4. CNRI is making Python 1.6.1 available to Licensee on an "AS IS"
+basis.  CNRI MAKES NO REPRESENTATIONS OR WARRANTIES, EXPRESS OR
+IMPLIED.  BY WAY OF EXAMPLE, BUT NOT LIMITATION, CNRI MAKES NO AND
+DISCLAIMS ANY REPRESENTATION OR WARRANTY OF MERCHANTABILITY OR FITNESS
+FOR ANY PARTICULAR PURPOSE OR THAT THE USE OF PYTHON 1.6.1 WILL NOT
+INFRINGE ANY THIRD PARTY RIGHTS.
+
+5. CNRI SHALL NOT BE LIABLE TO LICENSEE OR ANY OTHER USERS OF PYTHON
+1.6.1 FOR ANY INCIDENTAL, SPECIAL, OR CONSEQUENTIAL DAMAGES OR LOSS AS
+A RESULT OF MODIFYING, DISTRIBUTING, OR OTHERWISE USING PYTHON 1.6.1,
+OR ANY DERIVATIVE THEREOF, EVEN IF ADVISED OF THE POSSIBILITY THEREOF.
+
+6. This License Agreement will automatically terminate upon a material
+breach of its terms and conditions.
+
+7. This License Agreement shall be governed by the federal
+intellectual property law of the United States, including without
+limitation the federal copyright law, and, to the extent such
+U.S. federal law does not apply, by the law of the Commonwealth of
+Virginia, excluding Virginia's conflict of law provisions.
+Notwithstanding the foregoing, with regard to derivative works based
+on Python 1.6.1 that incorporate non-separable material that was
+previously distributed under the GNU General Public License (GPL), the
+law of the Commonwealth of Virginia shall govern this License
+Agreement only as to issues arising under or with respect to
+Paragraphs 4, 5, and 7 of this License Agreement.  Nothing in this
+License Agreement shall be deemed to create any relationship of
+agency, partnership, or joint venture between CNRI and Licensee.  This
+License Agreement does not grant permission to use CNRI trademarks or
+trade name in a trademark sense to endorse or promote products or
+services of Licensee, or any third party.
+
+8. By clicking on the "ACCEPT" button where indicated, or by copying,
+installing or otherwise using Python 1.6.1, Licensee agrees to be
+bound by the terms and conditions of this License Agreement.
+
+        ACCEPT
+
+
+CWI LICENSE AGREEMENT FOR PYTHON 0.9.0 THROUGH 1.2
+--------------------------------------------------
+
+Copyright (c) 1991 - 1995, Stichting Mathematisch Centrum Amsterdam,
+The Netherlands.  All rights reserved.
+
+Permission to use, copy, modify, and distribute this software and its
+documentation for any purpose and without fee is hereby granted,
+provided that the above copyright notice appear in all copies and that
+both that copyright notice and this permission notice appear in
+supporting documentation, and that the name of Stichting Mathematisch
+Centrum or CWI not be used in advertising or publicity pertaining to
+distribution of the software without specific, written prior
+permission.
+
+STICHTING MATHEMATISCH CENTRUM DISCLAIMS ALL WARRANTIES WITH REGARD TO
+THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND
+FITNESS, IN NO EVENT SHALL STICHTING MATHEMATISCH CENTRUM BE LIABLE
+FOR ANY SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
+ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT
+OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+
+
+Licenses and Acknowledgements for Incorporated Software
+=======================================================
+
+Mersenne Twister
+----------------
+
+The `_random' module includes code based on a download from
+`http://www.math.keio.ac.jp/~matumoto/MT2002/emt19937ar.html'.  The
+following are the verbatim comments from the original code:
+
+     A C-program for MT19937, with initialization improved 2002/1/26.
+     Coded by Takuji Nishimura and Makoto Matsumoto.
+
+     Before using, initialize the state by using init_genrand(seed)
+     or init_by_array(init_key, key_length).
+
+     Copyright (C) 1997 - 2002, Makoto Matsumoto and Takuji Nishimura,
+     All rights reserved.
+
+     Redistribution and use in source and binary forms, with or without
+     modification, are permitted provided that the following conditions
+     are met:
+
+      1. Redistributions of source code must retain the above copyright
+         notice, this list of conditions and the following disclaimer.
+
+      2. Redistributions in binary form must reproduce the above copyright
+         notice, this list of conditions and the following disclaimer in the
+         documentation and/or other materials provided with the distribution.
+
+      3. The names of its contributors may not be used to endorse or promote
+         products derived from this software without specific prior written
+         permission.
+
+     THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+     "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+     LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+     A PARTICULAR PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE COPYRIGHT
+     OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+     SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED
+     TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+     PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
+     LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
+     NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+     SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+     Any feedback is very welcome.
+     http://www.math.keio.ac.jp/matumoto/emt.html
+     email: matumoto@math.keio.ac.jp
+
+
+Sockets
+-------
+
+The `socket' module uses the functions, `getaddrinfo', and
+`getnameinfo', which are coded in separate source files from the WIDE
+Project, `http://www.wide.ad.jp/about/index.html'.
+
+     Copyright (C) 1995, 1996, 1997, and 1998 WIDE Project.
+     All rights reserved.
+
+     Redistribution and use in source and binary forms, with or without
+     modification, are permitted provided that the following conditions
+     are met:
+     1. Redistributions of source code must retain the above copyright
+        notice, this list of conditions and the following disclaimer.
+     2. Redistributions in binary form must reproduce the above copyright
+        notice, this list of conditions and the following disclaimer in the
+        documentation and/or other materials provided with the distribution.
+     3. Neither the name of the project nor the names of its contributors
+        may be used to endorse or promote products derived from this software
+        without specific prior written permission.
+
+     THIS SOFTWARE IS PROVIDED BY THE PROJECT AND CONTRIBUTORS ``AS IS'' AND
+     GAI_ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+     IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+     ARE DISCLAIMED.  IN NO EVENT SHALL THE PROJECT OR CONTRIBUTORS BE LIABLE
+     FOR GAI_ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+     CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+     SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+     INTERRUPTION) HOWEVER CAUSED AND ON GAI_ANY THEORY OF LIABILITY, WHETHER
+     IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+     ARISING IN GAI_ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED
+     OF THE POSSIBILITY OF SUCH DAMAGE.
+
+
+Floating point exception control
+--------------------------------
+
+The source for the `fpectl' module includes the following notice:
+
+     ---------------------------------------------------------------------  
+    /                       Copyright (c) 1996.                           \ 
+   |          The Regents of the University of California.                 |
+   |                        All rights reserved.                           |
+   |                                                                       |
+   |   Permission to use, copy, modify, and distribute this software for   |
+   |   any purpose without fee is hereby granted, provided that this en-   |
+   |   tire notice is included in all copies of any software which is or   |
+   |   includes  a  copy  or  modification  of  this software and in all   |
+   |   copies of the supporting documentation for such software.           |
+   |                                                                       |
+   |   This  work was produced at the University of California, Lawrence   |
+   |   Livermore National Laboratory under  contract  no.  W-7405-ENG-48   |
+   |   between  the  U.S.  Department  of  Energy and The Regents of the   |
+   |   University of California for the operation of UC LLNL.              |
+   |                                                                       |
+   |                              DISCLAIMER                               |
+   |                                                                       |
+   |   This  software was prepared as an account of work sponsored by an   |
+   |   agency of the United States Government. Neither the United States   |
+   |   Government  nor the University of California nor any of their em-   |
+   |   ployees, makes any warranty, express or implied, or  assumes  any   |
+   |   liability  or  responsibility  for the accuracy, completeness, or   |
+   |   usefulness of any information,  apparatus,  product,  or  process   |
+   |   disclosed,   or  represents  that  its  use  would  not  infringe   |
+   |   privately-owned rights. Reference herein to any specific  commer-   |
+   |   cial  products,  process,  or  service  by trade name, trademark,   |
+   |   manufacturer, or otherwise, does not  necessarily  constitute  or   |
+   |   imply  its endorsement, recommendation, or favoring by the United   |
+   |   States Government or the University of California. The views  and   |
+   |   opinions  of authors expressed herein do not necessarily state or   |
+   |   reflect those of the United States Government or  the  University   |
+   |   of  California,  and shall not be used for advertising or product   |
+    \  endorsement purposes.                                              / 
+     ---------------------------------------------------------------------
+
+
+Cookie management
+-----------------
+
+The `Cookie' module contains the following notice:
+
+      Copyright 2000 by Timothy O'Malley <timo@alum.mit.edu>
+
+                     All Rights Reserved
+
+      Permission to use, copy, modify, and distribute this software
+      and its documentation for any purpose and without fee is hereby
+      granted, provided that the above copyright notice appear in all
+      copies and that both that copyright notice and this permission
+      notice appear in supporting documentation, and that the name of
+      Timothy O'Malley  not be used in advertising or publicity
+      pertaining to distribution of the software without specific, written
+      prior permission.
+
+      Timothy O'Malley DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS
+      SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
+      AND FITNESS, IN NO EVENT SHALL Timothy O'Malley BE LIABLE FOR
+      ANY SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+      WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS,
+      WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS
+      ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
+      PERFORMANCE OF THIS SOFTWARE.
+
+
+Execution tracing
+-----------------
+
+The `trace' module contains the following notice:
+
+      portions copyright 2001, Autonomous Zones Industries, Inc., all rights...
+      err...  reserved and offered to the public under the terms of the
+      Python 2.2 license.
+      Author: Zooko O'Whielacronx
+      http://zooko.com/
+      mailto:zooko@zooko.com
+
+      Copyright 2000, Mojam Media, Inc., all rights reserved.
+      Author: Skip Montanaro
+
+      Copyright 1999, Bioreason, Inc., all rights reserved.
+      Author: Andrew Dalke
+
+      Copyright 1995-1997, Automatrix, Inc., all rights reserved.
+      Author: Skip Montanaro
+
+      Copyright 1991-1995, Stichting Mathematisch Centrum, all rights reserved.
+
+      Permission to use, copy, modify, and distribute this Python software and
+      its associated documentation for any purpose without fee is hereby
+      granted, provided that the above copyright notice appears in all copies,
+      and that both that copyright notice and this permission notice appear in
+      supporting documentation, and that the name of neither Automatrix,
+      Bioreason or Mojam Media be used in advertising or publicity pertaining
+      to distribution of the software without specific, written prior
+      permission.
+
+
+UUencode and UUdecode functions
+-------------------------------
+
+The `uu' module contains the following notice:
+
+      Copyright 1994 by Lance Ellinghouse
+      Cathedral City, California Republic, United States of America.
+                             All Rights Reserved
+      Permission to use, copy, modify, and distribute this software and its
+      documentation for any purpose and without fee is hereby granted,
+      provided that the above copyright notice appear in all copies and that
+      both that copyright notice and this permission notice appear in
+      supporting documentation, and that the name of Lance Ellinghouse
+      not be used in advertising or publicity pertaining to distribution
+      of the software without specific, written prior permission.
+      LANCE ELLINGHOUSE DISCLAIMS ALL WARRANTIES WITH REGARD TO
+      THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND
+      FITNESS, IN NO EVENT SHALL LANCE ELLINGHOUSE CENTRUM BE LIABLE
+      FOR ANY SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+      WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
+      ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT
+      OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+
+      Modified by Jack Jansen, CWI, July 1995:
+      - Use binascii module to do the actual line-by-line conversion
+        between ascii and binary. This results in a 1000-fold speedup. The C
+        version is still 5 times faster, though.
+      - Arguments more compliant with python standard
+
+
+XML Remote Procedure Calls
+--------------------------
+
+The `xmlrpclib' module contains the following notice:
+
+          The XML-RPC client interface is
+
+      Copyright (c) 1999-2002 by Secret Labs AB
+      Copyright (c) 1999-2002 by Fredrik Lundh
+
+      By obtaining, using, and/or copying this software and/or its
+      associated documentation, you agree that you have read, understood,
+      and will comply with the following terms and conditions:
+
+      Permission to use, copy, modify, and distribute this software and
+      its associated documentation for any purpose and without fee is
+      hereby granted, provided that the above copyright notice appears in
+      all copies, and that both that copyright notice and this permission
+      notice appear in supporting documentation, and that the name of
+      Secret Labs AB or the author not be used in advertising or publicity
+      pertaining to distribution of the software without specific, written
+      prior permission.
+
+      SECRET LABS AB AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH REGARD
+      TO THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANT-
+      ABILITY AND FITNESS.  IN NO EVENT SHALL SECRET LABS AB OR THE AUTHOR
+      BE LIABLE FOR ANY SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY
+      DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS,
+      WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS
+      ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE
+      OF THIS SOFTWARE.
+
+Licenses for Software linked to
+===============================
+
+Note that the choice of GPL compatibility outlined above doesn't extend
+to modules linked to particular libraries, since they change the
+effective License of the module binary.
+
+
+GNU Readline
+------------
+
+The 'readline' module makes use of GNU Readline.
+
+      The GNU Readline Library is free software; you can redistribute it
+      and/or modify it under the terms of the GNU General Public License as
+      published by the Free Software Foundation; either version 2, or (at
+      your option) any later version.
+
+      On Debian systems, you can find the complete statement in
+      /usr/share/doc/readline-common/copyright'. A copy of the GNU General
+      Public License is available in /usr/share/common-licenses/GPL-2'.
+
+
+OpenSSL
+-------
+
+The '_ssl' module makes use of OpenSSL.
+
+      The OpenSSL toolkit stays under a dual license, i.e. both the
+      conditions of the OpenSSL License and the original SSLeay license
+      apply to the toolkit. Actually both licenses are BSD-style Open
+      Source licenses. Note that both licenses are incompatible with
+      the GPL.
+
+      On Debian systems, you can find the complete license text in
+      /usr/share/doc/openssl/copyright'.
+
+
+Files with other licenses than the Python License
+-------------------------------------------------
+
+Files: Lib/profile.py Lib/pstats.py
+Copyright: Disney Enterprises, Inc.  All Rights Reserved.
+License: # Licensed to PSF under a Contributor Agreement
+  Licensed under the Apache License, Version 2.0 (the "License");
+  you may not use this file except in compliance with the License.
+  You may obtain a copy of the License at
+
+  http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
+  either express or implied.  See the License for the specific language
+  overning permissions and limitations under the License.
+
+  On Debian systems, the Apache 2.0 license can be found in
+  /usr/share/common-licenses/Apache-2.0.
+
+Files: Modules/zlib/*
+Copyright: (C) 1995-2010 Jean-loup Gailly and Mark Adler
+License: This software is provided 'as-is', without any express or implied
+  warranty.  In no event will the authors be held liable for any damages
+  arising from the use of this software.
+
+  Permission is granted to anyone to use this software for any purpose,
+  including commercial applications, and to alter it and redistribute it
+  freely, subject to the following restrictions:
+
+  1. The origin of this software must not be misrepresented; you must not
+     claim that you wrote the original software. If you use this software
+     in a product, an acknowledgment in the product documentation would be
+     appreciated but is not required.
+  2. Altered source versions must be plainly marked as such, and must not be
+     misrepresented as being the original software.
+  3. This notice may not be removed or altered from any source distribution.
+
+  Jean-loup Gailly        Mark Adler
+  jloup@gzip.org          madler@alumni.caltech.edu
+
+ If you use the zlib library in a product, we would appreciate *not* receiving
+ lengthy legal documents to sign.  The sources are provided for free but without
+ warranty of any kind.  The library has been entirely written by Jean-loup
+ Gailly and Mark Adler; it does not include third-party code.
+
+Files: Modules/_ctypes/libffi/*
+Copyright: Copyright (C) 1996-2009 Red Hat, Inc and others.
+License: Permission is hereby granted, free of charge, to any person obtaining
+   a copy of this software and associated documentation files (the
+   ``Software''), to deal in the Software without restriction, including
+   without limitation the rights to use, copy, modify, merge, publish,
+   distribute, sublicense, and/or sell copies of the Software, and to
+   permit persons to whom the Software is furnished to do so, subject to
+   the following conditions:
+
+   The above copyright notice and this permission notice shall be included
+   in all copies or substantial portions of the Software.
+
+   THE SOFTWARE IS PROVIDED ``AS IS'', WITHOUT WARRANTY OF ANY KIND,
+   EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+   MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+   NONINFRINGEMENT.  IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+   HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+   WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+   OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+   DEALINGS IN THE SOFTWARE.
+
+   Documentation:
+   Permission is granted to copy, distribute and/or modify this document
+   under the terms of the GNU General Public License as published by the
+   Free Software Foundation; either version 2, or (at your option) any
+   later version.  A copy of the license is included in the
+   section entitled ``GNU General Public License''.
+
+Files: Modules/expat/*
+Copyright: Copyright (c) 1998, 1999, 2000 Thai Open Source Software Center Ltd
+  and Clark Cooper
+  Copyright (c) 2001, 2002, 2003, 2004, 2005, 2006 Expat maintainers
+License: Permission is hereby granted, free of charge, to any person obtaining
+  a copy of this software and associated documentation files (the
+  "Software"), to deal in the Software without restriction, including
+  without limitation the rights to use, copy, modify, merge, publish,
+  distribute, sublicense, and/or sell copies of the Software, and to
+  permit persons to whom the Software is furnished to do so, subject to
+  the following conditions:
+ 
+  The above copyright notice and this permission notice shall be included
+  in all copies or substantial portions of the Software.
+ 
+  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+  EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+  MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+  IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
+  CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
+  TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+  SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+Files: Misc/python-mode.el
+Copyright: Copyright (C) 1992,1993,1994  Tim Peters
+License: This software is provided as-is, without express or implied
+        warranty.  Permission to use, copy, modify, distribute or sell this
+        software, without fee, for any purpose and by any individual or
+        organization, is hereby granted, provided that the above copyright
+        notice and this paragraph appear in all copies.
+
+Files: PC/_subprocess.c
+Copyright: Copyright (c) 2004 by Fredrik Lundh <fredrik@pythonware.com>
+        Copyright (c) 2004 by Secret Labs AB, http://www.pythonware.com
+        Copyright (c) 2004 by Peter Astrand <astrand@lysator.liu.se>
+License:
+ * Permission to use, copy, modify, and distribute this software and
+ * its associated documentation for any purpose and without fee is
+ * hereby granted, provided that the above copyright notice appears in
+ * all copies, and that both that copyright notice and this permission
+ * notice appear in supporting documentation, and that the name of the
+ * authors not be used in advertising or publicity pertaining to
+ * distribution of the software without specific, written prior
+ * permission.
+ *
+ * THE AUTHORS DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE,
+ * INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS.
+ * IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY SPECIAL, INDIRECT OR
+ * CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS
+ * OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT,
+ * NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION
+ * WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+
+Files: PC/winsound.c
+Copyright: Copyright (c) 1999 Toby Dickenson
+License:  * Permission to use this software in any way is granted without
+ * fee, provided that the copyright notice above appears in all
+ * copies. This software is provided "as is" without any warranty.
+ */
+
+/* Modified by Guido van Rossum */
+/* Beep added by Mark Hammond */
+/* Win9X Beep and platform identification added by Uncle Timmy */
+
+Files: Lib/ensurepip/_bundled/setuptools-*.whl
+License: see above, some license as Python.
+
+Files: Lib/ensurepip/_bundled/pip-*.whl
+Copyright: Copyright © 2008-2013 The pip developers:
+    Alex Grönholm
+    Alex Morega
+    Alexandre Conrad
+    Andrey Bulgakov
+    Antti Kaihola
+    Armin Ronacher
+    Aziz Köksal
+    Ben Rosser
+    Brian Rosner
+    Carl Meyer
+    Chris McDonough
+    Christian Oudard
+    Clay McClure
+    Cody Soyland
+    Daniel Holth
+    Dave Abrahams
+    David (d1b)
+    Dmitry Gladkov
+    Donald Stufft
+    Francesco
+    Geoffrey Lehée
+    Georgi Valkov
+    Hugo Lopes Tavares
+    Ian Bicking
+    Igor Sobreira
+    Ionel Maries Cristian
+    Jakub Vysoky
+    James Cleveland
+    Jannis Leidel
+    Jay Graves
+    John-Scott Atlakson
+    Jon Parise
+    Jonas Nockert
+    Josh Bronson
+    Kamal Bin Mustafa
+    Kelsey Hightower
+    Kenneth Belitzky
+    Kumar McMillan
+    Luke Macken
+    Masklinn
+    Marc Abramowitz
+    Marcus Smith
+    Markus Hametner
+    Matt Maker
+    Maxime R.
+    Miguel Araujo
+    Nick Stenning
+    Nowell Strite
+    Oliver Tonnhofer
+    Olivier Girardot
+    Patrick Jenkins
+    Paul Moore
+    Paul Nasrat
+    Paul Oswald
+    Paul van der Linden
+    Peter Waller
+    Phil Whelan
+    Piet Delport
+    Przemek Wrzos
+    Qiangning Hong
+    Rafael Caricio
+    Rene Dudfield
+    Roey Berman
+    Ronny Pfannschmidt
+    Rory McCann
+    Simon Cross
+    Stavros Korokithakis
+    Thomas Fenzl
+    Thomas Johansson
+    Vinay Sajip
+    Vitaly Babiy
+    W Trevor King
+    Wil Tan
+    Hsiaoming Yang
+License: Expat
+    Permission is hereby granted, free of charge, to any person obtaining
+    a copy of this software and associated documentation files (the
+    "Software"), to deal in the Software without restriction, including
+    without limitation the rights to use, copy, modify, merge, publish,
+    distribute, sublicense, and/or sell copies of the Software, and to
+    permit persons to whom the Software is furnished to do so, subject to
+    the following conditions:
+    .
+    The above copyright notice and this permission notice shall be
+    included in all copies or substantial portions of the Software.
+    .
+    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+    EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+    MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+    NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+    LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+    OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+    WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+Files: Lib/ensurepip/_bundled/pip-*.whl/pip/_vendor/ipaddress.py
+Copyright: 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010, 2011, 2012, 2013, 2014 Python Software Foundation; All Rights Reserved
+           2007 Google Inc.
+License: PSF-2
+
+Files: Lib/ensurepip/_bundled/pip-*.whl/pip/_vendor/retrying.py
+Copyright: (c) 2013-2014, <ray@blacklocus.com>
+           (c) 2014, Derek Wilson <dwilson@domaintools.com>
+           (c) 2014, Alex Kuang <akuang@bizo.com>
+License: Apache-2
+
+License: Apache-2
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+ .
+    http://www.apache.org/licenses/LICENSE-2.0
+ .
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+ .
+ On Debian-based systems the full text of the Apache version 2.0 license
+ can be found in /usr/share/common-licenses/Apache-2.0.
+
+Files: Lib/ensurepip/_bundled/pip-*.whl/pip/_vendor/six.py
+Copyright: Copyright (c) 2010-2014 Benjamin Peterson
+License: Expat
+
+Files: Lib/ensurepip/_bundled/pip-*.whl/pip/_vendor/cachecontrol
+Copyright: 2015 Eric Larson
+License: Apache-2.0
+ Copyright 2015 Eric Larson
+ .
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+ .
+     http://www.apache.org/licenses/LICENSE-2.0
+ .
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+ implied.
+ .
+ See the License for the specific language governing permissions and
+ limitations under the License.
+ .
+ On Debian systems, the license is available at
+ /usr/share/common-licenses/Apache-2.0
+
+Files: Lib/ensurepip/_bundled/pip-*.whl/pip/_vendor/colorama
+Files: Lib/ensurepip/_bundled/pip-*.whl/pip/_vendor/distlib
+Copyright: Copyright (C) 2012-2013 The Python Software Foundation
+           Copyright (C) 2012-2013 Vinay Sajip
+License: Python
+
+Files: Lib/ensurepip/_bundled/pip-*.whl/pip/_vendor/html5lib
+html5lib is Copyright (c) 2006 The Authors
+Authors:
+  James Graham - james@hoppipolla.co.uk
+  Anne van Kesteren - annevankesteren@gmail.com
+  Lachlan Hunt - lachlan.hunt@lachy.id.au
+  Matt McDonald - kanashii@kanashii.ca
+  Sam Ruby - rubys@intertwingly.net
+  Ian Hickson (Google) - ian@hixie.ch
+  Thomas Broyer - t.broyer@gmail.com
+
+License:
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of
+this software and associated documentation files (the "Software"), to deal in
+the Software without restriction, including without limitation the rights to
+use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies
+of the Software, and to permit persons to whom the Software is furnished to do
+so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+
+Files: Lib/ensurepip/_bundled/pip-*.whl/pip/_vendor/lockfile
+Copyright:
+    © 2014–2015 OpenStack Foundation <openstack-dev@lists.openstack.org>
+    © 2007–2012 Skip Montanaro <skip@pobox.com>
+    © 2008–2014 Ben Finney <ben+python@benfinney.id.au>
+License: Expat
+
+Files: Lib/ensurepip/_bundled/pip-*.whl/pip/_vendor/packaging
+Copyright: Copyright (C) 2014 Donald Stufft
+           Copyright (C) 2012-2013 Vinay Sajip
+License: Apache-2.0
+
+Files: Lib/ensurepip/_bundled/pip-*.whl/pip/_vendor/progress
+Copyright: Copyright (c) 2012 Giorgos Verigakis <verigak@gmail.com>
+License: ISC
+
+Files: Lib/ensurepip/_bundled/pip-*.whl/pip/_vendor/requests
+Copyright: 2016, Kenneth Reitz
+License: Apache
+
+Files: Lib/ensurepip/_bundled/pip-*.whl/pip/_vendor/requests/packages/urllib3/*
+Copyright: 2008-2016, Andrey Petrov
+License: Expat
+
+Files: Lib/ensurepip/_bundled/pip-*.whl/pip/_vendor/requests/packages/urllib3/packages/ordered_dict.py
+Copyright: 2009, Raymond Hettinger
+License: Expat
+
+Files: Lib/ensurepip/_bundled/pip-*.whl/pip/_vendor/requests/packages/urllib3/packages/ssl_match_hostname/__init__.py
+Copyright: 2011, Python Software Foundation
+License: PSF-2
+
+Files: Lib/ensurepip/_bundled/pip-*.whl/pip/_vendor/requests/packages/chardet/*
+Copyright: 2006-2008, Mark Pilgrim
+           2012-2013, Ian Cordasco
+License: LGPL-2.1+

syft/cataloger/java/archive_parser_test.go

@@ -130,6 +130,7 @@ func TestParseJar(t *testing.T) {
 							GroupID:    "io.jenkins.plugins",
 							ArtifactID: "example-jenkins-plugin",
 							Version:    "1.0-SNAPSHOT",
+							Extra:      map[string]string{},
 						},
 					},
 				},
@@ -185,6 +186,7 @@ func TestParseJar(t *testing.T) {
 							GroupID:    "org.anchore",
 							ArtifactID: "example-java-app-maven",
 							Version:    "0.1.0",
+							Extra:      map[string]string{},
 						},
 					},
 				},
@@ -203,6 +205,7 @@ func TestParseJar(t *testing.T) {
 							GroupID:    "joda-time",
 							ArtifactID: "joda-time",
 							Version:    "2.9.2",
+							Extra:      map[string]string{},
 						},
 					},
 				},

syft/cataloger/java/pom_properties.go

@@ -42,6 +42,11 @@ func parsePomProperties(path string, reader io.Reader) (*pkg.PomProperties, erro
 		return nil, fmt.Errorf("unable to parse pom.properties: %w", err)
 	}
 
+	// don't allow for a nil collection, ensure it is empty
+	if props.Extra == nil {
+		props.Extra = make(map[string]string)
+	}
+
 	props.Path = path
 
 	return &props, nil

syft/cataloger/java/pom_properties_test.go

@@ -2,10 +2,11 @@ package java
 
 import (
 	"encoding/json"
-	"github.com/anchore/syft/syft/pkg"
-	"github.com/go-test/deep"
 	"os"
 	"testing"
+
+	"github.com/anchore/syft/syft/pkg"
+	"github.com/go-test/deep"
 )
 
 func TestParseJavaPomProperties(t *testing.T) {
@@ -20,6 +21,7 @@ func TestParseJavaPomProperties(t *testing.T) {
 				GroupID:    "org.anchore",
 				ArtifactID: "example-java-app-maven",
 				Version:    "0.1.0",
+				Extra:      map[string]string{},
 			},
 		},
 		{

syft/cataloger/python/package_cataloger.go

@@ -3,10 +3,6 @@ package python
 import (
 	"bufio"
 	"fmt"
-	"path/filepath"
-	"strings"
-
-	"github.com/anchore/syft/internal/log"
 
 	"github.com/anchore/syft/syft/pkg"
 
@@ -14,8 +10,9 @@ import (
 )
 
 const (
-	eggMetadataGlob   = "**/*egg-info/PKG-INFO"
-	wheelMetadataGlob = "**/*dist-info/METADATA"
+	eggMetadataGlob     = "**/*egg-info/PKG-INFO"
+	eggFileMetadataGlob = "**/*.egg-info"
+	wheelMetadataGlob   = "**/*dist-info/METADATA"
 )
 
 type PackageCataloger struct{}
@@ -32,37 +29,73 @@ func (c *PackageCataloger) Name() string {
 
 // Catalog is given an object to resolve file references and content, this function returns any discovered Packages after analyzing python egg and wheel installations.
 func (c *PackageCataloger) Catalog(resolver source.Resolver) ([]pkg.Package, error) {
-	// nolint:prealloc
-	var fileMatches []source.Location
-
-	for _, glob := range []string{eggMetadataGlob, wheelMetadataGlob} {
-		matches, err := resolver.FilesByGlob(glob)
-		if err != nil {
-			return nil, fmt.Errorf("failed to find files by glob: %s", glob)
-		}
-		fileMatches = append(fileMatches, matches...)
+	entries, err := c.getPackageEntries(resolver)
+	if err != nil {
+		return nil, err
 	}
 
-	var pkgs []pkg.Package
-	for _, location := range fileMatches {
-		p, err := c.catalogEggOrWheel(resolver, location)
+	var packages []pkg.Package
+	for _, entry := range entries {
+		p, err := c.catalogEggOrWheel(entry)
 		if err != nil {
-			return nil, fmt.Errorf("unable to catalog python package=%+v: %w", location.Path, err)
+			return nil, fmt.Errorf("unable to catalog python package=%+v: %w", entry.Metadata.Location.RealPath, err)
 		}
 		if p != nil {
-			pkgs = append(pkgs, *p)
+			packages = append(packages, *p)
 		}
 	}
-	return pkgs, nil
+
+	return packages, nil
+}
+
+// getPackageEntries fetches the contents for all python packages within the given resolver.
+func (c *PackageCataloger) getPackageEntries(resolver source.Resolver) ([]*packageEntry, error) {
+	var metadataLocations []source.Location
+
+	// find all primary record paths
+	matches, err := resolver.FilesByGlob(eggMetadataGlob, eggFileMetadataGlob, wheelMetadataGlob)
+	if err != nil {
+		return nil, fmt.Errorf("failed to find files by glob: %w", err)
+	}
+	metadataLocations = append(metadataLocations, matches...)
+
+	// for every primary record path, craft all secondary record paths and build a request object to gather all file contents for each record
+	requester := source.NewContentRequester()
+	entries := make([]*packageEntry, len(metadataLocations))
+	for i, metadataLocation := range metadataLocations {
+		// build the entry to process (holding only path information)
+		entry := newPackageEntry(resolver, metadataLocation)
+
+		// populate the data onto the requester object
+		requester.Add(&entry.Metadata)
+		if entry.FileRecord != nil {
+			requester.Add(entry.FileRecord)
+		}
+		if entry.TopPackage != nil {
+			requester.Add(entry.TopPackage)
+		}
+
+		// keep track of the entry for later package processing
+		entries[i] = entry
+	}
+
+	// return the set of entries and execute the request for fetching contents
+	return entries, requester.Execute(resolver)
 }
 
 // catalogEggOrWheel takes the primary metadata file reference and returns the python package it represents.
-func (c *PackageCataloger) catalogEggOrWheel(resolver source.Resolver, metadataLocation source.Location) (*pkg.Package, error) {
-	metadata, sources, err := c.assembleEggOrWheelMetadata(resolver, metadataLocation)
+func (c *PackageCataloger) catalogEggOrWheel(entry *packageEntry) (*pkg.Package, error) {
+	metadata, sources, err := c.assembleEggOrWheelMetadata(entry)
 	if err != nil {
 		return nil, err
 	}
 
+	// This can happen for Python 2.7 where it is reported from an egg-info, but Python is
+	// the actual runtime, it isn't a "package". The special-casing here allows to skip it
+	if metadata.Name == "Python" {
+		return nil, nil
+	}
+
 	var licenses []string
 	if metadata.License != "" {
 		licenses = []string{metadata.License}
@@ -81,26 +114,45 @@ func (c *PackageCataloger) catalogEggOrWheel(resolver source.Resolver, metadataL
 	}, nil
 }
 
-// fetchRecordFiles finds a corresponding RECORD file for the given python package metadata file and returns the set of file records contained.
-func (c *PackageCataloger) fetchRecordFiles(resolver source.Resolver, metadataLocation source.Location) (files []pkg.PythonFileRecord, sources []source.Location, err error) {
+// assembleEggOrWheelMetadata discovers and accumulates python package metadata from multiple file sources and returns a single metadata object as well as a list of files where the metadata was derived from.
+func (c *PackageCataloger) assembleEggOrWheelMetadata(entry *packageEntry) (*pkg.PythonPackageMetadata, []source.Location, error) {
+	var sources = []source.Location{entry.Metadata.Location}
+
+	metadata, err := parseWheelOrEggMetadata(entry.Metadata.Location.RealPath, entry.Metadata.Contents)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	// attach any python files found for the given wheel/egg installation
+	r, s, err := c.processRecordFiles(entry.FileRecord)
+	if err != nil {
+		return nil, nil, err
+	}
+	sources = append(sources, s...)
+	metadata.Files = r
+
+	// attach any top-level package names found for the given wheel/egg installation
+	p, s, err := c.processTopLevelPackages(entry.TopPackage)
+	if err != nil {
+		return nil, nil, err
+	}
+	sources = append(sources, s...)
+	metadata.TopLevelPackages = p
+
+	return &metadata, sources, nil
+}
+
+// processRecordFiles takes a corresponding RECORD file for the given python package metadata file and returns the set of file records contained.
+func (c *PackageCataloger) processRecordFiles(entry *source.FileData) (files []pkg.PythonFileRecord, sources []source.Location, err error) {
 	// we've been given a file reference to a specific wheel METADATA file. note: this may be for a directory
 	// or for an image... for an image the METADATA file may be present within multiple layers, so it is important
 	// to reconcile the RECORD path to the same layer (or the next adjacent lower layer).
 
-	// lets find the RECORD file relative to the directory where the METADATA file resides (in path AND layer structure)
-	recordPath := filepath.Join(filepath.Dir(metadataLocation.Path), "RECORD")
-	recordRef := resolver.RelativeFileByPath(metadataLocation, recordPath)
-
-	if recordRef != nil {
-		sources = append(sources, *recordRef)
-
-		recordContents, err := resolver.FileContentsByLocation(*recordRef)
-		if err != nil {
-			return nil, nil, err
-		}
+	if entry != nil {
+		sources = append(sources, entry.Location)
 
 		// parse the record contents
-		records, err := parseWheelOrEggRecord(strings.NewReader(recordContents))
+		records, err := parseWheelOrEggRecord(entry.Contents)
 		if err != nil {
 			return nil, nil, err
 		}
@@ -110,26 +162,15 @@ func (c *PackageCataloger) fetchRecordFiles(resolver source.Resolver, metadataLo
 	return files, sources, nil
 }
 
-// fetchTopLevelPackages finds a corresponding top_level.txt file for the given python package metadata file and returns the set of package names contained.
-func (c *PackageCataloger) fetchTopLevelPackages(resolver source.Resolver, metadataLocation source.Location) (pkgs []string, sources []source.Location, err error) {
-	// a top_level.txt file specifies the python top-level packages (provided by this python package) installed into site-packages
-	parentDir := filepath.Dir(metadataLocation.Path)
-	topLevelPath := filepath.Join(parentDir, "top_level.txt")
-	topLevelRef := resolver.RelativeFileByPath(metadataLocation, topLevelPath)
-
-	if topLevelRef == nil {
-		log.Warnf("missing python package top_level.txt (package=%q)", metadataLocation.Path)
+// processTopLevelPackages takes a corresponding top_level.txt file for the given python package metadata file and returns the set of package names contained.
+func (c *PackageCataloger) processTopLevelPackages(entry *source.FileData) (pkgs []string, sources []source.Location, err error) {
+	if entry == nil {
 		return nil, nil, nil
 	}
 
-	sources = append(sources, *topLevelRef)
+	sources = append(sources, entry.Location)
 
-	topLevelContents, err := resolver.FileContentsByLocation(*topLevelRef)
-	if err != nil {
-		return nil, nil, err
-	}
-
-	scanner := bufio.NewScanner(strings.NewReader(topLevelContents))
+	scanner := bufio.NewScanner(entry.Contents)
 	for scanner.Scan() {
 		pkgs = append(pkgs, scanner.Text())
 	}
@@ -140,36 +181,3 @@ func (c *PackageCataloger) fetchTopLevelPackages(resolver source.Resolver, metad
 
 	return pkgs, sources, nil
 }
-
-// assembleEggOrWheelMetadata discovers and accumulates python package metadata from multiple file sources and returns a single metadata object as well as a list of files where the metadata was derived from.
-func (c *PackageCataloger) assembleEggOrWheelMetadata(resolver source.Resolver, metadataLocation source.Location) (*pkg.PythonPackageMetadata, []source.Location, error) {
-	var sources = []source.Location{metadataLocation}
-
-	metadataContents, err := resolver.FileContentsByLocation(metadataLocation)
-	if err != nil {
-		return nil, nil, err
-	}
-
-	metadata, err := parseWheelOrEggMetadata(metadataLocation.Path, strings.NewReader(metadataContents))
-	if err != nil {
-		return nil, nil, err
-	}
-
-	// attach any python files found for the given wheel/egg installation
-	r, s, err := c.fetchRecordFiles(resolver, metadataLocation)
-	if err != nil {
-		return nil, nil, err
-	}
-	sources = append(sources, s...)
-	metadata.Files = r
-
-	// attach any top-level package names found for the given wheel/egg installation
-	p, s, err := c.fetchTopLevelPackages(resolver, metadataLocation)
-	if err != nil {
-		return nil, nil, err
-	}
-	sources = append(sources, s...)
-	metadata.TopLevelPackages = p
-
-	return &metadata, sources, nil
-}

syft/cataloger/python/package_cataloger_test.go

@@ -1,131 +1,27 @@
 package python
 
 import (
-	"fmt"
-	"io"
-	"io/ioutil"
-	"os"
-	"strings"
 	"testing"
 
-	"github.com/anchore/syft/syft/source"
-
 	"github.com/anchore/syft/syft/pkg"
+	"github.com/anchore/syft/syft/source"
 	"github.com/go-test/deep"
 )
 
-type pythonTestResolverMock struct {
-	metadataReader io.Reader
-	recordReader   io.Reader
-	topLevelReader io.Reader
-	metadataRef    *source.Location
-	recordRef      *source.Location
-	topLevelRef    *source.Location
-	contents       map[source.Location]string
-}
-
-func newTestResolver(metaPath, recordPath, topPath string) *pythonTestResolverMock {
-	metadataReader, err := os.Open(metaPath)
-	if err != nil {
-		panic(fmt.Errorf("failed to open metadata: %+v", err))
-	}
-
-	var recordReader io.Reader
-	if recordPath != "" {
-		recordReader, err = os.Open(recordPath)
-		if err != nil {
-			panic(fmt.Errorf("failed to open record: %+v", err))
-		}
-	}
-
-	var topLevelReader io.Reader
-	if topPath != "" {
-		topLevelReader, err = os.Open(topPath)
-		if err != nil {
-			panic(fmt.Errorf("failed to open top level: %+v", err))
-		}
-	}
-
-	var recordRef *source.Location
-	if recordReader != nil {
-		ref := source.NewLocation("test-fixtures/dist-info/RECORD")
-		recordRef = &ref
-	}
-	var topLevelRef *source.Location
-	if topLevelReader != nil {
-		ref := source.NewLocation("test-fixtures/dist-info/top_level.txt")
-		topLevelRef = &ref
-	}
-	metadataRef := source.NewLocation("test-fixtures/dist-info/METADATA")
-	return &pythonTestResolverMock{
-		recordReader:   recordReader,
-		metadataReader: metadataReader,
-		topLevelReader: topLevelReader,
-		metadataRef:    &metadataRef,
-		recordRef:      recordRef,
-		topLevelRef:    topLevelRef,
-		contents:       make(map[source.Location]string),
-	}
-}
-
-func (r *pythonTestResolverMock) FileContentsByLocation(ref source.Location) (string, error) {
-	switch {
-	case r.topLevelRef != nil && ref.Path == r.topLevelRef.Path:
-		b, err := ioutil.ReadAll(r.topLevelReader)
-		if err != nil {
-			return "", err
-		}
-		return string(b), nil
-	case ref.Path == r.metadataRef.Path:
-		b, err := ioutil.ReadAll(r.metadataReader)
-		if err != nil {
-			return "", err
-		}
-		return string(b), nil
-	case ref.Path == r.recordRef.Path:
-		b, err := ioutil.ReadAll(r.recordReader)
-		if err != nil {
-			return "", err
-		}
-		return string(b), nil
-	}
-	return "", fmt.Errorf("invalid value given")
-}
-
-func (r *pythonTestResolverMock) MultipleFileContentsByLocation(_ []source.Location) (map[source.Location]string, error) {
-	return nil, fmt.Errorf("not implemented")
-}
-
-func (r *pythonTestResolverMock) FilesByPath(_ ...string) ([]source.Location, error) {
-	return nil, fmt.Errorf("not implemented")
-}
-
-func (r *pythonTestResolverMock) FilesByGlob(_ ...string) ([]source.Location, error) {
-	return nil, fmt.Errorf("not implemented")
-}
-func (r *pythonTestResolverMock) RelativeFileByPath(_ source.Location, path string) *source.Location {
-	switch {
-	case strings.Contains(path, "RECORD"):
-		return r.recordRef
-	case strings.Contains(path, "top_level.txt"):
-		return r.topLevelRef
-	default:
-		panic(fmt.Errorf("invalid RelativeFileByPath value given: %q", path))
-	}
-}
-
 func TestPythonPackageWheelCataloger(t *testing.T) {
 	tests := []struct {
-		MetadataFixture string
-		RecordFixture   string
-		TopLevelFixture string
-		ExpectedPackage pkg.Package
+		name            string
+		fixtures        []string
+		expectedPackage pkg.Package
 	}{
 		{
-			MetadataFixture: "test-fixtures/egg-info/PKG-INFO",
-			RecordFixture:   "test-fixtures/egg-info/RECORD",
-			TopLevelFixture: "test-fixtures/egg-info/top_level.txt",
-			ExpectedPackage: pkg.Package{
+			name: "egg-info directory",
+			fixtures: []string{
+				"test-fixtures/egg-info/PKG-INFO",
+				"test-fixtures/egg-info/RECORD",
+				"test-fixtures/egg-info/top_level.txt",
+			},
+			expectedPackage: pkg.Package{
 				Name:         "requests",
 				Version:      "2.22.0",
 				Type:         pkg.PythonPkg,
@@ -154,10 +50,13 @@ func TestPythonPackageWheelCataloger(t *testing.T) {
 			},
 		},
 		{
-			MetadataFixture: "test-fixtures/dist-info/METADATA",
-			RecordFixture:   "test-fixtures/dist-info/RECORD",
-			TopLevelFixture: "test-fixtures/dist-info/top_level.txt",
-			ExpectedPackage: pkg.Package{
+			name: "dist-info directory",
+			fixtures: []string{
+				"test-fixtures/dist-info/METADATA",
+				"test-fixtures/dist-info/RECORD",
+				"test-fixtures/dist-info/top_level.txt",
+			},
+			expectedPackage: pkg.Package{
 				Name:         "Pygments",
 				Version:      "2.6.1",
 				Type:         pkg.PythonPkg,
@@ -187,8 +86,9 @@ func TestPythonPackageWheelCataloger(t *testing.T) {
 		{
 			// in cases where the metadata file is available and the record is not we should still record there is a package
 			// additionally empty top_level.txt files should not result in an error
-			MetadataFixture: "test-fixtures/partial.dist-info/METADATA",
-			ExpectedPackage: pkg.Package{
+			name:     "partial dist-info directory",
+			fixtures: []string{"test-fixtures/partial.dist-info/METADATA"},
+			expectedPackage: pkg.Package{
 				Name:         "Pygments",
 				Version:      "2.6.1",
 				Type:         pkg.PythonPkg,
@@ -207,34 +107,78 @@ func TestPythonPackageWheelCataloger(t *testing.T) {
 				},
 			},
 		},
+		{
+			name:     "egg-info regular file",
+			fixtures: []string{"test-fixtures/test.egg-info"},
+			expectedPackage: pkg.Package{
+				Name:         "requests",
+				Version:      "2.22.0",
+				Type:         pkg.PythonPkg,
+				Language:     pkg.Python,
+				Licenses:     []string{"Apache 2.0"},
+				FoundBy:      "python-package-cataloger",
+				MetadataType: pkg.PythonPackageMetadataType,
+				Metadata: pkg.PythonPackageMetadata{
+					Name:                 "requests",
+					Version:              "2.22.0",
+					License:              "Apache 2.0",
+					Platform:             "UNKNOWN",
+					Author:               "Kenneth Reitz",
+					AuthorEmail:          "me@kennethreitz.org",
+					SitePackagesRootPath: "test-fixtures",
+				},
+			},
+		},
 	}
 
 	for _, test := range tests {
-		t.Run(test.MetadataFixture, func(t *testing.T) {
-			resolver := newTestResolver(test.MetadataFixture, test.RecordFixture, test.TopLevelFixture)
+		t.Run(test.name, func(t *testing.T) {
+			resolver := source.NewMockResolverForPaths(test.fixtures...)
 
-			// note that the source is the record ref created by the resolver mock... attach the expected values
-			test.ExpectedPackage.Locations = []source.Location{*resolver.metadataRef}
-			if resolver.recordRef != nil {
-				test.ExpectedPackage.Locations = append(test.ExpectedPackage.Locations, *resolver.recordRef)
+			locations, err := resolver.FilesByPath(test.fixtures...)
+			if err != nil {
+				t.Fatal(err)
 			}
 
-			if resolver.topLevelRef != nil {
-				test.ExpectedPackage.Locations = append(test.ExpectedPackage.Locations, *resolver.topLevelRef)
-			}
-			// end patching expected values with runtime data...
+			test.expectedPackage.Locations = locations
 
-			pyPkgCataloger := NewPythonPackageCataloger()
-
-			actual, err := pyPkgCataloger.catalogEggOrWheel(resolver, *resolver.metadataRef)
+			actual, err := NewPythonPackageCataloger().Catalog(resolver)
 			if err != nil {
 				t.Fatalf("failed to catalog python package: %+v", err)
 			}
 
-			for _, d := range deep.Equal(actual, &test.ExpectedPackage) {
+			if len(actual) != 1 {
+				t.Fatalf("unexpected number of packages: %d", len(actual))
+			}
+
+			for _, d := range deep.Equal(actual[0], test.expectedPackage) {
 				t.Errorf("diff: %+v", d)
 			}
 		})
 	}
+}
 
+func TestIgnorePackage(t *testing.T) {
+	tests := []struct {
+		MetadataFixture string
+	}{
+		{
+			MetadataFixture: "test-fixtures/Python-2.7.egg-info",
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.MetadataFixture, func(t *testing.T) {
+			resolver := source.NewMockResolverForPaths(test.MetadataFixture)
+
+			actual, err := NewPythonPackageCataloger().Catalog(resolver)
+			if err != nil {
+				t.Fatalf("failed to catalog python package: %+v", err)
+			}
+
+			if len(actual) != 0 {
+				t.Fatalf("Expected 0 packages but found: %d", len(actual))
+			}
+		})
+	}
 }

syft/cataloger/python/package_entry.go

@@ -0,0 +1,49 @@
+package python
+
+import (
+	"path/filepath"
+
+	"github.com/anchore/syft/syft/source"
+)
+
+type packageEntry struct {
+	Metadata   source.FileData
+	FileRecord *source.FileData
+	TopPackage *source.FileData
+}
+
+// newPackageEntry returns a new packageEntry to be processed relative to what information is available in the given FileResolver.
+func newPackageEntry(resolver source.FileResolver, metadataLocation source.Location) *packageEntry {
+	// we've been given a file reference to a specific wheel METADATA file. note: this may be for a directory
+	// or for an image... for an image the METADATA file may be present within multiple layers, so it is important
+	// to reconcile the RECORD path to the same layer (or a lower layer). The same is true with the top_level.txt file.
+
+	// lets find the RECORD file relative to the directory where the METADATA file resides (in path AND layer structure)
+	recordPath := filepath.Join(filepath.Dir(metadataLocation.RealPath), "RECORD")
+	recordLocation := resolver.RelativeFileByPath(metadataLocation, recordPath)
+
+	// a top_level.txt file specifies the python top-level packages (provided by this python package) installed into site-packages
+	parentDir := filepath.Dir(metadataLocation.RealPath)
+	topLevelPath := filepath.Join(parentDir, "top_level.txt")
+	topLevelLocation := resolver.RelativeFileByPath(metadataLocation, topLevelPath)
+
+	// build an entry that will later be populated with contents when the request is executed
+	entry := &packageEntry{
+		Metadata: source.FileData{
+			Location: metadataLocation,
+		},
+	}
+
+	if recordLocation != nil {
+		entry.FileRecord = &source.FileData{
+			Location: *recordLocation,
+		}
+	}
+
+	if topLevelLocation != nil {
+		entry.TopPackage = &source.FileData{
+			Location: *topLevelLocation,
+		}
+	}
+	return entry
+}

syft/cataloger/python/parse_wheel_egg_metadata.go

@@ -7,6 +7,8 @@ import (
 	"path/filepath"
 	"strings"
 
+	"github.com/anchore/syft/internal/file"
+
 	"github.com/mitchellh/mapstructure"
 
 	"github.com/anchore/syft/syft/pkg"
@@ -70,11 +72,28 @@ func parseWheelOrEggMetadata(path string, reader io.Reader) (pkg.PythonPackageMe
 
 	// add additional metadata not stored in the egg/wheel metadata file
 
-	metadata.SitePackagesRootPath = filepath.Clean(filepath.Join(filepath.Dir(path), ".."))
+	metadata.SitePackagesRootPath = determineSitePackagesRootPath(path)
 
 	return metadata, nil
 }
 
+// isEggRegularFile determines if the specified path is the regular file variant
+// of egg metadata (as opposed to a directory that contains more metadata
+// files).
+func isEggRegularFile(path string) bool {
+	return file.GlobMatch(eggFileMetadataGlob, path)
+}
+
+// determineSitePackagesRootPath returns the path of the site packages root,
+// given the egg metadata file or directory specified in the path.
+func determineSitePackagesRootPath(path string) string {
+	if isEggRegularFile(path) {
+		return filepath.Clean(filepath.Dir(path))
+	}
+
+	return filepath.Clean(filepath.Dir(filepath.Dir(path)))
+}
+
 // handleFieldBodyContinuation returns the updated value for the specified field after processing the specified line.
 // If the continuation cannot be processed, it returns an error.
 func handleFieldBodyContinuation(key, line string, fields map[string]string) (string, error) {

syft/cataloger/python/parse_wheel_egg_metadata_test.go

@@ -56,5 +56,64 @@ func TestParseWheelEggMetadata(t *testing.T) {
 			}
 		})
 	}
+}
 
+func TestIsRegularEggFile(t *testing.T) {
+	cases := []struct {
+		path     string
+		expected bool
+	}{
+		{
+			"/usr/lib64/python2.6/site-packages/M2Crypto-0.20.2-py2.6.egg-info",
+			true,
+		},
+		{
+			"/usr/lib64/python2.6/site-packages/M2Crypto-0.20.2-py2.6.egg-info/PKG-INFO",
+			false,
+		},
+		{
+			"/usr/lib64/python2.6/site-packages/M2Crypto-0.20.2-py2.6.dist-info/METADATA",
+			false,
+		},
+	}
+
+	for _, c := range cases {
+		t.Run(c.path, func(t *testing.T) {
+			actual := isEggRegularFile(c.path)
+
+			if actual != c.expected {
+				t.Errorf("expected %t but got %t", c.expected, actual)
+			}
+		})
+	}
+}
+
+func TestDetermineSitePackagesRootPath(t *testing.T) {
+	cases := []struct {
+		inputPath string
+		expected  string
+	}{
+		{
+			inputPath: "/usr/lib64/python2.6/site-packages/ethtool-0.6-py2.6.egg-info",
+			expected:  "/usr/lib64/python2.6/site-packages",
+		},
+		{
+			inputPath: "/usr/lib/python2.7/dist-packages/configobj-5.0.6.egg-info/top_level.txt",
+			expected:  "/usr/lib/python2.7/dist-packages",
+		},
+		{
+			inputPath: "/usr/lib/python2.7/dist-packages/six-1.10.0.egg-info/PKG-INFO",
+			expected:  "/usr/lib/python2.7/dist-packages",
+		},
+	}
+
+	for _, c := range cases {
+		t.Run(c.inputPath, func(t *testing.T) {
+			actual := determineSitePackagesRootPath(c.inputPath)
+
+			if actual != c.expected {
+				t.Errorf("expected %s but got %s", c.expected, actual)
+			}
+		})
+	}
 }

syft/cataloger/python/test-fixtures/Python-2.7.egg-info

@@ -0,0 +1,31 @@
+Metadata-Version: 1.1
+Name: Python
+Version: 2.7.13
+Summary: A high-level object-oriented programming language
+Home-page: http://www.python.org/2.7
+Author: Guido van Rossum and the Python community
+Author-email: python-dev@python.org
+License: PSF license
+Description: Python is an interpreted, interactive, object-oriented programming
+        language. It is often compared to Tcl, Perl, Scheme or Java.
+
+        Python combines remarkable power with very clear syntax. It has
+        modules, classes, exceptions, very high level dynamic data types, and
+        dynamic typing. There are interfaces to many system calls and
+        libraries, as well as to various windowing systems (X11, Motif, Tk,
+        Mac, MFC). New built-in modules are easily written in C or C++. Python
+        is also usable as an extension language for applications that need a
+        programmable interface.
+
+        The Python implementation is portable: it runs on many brands of UNIX,
+        on Windows, DOS, OS/2, Mac, Amiga... If your favorite system isn't
+        listed here, it may still be supported, if there's a C compiler for
+        it. Ask around on comp.lang.python -- or just try compiling Python
+        yourself.
+Platform: Many
+Classifier: Development Status :: 6 - Mature
+Classifier: License :: OSI Approved :: Python Software Foundation License
+Classifier: Natural Language :: English
+Classifier: Programming Language :: C
+Classifier: Programming Language :: Python
+Classifier: Topic :: Software Development

syft/cataloger/python/test-fixtures/test.egg-info

@@ -0,0 +1,134 @@
+Metadata-Version: 2.1
+Name: requests
+Version: 2.22.0
+Summary: Python HTTP for Humans.
+Home-page: http://python-requests.org
+Author: Kenneth Reitz
+Author-email: me@kennethreitz.org
+License: Apache 2.0
+Description: Requests: HTTP for Humans™
+        ==========================
+        
+        [![image](https://img.shields.io/pypi/v/requests.svg)](https://pypi.org/project/requests/)
+        [![image](https://img.shields.io/pypi/l/requests.svg)](https://pypi.org/project/requests/)
+        [![image](https://img.shields.io/pypi/pyversions/requests.svg)](https://pypi.org/project/requests/)
+        [![codecov.io](https://codecov.io/github/requests/requests/coverage.svg?branch=master)](https://codecov.io/github/requests/requests)
+        [![image](https://img.shields.io/github/contributors/requests/requests.svg)](https://github.com/requests/requests/graphs/contributors)
+        [![image](https://img.shields.io/badge/Say%20Thanks-!-1EAEDB.svg)](https://saythanks.io/to/kennethreitz)
+        
+        Requests is the only *Non-GMO* HTTP library for Python, safe for human
+        consumption.
+        
+        ![image](https://farm5.staticflickr.com/4317/35198386374_1939af3de6_k_d.jpg)
+        
+        Behold, the power of Requests:
+        
+        ``` {.sourceCode .python}
+        >>> import requests
+        >>> r = requests.get('https://api.github.com/user', auth=('user', 'pass'))
+        >>> r.status_code
+        200
+        >>> r.headers['content-type']
+        'application/json; charset=utf8'
+        >>> r.encoding
+        'utf-8'
+        >>> r.text
+        u'{"type":"User"...'
+        >>> r.json()
+        {u'disk_usage': 368627, u'private_gists': 484, ...}
+        ```
+        
+        See [the similar code, sans Requests](https://gist.github.com/973705).
+        
+        [![image](https://raw.githubusercontent.com/requests/requests/master/docs/_static/requests-logo-small.png)](http://docs.python-requests.org/)
+        
+        Requests allows you to send *organic, grass-fed* HTTP/1.1 requests,
+        without the need for manual labor. There's no need to manually add query
+        strings to your URLs, or to form-encode your POST data. Keep-alive and
+        HTTP connection pooling are 100% automatic, thanks to
+        [urllib3](https://github.com/shazow/urllib3).
+        
+        Besides, all the cool kids are doing it. Requests is one of the most
+        downloaded Python packages of all time, pulling in over 11,000,000
+        downloads every month. You don't want to be left out!
+        
+        Feature Support
+        ---------------
+        
+        Requests is ready for today's web.
+        
+        -   International Domains and URLs
+        -   Keep-Alive & Connection Pooling
+        -   Sessions with Cookie Persistence
+        -   Browser-style SSL Verification
+        -   Basic/Digest Authentication
+        -   Elegant Key/Value Cookies
+        -   Automatic Decompression
+        -   Automatic Content Decoding
+        -   Unicode Response Bodies
+        -   Multipart File Uploads
+        -   HTTP(S) Proxy Support
+        -   Connection Timeouts
+        -   Streaming Downloads
+        -   `.netrc` Support
+        -   Chunked Requests
+        
+        Requests officially supports Python 2.7 & 3.4–3.7, and runs great on
+        PyPy.
+        
+        Installation
+        ------------
+        
+        To install Requests, simply use [pipenv](http://pipenv.org/) (or pip, of
+        course):
+        
+        ``` {.sourceCode .bash}
+        $ pipenv install requests
+        ✨🍰✨
+        ```
+        
+        Satisfaction guaranteed.
+        
+        Documentation
+        -------------
+        
+        Fantastic documentation is available at
+        <http://docs.python-requests.org/>, for a limited time only.
+        
+        How to Contribute
+        -----------------
+        
+        1.  Become more familiar with the project by reading our [Contributor's Guide](http://docs.python-requests.org/en/latest/dev/contributing/) and our [development philosophy](http://docs.python-requests.org/en/latest/dev/philosophy/).
+        2.  Check for open issues or open a fresh issue to start a discussion
+            around a feature idea or a bug. There is a [Contributor
+            Friendly](https://github.com/requests/requests/issues?direction=desc&labels=Contributor+Friendly&page=1&sort=updated&state=open)
+            tag for issues that should be ideal for people who are not very
+            familiar with the codebase yet.
+        3.  Fork [the repository](https://github.com/requests/requests) on
+            GitHub to start making your changes to the **master** branch (or
+            branch off of it).
+        4.  Write a test which shows that the bug was fixed or that the feature
+            works as expected.
+        5.  Send a pull request and bug the maintainer until it gets merged and
+            published. :) Make sure to add yourself to
+            [AUTHORS](https://github.com/requests/requests/blob/master/AUTHORS.rst).
+        
+        
+Platform: UNKNOWN
+Classifier: Development Status :: 5 - Production/Stable
+Classifier: Intended Audience :: Developers
+Classifier: Natural Language :: English
+Classifier: License :: OSI Approved :: Apache Software License
+Classifier: Programming Language :: Python
+Classifier: Programming Language :: Python :: 2
+Classifier: Programming Language :: Python :: 2.7
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.5
+Classifier: Programming Language :: Python :: 3.6
+Classifier: Programming Language :: Python :: 3.7
+Classifier: Programming Language :: Python :: Implementation :: CPython
+Classifier: Programming Language :: Python :: Implementation :: PyPy
+Requires-Python: >=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*
+Description-Content-Type: text/markdown
+Provides-Extra: security
+Provides-Extra: socks

syft/cataloger/rpmdb/cataloger.go

@@ -5,7 +5,6 @@ package rpmdb
 
 import (
 	"fmt"
-	"strings"
 
 	"github.com/anchore/syft/syft/pkg"
 	"github.com/anchore/syft/syft/source"
@@ -37,14 +36,14 @@ func (c *Cataloger) Catalog(resolver source.Resolver) ([]pkg.Package, error) {
 
 	var pkgs []pkg.Package
 	for _, location := range fileMatches {
-		dbContents, err := resolver.FileContentsByLocation(location)
+		dbContentReader, err := resolver.FileContentsByLocation(location)
 		if err != nil {
 			return nil, err
 		}
 
-		pkgs, err = parseRpmDB(resolver, location, strings.NewReader(dbContents))
+		pkgs, err = parseRpmDB(resolver, location, dbContentReader)
 		if err != nil {
-			return nil, fmt.Errorf("unable to catalog rpmdb package=%+v: %w", location.Path, err)
+			return nil, fmt.Errorf("unable to catalog rpmdb package=%+v: %w", location.RealPath, err)
 		}
 	}
 	return pkgs, nil

syft/cataloger/rpmdb/parse_rpmdb.go

@@ -45,11 +45,6 @@ func parseRpmDB(resolver source.FileResolver, dbLocation source.Location, reader
 	allPkgs := make([]pkg.Package, 0)
 
 	for _, entry := range pkgList {
-		records, err := extractRpmdbFileRecords(resolver, entry)
-		if err != nil {
-			return nil, err
-		}
-
 		p := pkg.Package{
 			Name:         entry.Name,
 			Version:      fmt.Sprintf("%s-%s", entry.Version, entry.Release), // this is what engine does, instead of fmt.Sprintf("%d:%s-%s.%s", entry.Epoch, entry.Version, entry.Release, entry.Arch)
@@ -67,7 +62,7 @@ func parseRpmDB(resolver source.FileResolver, dbLocation source.Location, reader
 				Vendor:    entry.Vendor,
 				License:   entry.License,
 				Size:      entry.Size,
-				Files:     records,
+				Files:     extractRpmdbFileRecords(resolver, entry),
 			},
 		}
 
@@ -77,25 +72,19 @@ func parseRpmDB(resolver source.FileResolver, dbLocation source.Location, reader
 	return allPkgs, nil
 }
 
-func extractRpmdbFileRecords(resolver source.FileResolver, entry *rpmdb.PackageInfo) ([]pkg.RpmdbFileRecord, error) {
+func extractRpmdbFileRecords(resolver source.FileResolver, entry *rpmdb.PackageInfo) []pkg.RpmdbFileRecord {
 	var records = make([]pkg.RpmdbFileRecord, 0)
 
 	for _, record := range entry.Files {
-		refs, err := resolver.FilesByPath(record.Path)
-		if err != nil {
-			return nil, fmt.Errorf("failed to resolve path=%+v: %w", record.Path, err)
-		}
 		//only persist RPMDB file records which exist in the image/directory, otherwise ignore them
-		if len(refs) == 0 {
-			continue
+		if resolver.HasPath(record.Path) {
+			records = append(records, pkg.RpmdbFileRecord{
+				Path:   record.Path,
+				Mode:   pkg.RpmdbFileMode(record.Mode),
+				Size:   int(record.Size),
+				SHA256: record.SHA256,
+			})
 		}
-
-		records = append(records, pkg.RpmdbFileRecord{
-			Path:   record.Path,
-			Mode:   pkg.RpmdbFileMode(record.Mode),
-			Size:   int(record.Size),
-			SHA256: record.SHA256,
-		})
 	}
-	return records, nil
+	return records
 }

syft/cataloger/rpmdb/parse_rpmdb_test.go

@@ -21,6 +21,10 @@ func newTestFileResolver(ignorePaths bool) *rpmdbTestFileResolverMock {
 	}
 }
 
+func (r rpmdbTestFileResolverMock) HasPath(path string) bool {
+	return !r.ignorePaths
+}
+
 func (r *rpmdbTestFileResolverMock) FilesByPath(paths ...string) ([]source.Location, error) {
 	if r.ignorePaths {
 		// act as if no paths exist

syft/cataloger/ruby/catalogers.go

@@ -19,7 +19,7 @@ func NewGemFileLockCataloger() *common.GenericCataloger {
 // NewGemSpecCataloger returns a new Bundler cataloger object tailored for detecting installations of gems (e.g. Gemspec).
 func NewGemSpecCataloger() *common.GenericCataloger {
 	globParsers := map[string]common.ParserFn{
-		"**/specifications/*.gemspec": parseGemSpecEntries,
+		"**/specifications/**/*.gemspec": parseGemSpecEntries,
 	}
 
 	return common.NewGenericCataloger(nil, globParsers, "ruby-gemspec-cataloger")

syft/distro/identify.go

@@ -1,6 +1,7 @@
 package distro
 
 import (
+	"io/ioutil"
 	"regexp"
 	"strings"
 
@@ -16,28 +17,28 @@ type parseEntry struct {
 	fn   parseFunc
 }
 
+var identityFiles = []parseEntry{
+	{
+		// most distros provide a link at this location
+		path: "/etc/os-release",
+		fn:   parseOsRelease,
+	},
+	{
+		// standard location for rhel & debian distros
+		path: "/usr/lib/os-release",
+		fn:   parseOsRelease,
+	},
+	{
+		// check for busybox (important to check this last since other distros contain the busybox binary)
+		path: "/bin/busybox",
+		fn:   parseBusyBox,
+	},
+}
+
 // Identify parses distro-specific files to determine distro metadata like version and release.
 func Identify(resolver source.Resolver) *Distro {
 	var distro *Distro
 
-	identityFiles := []parseEntry{
-		{
-			// most distros provide a link at this location
-			path: "/etc/os-release",
-			fn:   parseOsRelease,
-		},
-		{
-			// standard location for rhel & debian distros
-			path: "/usr/lib/os-release",
-			fn:   parseOsRelease,
-		},
-		{
-			// check for busybox (important to check this last since other distros contain the busybox binary)
-			path: "/bin/busybox",
-			fn:   parseBusyBox,
-		},
-	}
-
 identifyLoop:
 	for _, entry := range identityFiles {
 		locations, err := resolver.FilesByPath(entry.path)
@@ -52,19 +53,25 @@ identifyLoop:
 		}
 
 		for _, location := range locations {
-			content, err := resolver.FileContentsByLocation(location)
+			contentReader, err := resolver.FileContentsByLocation(location)
 
 			if err != nil {
 				log.Debugf("unable to get contents from %s: %s", entry.path, err)
 				continue
 			}
 
-			if content == "" {
+			content, err := ioutil.ReadAll(contentReader)
+			if err != nil {
+				log.Errorf("unable to read %q: %+v", location.RealPath, err)
+				break
+			}
+
+			if len(content) == 0 {
 				log.Debugf("no contents in file, skipping: %s", entry.path)
 				continue
 			}
 
-			if candidateDistro := entry.fn(content); candidateDistro != nil {
+			if candidateDistro := entry.fn(string(content)); candidateDistro != nil {
 				distro = candidateDistro
 				break identifyLoop
 			}

syft/event/event.go

@@ -15,4 +15,7 @@ const (
 
 	// CatalogerFinished is a partybus event that occurs when the package cataloging has completed
 	CatalogerFinished partybus.EventType = "syft-cataloger-finished-event"
+
+	// ImportStarted is a partybus event that occurs when an SBOM upload process has begun
+	ImportStarted partybus.EventType = "syft-import-started-event"
 )

syft/event/parsers/parsers.go

@@ -6,6 +6,8 @@ package parsers
 import (
 	"fmt"
 
+	"github.com/wagoodman/go-progress"
+
 	"github.com/anchore/syft/syft/cataloger"
 	"github.com/anchore/syft/syft/event"
 	"github.com/anchore/syft/syft/presenter"
@@ -75,3 +77,21 @@ func ParseAppUpdateAvailable(e partybus.Event) (string, error) {
 
 	return newVersion, nil
 }
+
+func ParseImportStarted(e partybus.Event) (string, progress.StagedProgressable, error) {
+	if err := checkEventType(e.Type, event.ImportStarted); err != nil {
+		return "", nil, err
+	}
+
+	host, ok := e.Source.(string)
+	if !ok {
+		return "", nil, newPayloadErr(e.Type, "Source", e.Source)
+	}
+
+	prog, ok := e.Value.(progress.StagedProgressable)
+	if !ok {
+		return "", nil, newPayloadErr(e.Type, "Value", e.Value)
+	}
+
+	return host, prog, nil
+}

syft/pkg/dpkg_metadata.go

@@ -11,6 +11,7 @@ type DpkgMetadata struct {
 	Package       string           `mapstructure:"Package" json:"package"`
 	Source        string           `mapstructure:"Source" json:"source"`
 	Version       string           `mapstructure:"Version" json:"version"`
+	SourceVersion string           `mapstructure:"SourceVersion" json:"sourceVersion"`
 	Architecture  string           `mapstructure:"Architecture" json:"architecture"`
 	Maintainer    string           `mapstructure:"Maintainer" json:"maintainer"`
 	InstalledSize int              `mapstructure:"InstalledSize" json:"installedSize"`

syft/presenter/cyclonedx/bom-descriptor.go

@@ -11,25 +11,25 @@ import (
 
 // BomDescriptor represents all metadata surrounding the BOM report (such as when the BOM was made, with which tool, and the item being cataloged).
 type BomDescriptor struct {
-	XMLName   xml.Name     `xml:"bd:metadata"`
-	Timestamp string       `xml:"bd:timestamp,omitempty"` // The date and time (timestamp) when the document was created
-	Tool      *BdTool      `xml:"bd:tool"`                // The tool used to create the BOM.
-	Component *BdComponent `xml:"bd:component"`           // The component that the BOM describes.
+	XMLName   xml.Name     `xml:"metadata"`
+	Timestamp string       `xml:"timestamp,omitempty"` // The date and time (timestamp) when the document was created
+	Tools     []BdTool     `xml:"tools>tool"`          // The tool used to create the BOM.
+	Component *BdComponent `xml:"component"`           // The component that the BOM describes.
 }
 
 // BdTool represents the tool that created the BOM report.
 type BdTool struct {
-	XMLName xml.Name `xml:"bd:tool"`
-	Vendor  string   `xml:"bd:vendor,omitempty"`  // The vendor of the tool used to create the BOM.
-	Name    string   `xml:"bd:name,omitempty"`    // The name of the tool used to create the BOM.
-	Version string   `xml:"bd:version,omitempty"` // The version of the tool used to create the BOM.
+	XMLName xml.Name `xml:"tool"`
+	Vendor  string   `xml:"vendor,omitempty"`  // The vendor of the tool used to create the BOM.
+	Name    string   `xml:"name,omitempty"`    // The name of the tool used to create the BOM.
+	Version string   `xml:"version,omitempty"` // The version of the tool used to create the BOM.
 	// TODO: hashes, author, manufacture, supplier
 	// TODO: add user-defined fields for the remaining build/version parameters
 }
 
 // BdComponent represents the software/package being cataloged.
 type BdComponent struct {
-	XMLName xml.Name `xml:"bd:component"`
+	XMLName xml.Name `xml:"component"`
 	Component
 }
 
@@ -38,10 +38,12 @@ func NewBomDescriptor(name, version string, srcMetadata source.Metadata) *BomDes
 	descriptor := BomDescriptor{
 		XMLName:   xml.Name{},
 		Timestamp: time.Now().Format(time.RFC3339),
-		Tool: &BdTool{
-			Vendor:  "anchore",
-			Name:    name,
-			Version: version,
+		Tools: []BdTool{
+			{
+				Vendor:  "anchore",
+				Name:    name,
+				Version: version,
+			},
 		},
 	}
 
@@ -51,7 +53,7 @@ func NewBomDescriptor(name, version string, srcMetadata source.Metadata) *BomDes
 			Component: Component{
 				Type:    "container",
 				Name:    srcMetadata.ImageMetadata.UserInput,
-				Version: srcMetadata.ImageMetadata.Digest,
+				Version: srcMetadata.ImageMetadata.ManifestDigest,
 			},
 		}
 	case source.DirectoryScheme:

syft/presenter/cyclonedx/document.go

@@ -16,11 +16,10 @@ import (
 type Document struct {
 	XMLName       xml.Name       `xml:"bom"`
 	XMLNs         string         `xml:"xmlns,attr"`
-	XMLNsBd       string         `xml:"xmlns:bd,attr"`
 	Version       int            `xml:"version,attr"`
 	SerialNumber  string         `xml:"serialNumber,attr"`
+	BomDescriptor *BomDescriptor `xml:"metadata"`             // The BOM descriptor extension
 	Components    []Component    `xml:"components>component"` // The BOM contents
-	BomDescriptor *BomDescriptor `xml:"bd:metadata"`          // The BOM descriptor extension
 }
 
 // NewDocumentFromCatalog returns a CycloneDX Document object populated with the catalog contents.
@@ -29,7 +28,6 @@ func NewDocument(catalog *pkg.Catalog, srcMetadata source.Metadata) Document {
 
 	doc := Document{
 		XMLNs:         "http://cyclonedx.org/schema/bom/1.2",
-		XMLNsBd:       "http://cyclonedx.org/schema/ext/bom-descriptor/1.0",
 		Version:       1,
 		SerialNumber:  uuid.New().URN(),
 		BomDescriptor: NewBomDescriptor(internal.ApplicationName, versionInfo.Version, srcMetadata),

syft/presenter/cyclonedx/presenter_test.go

@@ -6,6 +6,8 @@ import (
 	"regexp"
 	"testing"
 
+	"github.com/anchore/stereoscope/pkg/filetree"
+
 	"github.com/anchore/stereoscope/pkg/imagetest"
 
 	"github.com/anchore/go-testutils"
@@ -28,7 +30,7 @@ func TestCycloneDxDirsPresenter(t *testing.T) {
 		Type:    pkg.DebPkg,
 		FoundBy: "the-cataloger-1",
 		Locations: []source.Location{
-			{Path: "/some/path/pkg1"},
+			{RealPath: "/some/path/pkg1"},
 		},
 		Metadata: pkg.DpkgMetadata{
 			Package:      "package1",
@@ -42,7 +44,7 @@ func TestCycloneDxDirsPresenter(t *testing.T) {
 		Type:    pkg.DebPkg,
 		FoundBy: "the-cataloger-2",
 		Locations: []source.Location{
-			{Path: "/some/path/pkg1"},
+			{RealPath: "/some/path/pkg1"},
 		},
 		Licenses: []string{
 			"MIT",
@@ -94,12 +96,15 @@ func TestCycloneDxImgsPresenter(t *testing.T) {
 	img, cleanup := imagetest.GetFixtureImage(t, "docker-archive", "image-simple")
 	defer cleanup()
 
+	_, ref1, _ := img.SquashedTree().File("/somefile-1.txt", filetree.FollowBasenameLinks)
+	_, ref2, _ := img.SquashedTree().File("/somefile-2.txt", filetree.FollowBasenameLinks)
+
 	// populate catalog with test data
 	catalog.Add(pkg.Package{
 		Name:    "package1",
 		Version: "1.0.1",
 		Locations: []source.Location{
-			source.NewLocationFromImage(*img.SquashedTree().File("/somefile-1.txt"), img),
+			source.NewLocationFromImage(string(ref1.RealPath), *ref1, img),
 		},
 		Type:    pkg.RpmPkg,
 		FoundBy: "the-cataloger-1",
@@ -109,7 +114,7 @@ func TestCycloneDxImgsPresenter(t *testing.T) {
 		Name:    "package2",
 		Version: "2.0.1",
 		Locations: []source.Location{
-			source.NewLocationFromImage(*img.SquashedTree().File("/somefile-2.txt"), img),
+			source.NewLocationFromImage(string(ref2.RealPath), *ref2, img),
 		},
 		Type:    pkg.RpmPkg,
 		FoundBy: "the-cataloger-2",
@@ -120,7 +125,7 @@ func TestCycloneDxImgsPresenter(t *testing.T) {
 		PURL: "the-purl-2",
 	})
 
-	s, err := source.NewFromImage(img, source.AllLayersScope, "user-image-input")
+	s, err := source.NewFromImage(img, source.SquashedScope, "user-image-input")
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -131,7 +136,7 @@ func TestCycloneDxImgsPresenter(t *testing.T) {
 	// testing. At that time, this line will no longer be necessary.
 	//
 	// This value is sourced from the "version" node in "./test-fixtures/snapshot/TestCycloneDxImgsPresenter.golden"
-	s.Metadata.ImageMetadata.Digest = "sha256:2731251dc34951c0e50fcc643b4c5f74922dad1a5d98f302b504cf46cd5d9368"
+	s.Metadata.ImageMetadata.ManifestDigest = "sha256:2731251dc34951c0e50fcc643b4c5f74922dad1a5d98f302b504cf46cd5d9368"
 
 	pres := NewPresenter(catalog, s.Metadata)
 

syft/presenter/cyclonedx/test-fixtures/snapshot/TestCycloneDxDirsPresenter.golden

@@ -1,5 +1,19 @@
 <?xml version="1.0" encoding="UTF-8"?>
-<bom xmlns="http://cyclonedx.org/schema/bom/1.2" xmlns:bd="http://cyclonedx.org/schema/ext/bom-descriptor/1.0" version="1" serialNumber="urn:uuid:aa1738d1-1c6e-45e4-a530-5a04d2c2b1a7">
+<bom xmlns="http://cyclonedx.org/schema/bom/1.2" version="1" serialNumber="urn:uuid:544b9f08-b39d-4d0c-8723-32609567e8ed">
+  <metadata>
+    <timestamp>2020-12-28T13:56:29-05:00</timestamp>
+    <tools>
+      <tool>
+        <vendor>anchore</vendor>
+        <name>syft</name>
+        <version>[not provided]</version>
+      </tool>
+    </tools>
+    <component type="file">
+      <name>/some/path</name>
+      <version></version>
+    </component>
+  </metadata>
   <components>
     <component type="library">
       <name>package1</name>
@@ -18,16 +32,4 @@
       </licenses>
     </component>
   </components>
-  <bd:metadata>
-    <bd:timestamp>2020-11-19T10:11:26-05:00</bd:timestamp>
-    <bd:tool>
-      <bd:vendor>anchore</bd:vendor>
-      <bd:name>syft</bd:name>
-      <bd:version>[not provided]</bd:version>
-    </bd:tool>
-    <bd:component type="file">
-      <name>/some/path</name>
-      <version></version>
-    </bd:component>
-  </bd:metadata>
 </bom>

syft/presenter/cyclonedx/test-fixtures/snapshot/TestCycloneDxImgsPresenter.golden

@@ -1,5 +1,19 @@
 <?xml version="1.0" encoding="UTF-8"?>
-<bom xmlns="http://cyclonedx.org/schema/bom/1.2" xmlns:bd="http://cyclonedx.org/schema/ext/bom-descriptor/1.0" version="1" serialNumber="urn:uuid:8a4c332c-365c-454d-a43c-d9344577d105">
+<bom xmlns="http://cyclonedx.org/schema/bom/1.2" version="1" serialNumber="urn:uuid:e1515688-2fd6-422a-acb2-97405a44cdfe">
+  <metadata>
+    <timestamp>2020-12-28T13:56:29-05:00</timestamp>
+    <tools>
+      <tool>
+        <vendor>anchore</vendor>
+        <name>syft</name>
+        <version>[not provided]</version>
+      </tool>
+    </tools>
+    <component type="container">
+      <name>user-image-input</name>
+      <version>sha256:2731251dc34951c0e50fcc643b4c5f74922dad1a5d98f302b504cf46cd5d9368</version>
+    </component>
+  </metadata>
   <components>
     <component type="library">
       <name>package1</name>
@@ -20,16 +34,4 @@
       <purl>the-purl-2</purl>
     </component>
   </components>
-  <bd:metadata>
-    <bd:timestamp>2020-11-19T10:11:26-05:00</bd:timestamp>
-    <bd:tool>
-      <bd:vendor>anchore</bd:vendor>
-      <bd:name>syft</bd:name>
-      <bd:version>[not provided]</bd:version>
-    </bd:tool>
-    <bd:component type="container">
-      <name>user-image-input</name>
-      <version>sha256:2731251dc34951c0e50fcc643b4c5f74922dad1a5d98f302b504cf46cd5d9368</version>
-    </bd:component>
-  </bd:metadata>
 </bom>

syft/presenter/json/document.go

@@ -1,6 +1,8 @@
 package json
 
 import (
+	"fmt"
+
 	"github.com/anchore/syft/internal"
 	"github.com/anchore/syft/internal/version"
 	"github.com/anchore/syft/syft/distro"
@@ -14,6 +16,7 @@ type Document struct {
 	Source     Source       `json:"source"`     // Source represents the original object that was cataloged
 	Distro     Distribution `json:"distro"`     // Distro represents the Linux distribution that was detected from the source
 	Descriptor Descriptor   `json:"descriptor"` // Descriptor is a block containing self-describing information about syft
+	Schema     Schema       `json:"schema"`     // Schema is a block reserved for defining the version for the shape of this JSON document and where to find the schema document to validate the shape
 }
 
 // NewDocument creates and populates a new JSON document struct from the given cataloging results.
@@ -31,6 +34,10 @@ func NewDocument(catalog *pkg.Catalog, srcMetadata source.Metadata, d *distro.Di
 			Name:    internal.ApplicationName,
 			Version: version.FromBuild().Version,
 		},
+		Schema: Schema{
+			Version: internal.JSONSchemaVersion,
+			URL:     fmt.Sprintf("https://raw.githubusercontent.com/anchore/syft/main/schema/json/schema-%s.json", internal.JSONSchemaVersion),
+		},
 	}
 
 	for _, p := range catalog.Sorted() {

syft/presenter/json/package.go

@@ -18,19 +18,19 @@ type Package struct {
 type packageBasicMetadata struct {
 	Name      string            `json:"name"`
 	Version   string            `json:"version"`
-	Type      pkg.Type          `json:"type"`
+	Type      string            `json:"type"`
 	FoundBy   string            `json:"foundBy"`
 	Locations []source.Location `json:"locations"`
 	Licenses  []string          `json:"licenses"`
-	Language  pkg.Language      `json:"language"`
+	Language  string            `json:"language"`
 	CPEs      []string          `json:"cpes"`
 	PURL      string            `json:"purl"`
 }
 
 // packageCustomMetadata contains ambiguous values (type-wise) from pkg.Package.
 type packageCustomMetadata struct {
-	MetadataType pkg.MetadataType `json:"metadataType"`
-	Metadata     interface{}      `json:"metadata"`
+	MetadataType string      `json:"metadataType"`
+	Metadata     interface{} `json:"metadata"`
 }
 
 // packageMetadataUnpacker is all values needed from Package to disambiguate ambiguous fields during json unmarshaling.
@@ -62,16 +62,16 @@ func NewPackage(p *pkg.Package) (Package, error) {
 		packageBasicMetadata: packageBasicMetadata{
 			Name:      p.Name,
 			Version:   p.Version,
-			Type:      p.Type,
+			Type:      string(p.Type),
 			FoundBy:   p.FoundBy,
 			Locations: locations,
 			Licenses:  licenses,
-			Language:  p.Language,
+			Language:  string(p.Language),
 			CPEs:      cpes,
 			PURL:      p.PURL,
 		},
 		packageCustomMetadata: packageCustomMetadata{
-			MetadataType: p.MetadataType,
+			MetadataType: string(p.MetadataType),
 			Metadata:     p.Metadata,
 		},
 	}, nil
@@ -93,12 +93,12 @@ func (a Package) ToPackage() (pkg.Package, error) {
 		Version:      a.Version,
 		FoundBy:      a.FoundBy,
 		Licenses:     a.Licenses,
-		Language:     a.Language,
+		Language:     pkg.Language(a.Language),
 		Locations:    a.Locations,
 		CPEs:         cpes,
 		PURL:         a.PURL,
-		Type:         a.Type,
-		MetadataType: a.MetadataType,
+		Type:         pkg.Type(a.Type),
+		MetadataType: pkg.MetadataType(a.MetadataType),
 		Metadata:     a.Metadata,
 	}, nil
 }
@@ -117,9 +117,9 @@ func (a *Package) UnmarshalJSON(b []byte) error {
 		return err
 	}
 
-	a.MetadataType = pkg.MetadataType(unpacker.MetadataType)
+	a.MetadataType = unpacker.MetadataType
 
-	switch a.MetadataType {
+	switch pkg.MetadataType(a.MetadataType) {
 	case pkg.RpmdbMetadataType:
 		var payload pkg.RpmdbMetadata
 		if err := json.Unmarshal(unpacker.Metadata, &payload); err != nil {

syft/presenter/json/presenter_test.go

@@ -5,6 +5,8 @@ import (
 	"flag"
 	"testing"
 
+	"github.com/anchore/stereoscope/pkg/filetree"
+
 	"github.com/anchore/go-testutils"
 	"github.com/anchore/stereoscope/pkg/imagetest"
 	"github.com/anchore/syft/syft/distro"
@@ -34,7 +36,7 @@ func TestJsonDirsPresenter(t *testing.T) {
 		Type:    pkg.PythonPkg,
 		FoundBy: "the-cataloger-1",
 		Locations: []source.Location{
-			{Path: "/some/path/pkg1"},
+			{RealPath: "/some/path/pkg1"},
 		},
 		Language:     pkg.Python,
 		MetadataType: pkg.PythonPackageMetadataType,
@@ -54,7 +56,7 @@ func TestJsonDirsPresenter(t *testing.T) {
 		Type:    pkg.DebPkg,
 		FoundBy: "the-cataloger-2",
 		Locations: []source.Location{
-			{Path: "/some/path/pkg1"},
+			{RealPath: "/some/path/pkg1"},
 		},
 		MetadataType: pkg.DpkgMetadataType,
 		Metadata: pkg.DpkgMetadata{
@@ -106,12 +108,15 @@ func TestJsonImgsPresenter(t *testing.T) {
 	catalog := pkg.NewCatalog()
 	img := imagetest.GetGoldenFixtureImage(t, testImage)
 
+	_, ref1, _ := img.SquashedTree().File("/somefile-1.txt", filetree.FollowBasenameLinks)
+	_, ref2, _ := img.SquashedTree().File("/somefile-2.txt", filetree.FollowBasenameLinks)
+
 	// populate catalog with test data
 	catalog.Add(pkg.Package{
 		Name:    "package-1",
 		Version: "1.0.1",
 		Locations: []source.Location{
-			source.NewLocationFromImage(*img.SquashedTree().File("/somefile-1.txt"), img),
+			source.NewLocationFromImage(string(ref1.RealPath), *ref1, img),
 		},
 		Type:         pkg.PythonPkg,
 		FoundBy:      "the-cataloger-1",
@@ -131,7 +136,7 @@ func TestJsonImgsPresenter(t *testing.T) {
 		Name:    "package-2",
 		Version: "2.0.1",
 		Locations: []source.Location{
-			source.NewLocationFromImage(*img.SquashedTree().File("/somefile-2.txt"), img),
+			source.NewLocationFromImage(string(ref2.RealPath), *ref2, img),
 		},
 		Type:         pkg.DebPkg,
 		FoundBy:      "the-cataloger-2",
@@ -146,7 +151,10 @@ func TestJsonImgsPresenter(t *testing.T) {
 		},
 	})
 
-	s, err := source.NewFromImage(img, source.AllLayersScope, "user-image-input")
+	// this is a hard coded value that is not given by the fixture helper and must be provided manually
+	img.Metadata.ManifestDigest = "sha256:2731251dc34951c0e50fcc643b4c5f74922dad1a5d98f302b504cf46cd5d9368"
+
+	s, err := source.NewFromImage(img, source.SquashedScope, "user-image-input")
 	var d *distro.Distro
 	pres := NewPresenter(catalog, s.Metadata, d)
 

syft/presenter/json/schema.go

@@ -0,0 +1,6 @@
+package json
+
+type Schema struct {
+	Version string `json:"version"`
+	URL     string `json:"url"`
+}

syft/presenter/json/test-fixtures/snapshot/TestJsonDirsPresenter.golden

@@ -50,6 +50,7 @@
     "package": "package-2",
     "source": "",
     "version": "2.0.1",
+    "sourceVersion": "",
     "architecture": "",
     "maintainer": "",
     "installedSize": 0,
@@ -69,5 +70,9 @@
  "descriptor": {
   "name": "syft",
   "version": "[not provided]"
+ },
+ "schema": {
+  "version": "1.0.1",
+  "url": "https://raw.githubusercontent.com/anchore/syft/main/schema/json/schema-1.0.1.json"
  }
 }

syft/presenter/json/test-fixtures/snapshot/TestJsonImgsPresenter.golden

@@ -52,6 +52,7 @@
     "package": "package-2",
     "source": "",
     "version": "2.0.1",
+    "sourceVersion": "",
     "architecture": "",
     "maintainer": "",
     "installedSize": 0,
@@ -63,7 +64,14 @@
   "type": "image",
   "target": {
    "userInput": "user-image-input",
-   "scope": "AllLayers",
+   "imageID": "sha256:2731251dc34951c0e50fcc643b4c5f74922dad1a5d98f302b504cf46cd5d9368",
+   "manifestDigest": "sha256:2731251dc34951c0e50fcc643b4c5f74922dad1a5d98f302b504cf46cd5d9368",
+   "mediaType": "application/vnd.docker.distribution.manifest.v2+json",
+   "tags": [
+    "stereoscope-fixture-image-simple:04e16e44161c8888a1a963720fd0443cbf7eef8101434c431de8725cd98cc9f7"
+   ],
+   "imageSize": 65,
+   "scope": "Squashed",
    "layers": [
     {
      "mediaType": "application/vnd.docker.image.rootfs.diff.tar.gzip",
@@ -81,12 +89,8 @@
      "size": 27
     }
    ],
-   "size": 65,
-   "digest": "sha256:2731251dc34951c0e50fcc643b4c5f74922dad1a5d98f302b504cf46cd5d9368",
-   "mediaType": "application/vnd.docker.distribution.manifest.v2+json",
-   "tags": [
-    "stereoscope-fixture-image-simple:04e16e44161c8888a1a963720fd0443cbf7eef8101434c431de8725cd98cc9f7"
-   ]
+   "manifest": "eyJzY2hlbWFWZXJzaW9uIjoyLCJtZWRpYVR5cGUiOiJhcHBsaWNhdGlvbi92bmQuZG9ja2VyLmRpc3RyaWJ1dGlvbi5tYW5pZmVzdC52Mitqc29uIiwiY29uZmlnIjp7Im1lZGlhVHlwZSI6ImFwcGxpY2F0aW9uL3ZuZC5kb2NrZXIuY29udGFpbmVyLmltYWdlLnYxK2pzb24iLCJzaXplIjoxODAxLCJkaWdlc3QiOiJzaGEyNTY6MjczMTI1MWRjMzQ5NTFjMGU1MGZjYzY0M2I0YzVmNzQ5MjJkYWQxYTVkOThmMzAyYjUwNGNmNDZjZDVkOTM2OCJ9LCJsYXllcnMiOlt7Im1lZGlhVHlwZSI6ImFwcGxpY2F0aW9uL3ZuZC5kb2NrZXIuaW1hZ2Uucm9vdGZzLmRpZmYudGFyLmd6aXAiLCJzaXplIjoyMDQ4LCJkaWdlc3QiOiJzaGEyNTY6ZTE1OGI1N2Q2ZjVhOTZlZjVmZDIyZjJmZTc2YzcwYjViYTZmZjViMjYxOWY5ZDgzMTI1YjJhYWQwNDkyYWM3YiJ9LHsibWVkaWFUeXBlIjoiYXBwbGljYXRpb24vdm5kLmRvY2tlci5pbWFnZS5yb290ZnMuZGlmZi50YXIuZ3ppcCIsInNpemUiOjIwNDgsImRpZ2VzdCI6InNoYTI1NjpkYTIxMDU2ZTdiZjQzMDhlY2VhMGMwODM2ODQ4YTdmZTkyZjM4ZmRjZjM1YmMwOWVlNmQ5OGU3YWI3YmVlZWJmIn0seyJtZWRpYVR5cGUiOiJhcHBsaWNhdGlvbi92bmQuZG9ja2VyLmltYWdlLnJvb3Rmcy5kaWZmLnRhci5nemlwIiwic2l6ZSI6MzU4NCwiZGlnZXN0Ijoic2hhMjU2OmYwZTE4YWE2MDMyYzI0NjU5YTljNzQxZmMzNmNhNTZmNTg5NzgyZWExMzIwNjFjY2Y2ZjUyYjk1MjQwM2RhOTQifV19",
+   "config": "eyJhcmNoaXRlY3R1cmUiOiJhbWQ2NCIsImNvbmZpZyI6eyJIb3N0bmFtZSI6IiIsIkRvbWFpbm5hbWUiOiIiLCJVc2VyIjoiIiwiQXR0YWNoU3RkaW4iOmZhbHNlLCJBdHRhY2hTdGRvdXQiOmZhbHNlLCJBdHRhY2hTdGRlcnIiOmZhbHNlLCJUdHkiOmZhbHNlLCJPcGVuU3RkaW4iOmZhbHNlLCJTdGRpbk9uY2UiOmZhbHNlLCJFbnYiOlsiUEFUSD0vdXNyL2xvY2FsL3NiaW46L3Vzci9sb2NhbC9iaW46L3Vzci9zYmluOi91c3IvYmluOi9zYmluOi9iaW4iXSwiQ21kIjpudWxsLCJJbWFnZSI6InNoYTI1NjpiMjQ5NTQwZjc4NDFlMmQxYTVjOTA2NzBkODhhOTgzYmI2Njc3NDNlNTc1N2FmZWI3MzBkMTQ1ZjQyOGEwN2M0IiwiVm9sdW1lcyI6bnVsbCwiV29ya2luZ0RpciI6IiIsIkVudHJ5cG9pbnQiOm51bGwsIk9uQnVpbGQiOm51bGwsIkxhYmVscyI6bnVsbH0sImNvbnRhaW5lcl9jb25maWciOnsiSG9zdG5hbWUiOiIiLCJEb21haW5uYW1lIjoiIiwiVXNlciI6IiIsIkF0dGFjaFN0ZGluIjpmYWxzZSwiQXR0YWNoU3Rkb3V0IjpmYWxzZSwiQXR0YWNoU3RkZXJyIjpmYWxzZSwiVHR5IjpmYWxzZSwiT3BlblN0ZGluIjpmYWxzZSwiU3RkaW5PbmNlIjpmYWxzZSwiRW52IjpbIlBBVEg9L3Vzci9sb2NhbC9zYmluOi91c3IvbG9jYWwvYmluOi91c3Ivc2JpbjovdXNyL2Jpbjovc2JpbjovYmluIl0sIkNtZCI6WyIvYmluL3NoIiwiLWMiLCIjKG5vcCkgQUREIGRpcjpjOTM3YzZhYTUwODkwN2UyODUwOWI2NDRhMTJmOGQ2YzY3ZDM0ZTk2OWY4M2IxNGRlZTkzZWExN2Q3NjkwMjhhIGluIC8gIl0sIkltYWdlIjoic2hhMjU2OmIyNDk1NDBmNzg0MWUyZDFhNWM5MDY3MGQ4OGE5ODNiYjY2Nzc0M2U1NzU3YWZlYjczMGQxNDVmNDI4YTA3YzQiLCJWb2x1bWVzIjpudWxsLCJXb3JraW5nRGlyIjoiIiwiRW50cnlwb2ludCI6bnVsbCwiT25CdWlsZCI6bnVsbCwiTGFiZWxzIjpudWxsfSwiY3JlYXRlZCI6IjIwMjAtMDktMjNUMTE6NTI6NDMuMTI0NjY2OFoiLCJkb2NrZXJfdmVyc2lvbiI6IjE5LjAzLjEyIiwiaGlzdG9yeSI6W3siY3JlYXRlZCI6IjIwMjAtMDktMjNUMTE6NTI6NDIuODczMDEyNloiLCJjcmVhdGVkX2J5IjoiL2Jpbi9zaCAtYyAjKG5vcCkgQUREIGZpbGU6YWMzMmRhMjNkNTFlODAxZjAyZjkyNDEyM2VkMzA5OTBlYjNmMGZlYzFiOWVkNGYwYjA2YzI0ZTg4YjljMzY5NSBpbiAvc29tZWZpbGUtMS50eHQgIn0seyJjcmVhdGVkIjoiMjAyMC0wOS0yM1QxMTo1Mjo0Mi45ODY1OTg5WiIsImNyZWF0ZWRfYnkiOiIvYmluL3NoIC1jICMobm9wKSBBREQgZmlsZTpkZjNiNzQ0ZjU0YTliMTZiOWI5YWVkNDBlM2U5OGQ5Y2EyYjQ5ZjVhNzdkOWZhOGE5NzY5MGQ3YmFmNTg4ODIwIGluIC9zb21lZmlsZS0yLnR4dCAifSx7ImNyZWF0ZWQiOiIyMDIwLTA5LTIzVDExOjUyOjQzLjEyNDY2NjhaIiwiY3JlYXRlZF9ieSI6Ii9iaW4vc2ggLWMgIyhub3ApIEFERCBkaXI6YzkzN2M2YWE1MDg5MDdlMjg1MDliNjQ0YTEyZjhkNmM2N2QzNGU5NjlmODNiMTRkZWU5M2VhMTdkNzY5MDI4YSBpbiAvICJ9XSwib3MiOiJsaW51eCIsInJvb3RmcyI6eyJ0eXBlIjoibGF5ZXJzIiwiZGlmZl9pZHMiOlsic2hhMjU2OmUxNThiNTdkNmY1YTk2ZWY1ZmQyMmYyZmU3NmM3MGI1YmE2ZmY1YjI2MTlmOWQ4MzEyNWIyYWFkMDQ5MmFjN2IiLCJzaGEyNTY6ZGEyMTA1NmU3YmY0MzA4ZWNlYTBjMDgzNjg0OGE3ZmU5MmYzOGZkY2YzNWJjMDllZTZkOThlN2FiN2JlZWViZiIsInNoYTI1NjpmMGUxOGFhNjAzMmMyNDY1OWE5Yzc0MWZjMzZjYTU2ZjU4OTc4MmVhMTMyMDYxY2NmNmY1MmI5NTI0MDNkYTk0Il19fQ=="
   }
  },
  "distro": {
@@ -97,5 +101,9 @@
  "descriptor": {
   "name": "syft",
   "version": "[not provided]"
+ },
+ "schema": {
+  "version": "1.0.1",
+  "url": "https://raw.githubusercontent.com/anchore/syft/main/schema/json/schema-1.0.1.json"
  }
 }

syft/presenter/table/presenter_test.go

@@ -5,6 +5,8 @@ import (
 	"flag"
 	"testing"
 
+	"github.com/anchore/stereoscope/pkg/filetree"
+
 	"github.com/go-test/deep"
 
 	"github.com/anchore/go-testutils"
@@ -25,12 +27,15 @@ func TestTablePresenter(t *testing.T) {
 	img, cleanup := imagetest.GetFixtureImage(t, "docker-archive", testImage)
 	defer cleanup()
 
+	_, ref1, _ := img.SquashedTree().File("/somefile-1.txt", filetree.FollowBasenameLinks)
+	_, ref2, _ := img.SquashedTree().File("/somefile-2.txt", filetree.FollowBasenameLinks)
+
 	// populate catalog with test data
 	catalog.Add(pkg.Package{
 		Name:    "package-1",
 		Version: "1.0.1",
 		Locations: []source.Location{
-			source.NewLocationFromImage(*img.SquashedTree().File("/somefile-1.txt"), img),
+			source.NewLocationFromImage(string(ref1.RealPath), *ref1, img),
 		},
 		Type: pkg.DebPkg,
 	})
@@ -38,7 +43,7 @@ func TestTablePresenter(t *testing.T) {
 		Name:    "package-2",
 		Version: "2.0.1",
 		Locations: []source.Location{
-			source.NewLocationFromImage(*img.SquashedTree().File("/somefile-2.txt"), img),
+			source.NewLocationFromImage(string(ref2.RealPath), *ref2, img),
 		},
 		Type: pkg.DebPkg,
 	})

syft/presenter/text/presenter_test.go

@@ -5,6 +5,8 @@ import (
 	"flag"
 	"testing"
 
+	"github.com/anchore/stereoscope/pkg/filetree"
+
 	"github.com/anchore/go-testutils"
 	"github.com/anchore/stereoscope/pkg/imagetest"
 	"github.com/anchore/syft/syft/pkg"
@@ -70,12 +72,15 @@ func TestTextImgPresenter(t *testing.T) {
 	img, cleanup := imagetest.GetFixtureImage(t, "docker-archive", "image-simple")
 	defer cleanup()
 
+	_, ref1, _ := img.SquashedTree().File("/somefile-1.txt", filetree.FollowBasenameLinks)
+	_, ref2, _ := img.SquashedTree().File("/somefile-2.txt", filetree.FollowBasenameLinks)
+
 	// populate catalog with test data
 	catalog.Add(pkg.Package{
 		Name:    "package-1",
 		Version: "1.0.1",
 		Locations: []source.Location{
-			source.NewLocationFromImage(*img.SquashedTree().File("/somefile-1.txt"), img),
+			source.NewLocationFromImage(string(ref1.RealPath), *ref1, img),
 		},
 		FoundBy: "dpkg",
 		Type:    pkg.DebPkg,
@@ -84,7 +89,7 @@ func TestTextImgPresenter(t *testing.T) {
 		Name:    "package-2",
 		Version: "2.0.1",
 		Locations: []source.Location{
-			source.NewLocationFromImage(*img.SquashedTree().File("/somefile-2.txt"), img),
+			source.NewLocationFromImage(string(ref2.RealPath), *ref2, img),
 		},
 		FoundBy:  "dpkg",
 		Metadata: PackageInfo{Name: "package-2", Version: "1.0.2"},
@@ -97,7 +102,7 @@ func TestTextImgPresenter(t *testing.T) {
 		l.Metadata.Digest = "sha256:ad8ecdc058976c07e7e347cb89fa9ad86a294b5ceaae6d09713fb035f84115abf3c4a2388a4af3aa60f13b94f4c6846930bdf53"
 	}
 
-	s, err := source.NewFromImage(img, source.AllLayersScope, "user-image-input")
+	s, err := source.NewFromImage(img, source.SquashedScope, "user-image-input")
 	if err != nil {
 		t.Fatal(err)
 	}

syft/source/all_layers_resolver.go

@@ -2,10 +2,15 @@ package source
 
 import (
 	"archive/tar"
+	"bytes"
 	"fmt"
+	"io"
+	"io/ioutil"
 
 	"github.com/anchore/stereoscope/pkg/file"
+	"github.com/anchore/stereoscope/pkg/filetree"
 	"github.com/anchore/stereoscope/pkg/image"
+	"github.com/anchore/syft/internal/log"
 )
 
 var _ Resolver = (*AllLayersResolver)(nil)
@@ -32,6 +37,18 @@ func NewAllLayersResolver(img *image.Image) (*AllLayersResolver, error) {
 	}, nil
 }
 
+// HasPath indicates if the given path exists in the underlying source.
+func (r *AllLayersResolver) HasPath(path string) bool {
+	p := file.Path(path)
+	for _, layerIdx := range r.layers {
+		tree := r.img.Layers[layerIdx].Tree
+		if tree.HasPath(p) {
+			return true
+		}
+	}
+	return false
+}
+
 func (r *AllLayersResolver) fileByRef(ref file.Reference, uniqueFileIDs file.ReferenceSet, layerIdx int) ([]file.Reference, error) {
 	uniqueFiles := make([]file.Reference, 0)
 
@@ -70,19 +87,22 @@ func (r *AllLayersResolver) FilesByPath(paths ...string) ([]Location, error) {
 	for _, path := range paths {
 		for idx, layerIdx := range r.layers {
 			tree := r.img.Layers[layerIdx].Tree
-			ref := tree.File(file.Path(path))
+			_, ref, err := tree.File(file.Path(path), filetree.FollowBasenameLinks, filetree.DoNotFollowDeadBasenameLinks)
+			if err != nil {
+				return nil, err
+			}
 			if ref == nil {
 				// no file found, keep looking through layers
 				continue
 			}
 
 			// don't consider directories (special case: there is no path information for /)
-			if ref.Path == "/" {
+			if ref.RealPath == "/" {
 				continue
 			} else if r.img.FileCatalog.Exists(*ref) {
 				metadata, err := r.img.FileCatalog.Get(*ref)
 				if err != nil {
-					return nil, fmt.Errorf("unable to get file metadata for path=%q: %w", ref.Path, err)
+					return nil, fmt.Errorf("unable to get file metadata for path=%q: %w", ref.RealPath, err)
 				}
 				if metadata.Metadata.IsDir {
 					continue
@@ -94,7 +114,7 @@ func (r *AllLayersResolver) FilesByPath(paths ...string) ([]Location, error) {
 				return nil, err
 			}
 			for _, result := range results {
-				uniqueLocations = append(uniqueLocations, NewLocationFromImage(result, r.img))
+				uniqueLocations = append(uniqueLocations, NewLocationFromImage(path, result, r.img))
 			}
 		}
 	}
@@ -109,31 +129,31 @@ func (r *AllLayersResolver) FilesByGlob(patterns ...string) ([]Location, error)
 
 	for _, pattern := range patterns {
 		for idx, layerIdx := range r.layers {
-			refs, err := r.img.Layers[layerIdx].Tree.FilesByGlob(pattern)
+			results, err := r.img.Layers[layerIdx].Tree.FilesByGlob(pattern, filetree.DoNotFollowDeadBasenameLinks)
 			if err != nil {
 				return nil, fmt.Errorf("failed to resolve files by glob (%s): %w", pattern, err)
 			}
 
-			for _, ref := range refs {
+			for _, result := range results {
 				// don't consider directories (special case: there is no path information for /)
-				if ref.Path == "/" {
+				if result.RealPath == "/" {
 					continue
-				} else if r.img.FileCatalog.Exists(ref) {
-					metadata, err := r.img.FileCatalog.Get(ref)
+				} else if r.img.FileCatalog.Exists(result.Reference) {
+					metadata, err := r.img.FileCatalog.Get(result.Reference)
 					if err != nil {
-						return nil, fmt.Errorf("unable to get file metadata for path=%q: %w", ref.Path, err)
+						return nil, fmt.Errorf("unable to get file metadata for path=%q: %w", result.MatchPath, err)
 					}
 					if metadata.Metadata.IsDir {
 						continue
 					}
 				}
 
-				results, err := r.fileByRef(ref, uniqueFileIDs, idx)
+				refResults, err := r.fileByRef(result.Reference, uniqueFileIDs, idx)
 				if err != nil {
 					return nil, err
 				}
-				for _, result := range results {
-					uniqueLocations = append(uniqueLocations, NewLocationFromImage(result, r.img))
+				for _, refResult := range refResults {
+					uniqueLocations = append(uniqueLocations, NewLocationFromImage(string(result.MatchPath), refResult, r.img))
 				}
 			}
 		}
@@ -150,37 +170,41 @@ func (r *AllLayersResolver) RelativeFileByPath(location Location, path string) *
 		return nil
 	}
 
-	relativeRef := entry.Source.SquashedTree.File(file.Path(path))
-	if relativeRef == nil {
+	exists, relativeRef, err := entry.Layer.SquashedTree.File(file.Path(path), filetree.FollowBasenameLinks)
+	if err != nil {
+		log.Errorf("failed to find path=%q in squash: %+w", path, err)
+		return nil
+	}
+	if !exists && relativeRef == nil {
 		return nil
 	}
 
-	relativeLocation := NewLocationFromImage(*relativeRef, r.img)
+	relativeLocation := NewLocationFromImage(path, *relativeRef, r.img)
 
 	return &relativeLocation
 }
 
 // MultipleFileContentsByLocation returns the file contents for all file.References relative to the image. Note that a
 // file.Reference is a path relative to a particular layer.
-func (r *AllLayersResolver) MultipleFileContentsByLocation(locations []Location) (map[Location]string, error) {
+func (r *AllLayersResolver) MultipleFileContentsByLocation(locations []Location) (map[Location]io.ReadCloser, error) {
 	return mapLocationRefs(r.img.MultipleFileContentsByRef, locations)
 }
 
 // FileContentsByLocation fetches file contents for a single file reference, irregardless of the source layer.
 // If the path does not exist an error is returned.
-func (r *AllLayersResolver) FileContentsByLocation(location Location) (string, error) {
+func (r *AllLayersResolver) FileContentsByLocation(location Location) (io.ReadCloser, error) {
 	return r.img.FileContentsByRef(location.ref)
 }
 
-type multiContentFetcher func(refs ...file.Reference) (map[file.Reference]string, error)
+type multiContentFetcher func(refs ...file.Reference) (map[file.Reference]io.ReadCloser, error)
 
-func mapLocationRefs(callback multiContentFetcher, locations []Location) (map[Location]string, error) {
+func mapLocationRefs(callback multiContentFetcher, locations []Location) (map[Location]io.ReadCloser, error) {
 	var fileRefs = make([]file.Reference, len(locations))
-	var locationByRefs = make(map[file.Reference]Location)
-	var results = make(map[Location]string)
+	var locationByRefs = make(map[file.Reference][]Location)
+	var results = make(map[Location]io.ReadCloser)
 
 	for i, location := range locations {
-		locationByRefs[location.ref] = location
+		locationByRefs[location.ref] = append(locationByRefs[location.ref], location)
 		fileRefs[i] = location.ref
 	}
 
@@ -189,8 +213,26 @@ func mapLocationRefs(callback multiContentFetcher, locations []Location) (map[Lo
 		return nil, err
 	}
 
+	// TODO: this is not tested, we need a test case that covers a mapLocationRefs which has multiple Locations with the same reference in the request. The io.Reader should be copied.
 	for ref, content := range contentsByRef {
-		results[locationByRefs[ref]] = content
+		mappedLocations := locationByRefs[ref]
+		switch {
+		case len(mappedLocations) > 1:
+			// TODO: fixme... this can lead to lots of unexpected memory usage in unusual circumstances (cache is not leveraged for large files).
+			// stereoscope wont duplicate content requests if the caller asks for the same file multiple times... thats up to the caller
+			contentsBytes, err := ioutil.ReadAll(content)
+			if err != nil {
+				return nil, fmt.Errorf("unable to read ref=%+v :%w", ref, err)
+			}
+			for _, loc := range mappedLocations {
+				results[loc] = ioutil.NopCloser(bytes.NewReader(contentsBytes))
+			}
+
+		case len(mappedLocations) == 1:
+			results[locationByRefs[ref][0]] = content
+		default:
+			return nil, fmt.Errorf("unexpected ref-location count=%d for ref=%v", len(mappedLocations), ref)
+		}
 	}
 	return results, nil
 }

syft/source/all_layers_resolver_test.go

@@ -13,9 +13,10 @@ type resolution struct {
 
 func TestAllLayersResolver_FilesByPath(t *testing.T) {
 	cases := []struct {
-		name        string
-		linkPath    string
-		resolutions []resolution
+		name                 string
+		linkPath             string
+		resolutions          []resolution
+		forcePositiveHasPath bool
 	}{
 		{
 			name:     "link with previous data",
@@ -41,10 +42,6 @@ func TestAllLayersResolver_FilesByPath(t *testing.T) {
 			name:     "link with overridden data",
 			linkPath: "/link-2",
 			resolutions: []resolution{
-				{
-					layer: 3,
-					path:  "/link-2",
-				},
 				{
 					layer: 4,
 					path:  "/file-2.txt",
@@ -70,19 +67,17 @@ func TestAllLayersResolver_FilesByPath(t *testing.T) {
 			},
 		},
 		{
-			name:     "dead link",
-			linkPath: "/link-dead",
-			resolutions: []resolution{
-				{
-					layer: 8,
-					path:  "/link-dead",
-				},
-			},
+			name:                 "dead link",
+			linkPath:             "/link-dead",
+			resolutions:          []resolution{},
+			forcePositiveHasPath: true,
 		},
 		{
 			name:        "ignore directories",
 			linkPath:    "/bin",
 			resolutions: []resolution{},
+			// directories don't resolve BUT do exist
+			forcePositiveHasPath: true,
 		},
 	}
 	for _, c := range cases {
@@ -95,6 +90,17 @@ func TestAllLayersResolver_FilesByPath(t *testing.T) {
 				t.Fatalf("could not create resolver: %+v", err)
 			}
 
+			hasPath := resolver.HasPath(c.linkPath)
+			if !c.forcePositiveHasPath {
+				if len(c.resolutions) > 0 && !hasPath {
+					t.Errorf("expected HasPath() to indicate existance, but did not")
+				} else if len(c.resolutions) == 0 && hasPath {
+					t.Errorf("expeced HasPath() to NOT indicate existance, but does")
+				}
+			} else if !hasPath {
+				t.Errorf("expected HasPath() to indicate existance, but did not (force path)")
+			}
+
 			refs, err := resolver.FilesByPath(c.linkPath)
 			if err != nil {
 				t.Fatalf("could not use resolver: %+v", err)
@@ -107,8 +113,12 @@ func TestAllLayersResolver_FilesByPath(t *testing.T) {
 			for idx, actual := range refs {
 				expected := c.resolutions[idx]
 
-				if actual.Path != expected.path {
-					t.Errorf("bad resolve path: '%s'!='%s'", actual.Path, expected.path)
+				if string(actual.ref.RealPath) != expected.path {
+					t.Errorf("bad resolve path: '%s'!='%s'", string(actual.ref.RealPath), expected.path)
+				}
+
+				if expected.path != "" && string(actual.ref.RealPath) != actual.RealPath {
+					t.Errorf("we should always prefer real paths over ones with links")
 				}
 
 				entry, err := img.FileCatalog.Get(actual.ref)
@@ -116,8 +126,8 @@ func TestAllLayersResolver_FilesByPath(t *testing.T) {
 					t.Fatalf("failed to get metadata: %+v", err)
 				}
 
-				if entry.Source.Metadata.Index != expected.layer {
-					t.Errorf("bad resolve layer: '%d'!='%d'", entry.Source.Metadata.Index, expected.layer)
+				if entry.Layer.Metadata.Index != expected.layer {
+					t.Errorf("bad resolve layer: '%d'!='%d'", entry.Layer.Metadata.Index, expected.layer)
 				}
 			}
 		})
@@ -132,7 +142,7 @@ func TestAllLayersResolver_FilesByGlob(t *testing.T) {
 	}{
 		{
 			name: "link with previous data",
-			glob: "**ink-1",
+			glob: "**/*ink-1",
 			resolutions: []resolution{
 				{
 					layer: 1,
@@ -142,7 +152,7 @@ func TestAllLayersResolver_FilesByGlob(t *testing.T) {
 		},
 		{
 			name: "link with in layer data",
-			glob: "**nk-within",
+			glob: "**/*nk-within",
 			resolutions: []resolution{
 				{
 					layer: 5,
@@ -152,12 +162,8 @@ func TestAllLayersResolver_FilesByGlob(t *testing.T) {
 		},
 		{
 			name: "link with overridden data",
-			glob: "**ink-2",
+			glob: "**/*ink-2",
 			resolutions: []resolution{
-				{
-					layer: 3,
-					path:  "/link-2",
-				},
 				{
 					layer: 4,
 					path:  "/file-2.txt",
@@ -170,7 +176,7 @@ func TestAllLayersResolver_FilesByGlob(t *testing.T) {
 		},
 		{
 			name: "indirect link (with overridden data)",
-			glob: "**nk-indirect",
+			glob: "**/*nk-indirect",
 			resolutions: []resolution{
 				{
 					layer: 4,
@@ -183,14 +189,9 @@ func TestAllLayersResolver_FilesByGlob(t *testing.T) {
 			},
 		},
 		{
-			name: "dead link",
-			glob: "**k-dead",
-			resolutions: []resolution{
-				{
-					layer: 8,
-					path:  "/link-dead",
-				},
-			},
+			name:        "dead link",
+			glob:        "**/*k-dead",
+			resolutions: []resolution{},
 		},
 		{
 			name:        "ignore directories",
@@ -220,8 +221,12 @@ func TestAllLayersResolver_FilesByGlob(t *testing.T) {
 			for idx, actual := range refs {
 				expected := c.resolutions[idx]
 
-				if actual.Path != expected.path {
-					t.Errorf("bad resolve path: '%s'!='%s'", actual.Path, expected.path)
+				if string(actual.ref.RealPath) != expected.path {
+					t.Errorf("bad resolve path: '%s'!='%s'", string(actual.ref.RealPath), expected.path)
+				}
+
+				if expected.path != "" && string(actual.ref.RealPath) != actual.RealPath {
+					t.Errorf("we should always prefer real paths over ones with links")
 				}
 
 				entry, err := img.FileCatalog.Get(actual.ref)
@@ -229,8 +234,8 @@ func TestAllLayersResolver_FilesByGlob(t *testing.T) {
 					t.Fatalf("failed to get metadata: %+v", err)
 				}
 
-				if entry.Source.Metadata.Index != expected.layer {
-					t.Errorf("bad resolve layer: '%d'!='%d'", entry.Source.Metadata.Index, expected.layer)
+				if entry.Layer.Metadata.Index != expected.layer {
+					t.Errorf("bad resolve layer: '%d'!='%d'", entry.Layer.Metadata.Index, expected.layer)
 				}
 			}
 		})

syft/source/content_requester.go

@@ -0,0 +1,56 @@
+package source
+
+import "sync"
+
+// ContentRequester is an object tailored for taking source.Location objects which file contents will be resolved
+// upon invoking Execute().
+type ContentRequester struct {
+	request map[Location][]*FileData
+	lock    sync.Mutex
+}
+
+// NewContentRequester creates a new ContentRequester object with the given initial request data.
+func NewContentRequester(data ...*FileData) *ContentRequester {
+	requester := &ContentRequester{
+		request: make(map[Location][]*FileData),
+	}
+	for _, d := range data {
+		requester.Add(d)
+	}
+	return requester
+}
+
+// Add appends a new single FileData containing a source.Location to later have the contents fetched and stored within
+// the given FileData object.
+func (r *ContentRequester) Add(data *FileData) {
+	r.lock.Lock()
+	defer r.lock.Unlock()
+
+	r.request[data.Location] = append(r.request[data.Location], data)
+}
+
+// Execute takes the previously provided source.Location's and resolves the file contents, storing the results within
+// the previously provided FileData objects.
+func (r *ContentRequester) Execute(resolver ContentResolver) error {
+	r.lock.Lock()
+	defer r.lock.Unlock()
+
+	var locations = make([]Location, len(r.request))
+	idx := 0
+	for l := range r.request {
+		locations[idx] = l
+		idx++
+	}
+
+	response, err := resolver.MultipleFileContentsByLocation(locations)
+	if err != nil {
+		return err
+	}
+
+	for l, contents := range response {
+		for i := range r.request[l] {
+			r.request[l][i].Contents = contents
+		}
+	}
+	return nil
+}

syft/source/content_requester_test.go

@@ -0,0 +1,74 @@
+package source
+
+import (
+	"io/ioutil"
+	"testing"
+
+	"github.com/anchore/stereoscope/pkg/imagetest"
+	"github.com/sergi/go-diff/diffmatchpatch"
+)
+
+func TestContentRequester(t *testing.T) {
+	tests := []struct {
+		fixture          string
+		expectedContents map[string]string
+	}{
+		{
+			fixture: "image-simple",
+			expectedContents: map[string]string{
+				"/somefile-1.txt":           "this file has contents",
+				"/somefile-2.txt":           "file-2 contents!",
+				"/really/nested/file-3.txt": "another file!\nwith lines...",
+			},
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.fixture, func(t *testing.T) {
+			img, cleanup := imagetest.GetFixtureImage(t, "docker-archive", "image-simple")
+			defer cleanup()
+
+			resolver, err := NewAllLayersResolver(img)
+			if err != nil {
+				t.Fatalf("could not create resolver: %+v", err)
+			}
+
+			var data []*FileData
+			for path := range test.expectedContents {
+
+				locations, err := resolver.FilesByPath(path)
+				if err != nil {
+					t.Fatalf("could not build request: %+v", err)
+				}
+				if len(locations) != 1 {
+					t.Fatalf("bad resolver paths: %+v", locations)
+				}
+
+				data = append(data, &FileData{
+					Location: locations[0],
+				})
+			}
+
+			if err := NewContentRequester(data...).Execute(resolver); err != nil {
+				t.Fatalf("could not execute request: %+v", err)
+			}
+
+			for _, entry := range data {
+				if expected, ok := test.expectedContents[entry.Location.RealPath]; ok {
+					actualBytes, err := ioutil.ReadAll(entry.Contents)
+					if err != nil {
+						t.Fatalf("could not read %q: %+v", entry.Location.RealPath, err)
+					}
+					for expected != string(actualBytes) {
+						t.Errorf("mismatched contents for %q", entry.Location.RealPath)
+						dmp := diffmatchpatch.New()
+						diffs := dmp.DiffMain(expected, string(actualBytes), true)
+						t.Errorf("diff: %s", dmp.DiffPrettyText(diffs))
+					}
+					continue
+				}
+				t.Errorf("could not find %q", entry.Location.RealPath)
+			}
+		})
+	}
+}

syft/source/directory_resolver.go

@@ -2,13 +2,14 @@ package source
 
 import (
 	"fmt"
-	"io/ioutil"
+	"io"
 	"os"
 	"path"
 	"path/filepath"
 
+	"github.com/anchore/stereoscope/pkg/file"
 	"github.com/anchore/syft/internal/log"
-	"github.com/bmatcuk/doublestar"
+	"github.com/bmatcuk/doublestar/v2"
 )
 
 var _ Resolver = (*DirectoryResolver)(nil)
@@ -18,27 +19,37 @@ type DirectoryResolver struct {
 	Path string
 }
 
+func (r DirectoryResolver) requestPath(userPath string) string {
+	fullPath := userPath
+	if filepath.IsAbs(fullPath) {
+		// a path relative to root should be prefixed with the resolvers directory path, otherwise it should be left as is
+		fullPath = path.Join(r.Path, fullPath)
+	}
+	return fullPath
+}
+
+// HasPath indicates if the given path exists in the underlying source.
+func (r *DirectoryResolver) HasPath(userPath string) bool {
+	_, err := os.Stat(r.requestPath(userPath))
+	return !os.IsNotExist(err)
+}
+
 // Stringer to represent a directory path data source
-func (s DirectoryResolver) String() string {
-	return fmt.Sprintf("dir:%s", s.Path)
+func (r DirectoryResolver) String() string {
+	return fmt.Sprintf("dir:%s", r.Path)
 }
 
 // FilesByPath returns all file.References that match the given paths from the directory.
-func (s DirectoryResolver) FilesByPath(userPaths ...string) ([]Location, error) {
+func (r DirectoryResolver) FilesByPath(userPaths ...string) ([]Location, error) {
 	var references = make([]Location, 0)
 
 	for _, userPath := range userPaths {
-		userStrPath := userPath
-
-		if filepath.IsAbs(userStrPath) {
-			// a path relative to root should be prefixed with the resolvers directory path, otherwise it should be left as is
-			userStrPath = path.Join(s.Path, userStrPath)
-		}
+		userStrPath := r.requestPath(userPath)
 		fileMeta, err := os.Stat(userStrPath)
 		if os.IsNotExist(err) {
 			continue
 		} else if err != nil {
-			log.Errorf("path (%s) is not valid: %v", userStrPath, err)
+			log.Errorf("path (%r) is not valid: %v", userStrPath, err)
 		}
 
 		// don't consider directories
@@ -53,11 +64,11 @@ func (s DirectoryResolver) FilesByPath(userPaths ...string) ([]Location, error)
 }
 
 // FilesByGlob returns all file.References that match the given path glob pattern from any layer in the image.
-func (s DirectoryResolver) FilesByGlob(patterns ...string) ([]Location, error) {
+func (r DirectoryResolver) FilesByGlob(patterns ...string) ([]Location, error) {
 	result := make([]Location, 0)
 
 	for _, pattern := range patterns {
-		pathPattern := path.Join(s.Path, pattern)
+		pathPattern := path.Join(r.Path, pattern)
 		pathMatches, err := doublestar.Glob(pathPattern)
 		if err != nil {
 			return nil, err
@@ -83,8 +94,8 @@ func (s DirectoryResolver) FilesByGlob(patterns ...string) ([]Location, error) {
 // RelativeFileByPath fetches a single file at the given path relative to the layer squash of the given reference.
 // This is helpful when attempting to find a file that is in the same layer or lower as another file. For the
 // DirectoryResolver, this is a simple path lookup.
-func (s *DirectoryResolver) RelativeFileByPath(_ Location, path string) *Location {
-	paths, err := s.FilesByPath(path)
+func (r *DirectoryResolver) RelativeFileByPath(_ Location, path string) *Location {
+	paths, err := r.FilesByPath(path)
 	if err != nil {
 		return nil
 	}
@@ -96,35 +107,16 @@ func (s *DirectoryResolver) RelativeFileByPath(_ Location, path string) *Locatio
 }
 
 // MultipleFileContentsByLocation returns the file contents for all file.References relative a directory.
-func (s DirectoryResolver) MultipleFileContentsByLocation(locations []Location) (map[Location]string, error) {
-	refContents := make(map[Location]string)
+func (r DirectoryResolver) MultipleFileContentsByLocation(locations []Location) (map[Location]io.ReadCloser, error) {
+	refContents := make(map[Location]io.ReadCloser)
 	for _, location := range locations {
-		contents, err := fileContents(location.Path)
-
-		if err != nil {
-			return nil, fmt.Errorf("could not read contents of file: %s", location.Path)
-		}
-		refContents[location] = string(contents)
+		refContents[location] = file.NewDeferredReadCloser(location.RealPath)
 	}
 	return refContents, nil
 }
 
 // FileContentsByLocation fetches file contents for a single file reference relative to a directory.
 // If the path does not exist an error is returned.
-func (s DirectoryResolver) FileContentsByLocation(location Location) (string, error) {
-	contents, err := fileContents(location.Path)
-	if err != nil {
-		return "", fmt.Errorf("could not read contents of file: %s", location.Path)
-	}
-
-	return string(contents), nil
-}
-
-func fileContents(path string) ([]byte, error) {
-	contents, err := ioutil.ReadFile(path)
-
-	if err != nil {
-		return nil, err
-	}
-	return contents, nil
+func (r DirectoryResolver) FileContentsByLocation(location Location) (io.ReadCloser, error) {
+	return file.NewDeferredReadCloser(location.RealPath), nil
 }

syft/source/directory_resolver_test.go

@@ -6,11 +6,12 @@ import (
 
 func TestDirectoryResolver_FilesByPath(t *testing.T) {
 	cases := []struct {
-		name     string
-		root     string
-		input    string
-		expected string
-		refCount int
+		name                 string
+		root                 string
+		input                string
+		expected             string
+		refCount             int
+		forcePositiveHasPath bool
 	}{
 		{
 			name:     "finds a file (relative)",
@@ -47,15 +48,28 @@ func TestDirectoryResolver_FilesByPath(t *testing.T) {
 			refCount: 1,
 		},
 		{
-			name:     "directories ignored",
-			root:     "./test-fixtures/",
-			input:    "/image-symlinks",
-			refCount: 0,
+			name:                 "directories ignored",
+			root:                 "./test-fixtures/",
+			input:                "/image-symlinks",
+			refCount:             0,
+			forcePositiveHasPath: true,
 		},
 	}
 	for _, c := range cases {
 		t.Run(c.name, func(t *testing.T) {
 			resolver := DirectoryResolver{c.root}
+
+			hasPath := resolver.HasPath(c.input)
+			if !c.forcePositiveHasPath {
+				if c.refCount != 0 && !hasPath {
+					t.Errorf("expected HasPath() to indicate existance, but did not")
+				} else if c.refCount == 0 && hasPath {
+					t.Errorf("expeced HasPath() to NOT indicate existance, but does")
+				}
+			} else if !hasPath {
+				t.Errorf("expected HasPath() to indicate existance, but did not (force path)")
+			}
+
 			refs, err := resolver.FilesByPath(c.input)
 			if err != nil {
 				t.Fatalf("could not use resolver: %+v, %+v", err, refs)
@@ -66,8 +80,8 @@ func TestDirectoryResolver_FilesByPath(t *testing.T) {
 			}
 
 			for _, actual := range refs {
-				if actual.Path != c.expected {
-					t.Errorf("bad resolve path: '%s'!='%s'", actual.Path, c.expected)
+				if actual.RealPath != c.expected {
+					t.Errorf("bad resolve path: '%s'!='%s'", actual.RealPath, c.expected)
 				}
 			}
 		})
@@ -171,8 +185,9 @@ func TestDirectoryResolver_FilesByGlobMultiple(t *testing.T) {
 			t.Fatalf("could not use resolver: %+v, %+v", err, refs)
 		}
 
-		if len(refs) != 2 {
-			t.Errorf("unexpected number of refs: %d != 2", len(refs))
+		expected := 2
+		if len(refs) != expected {
+			t.Errorf("unexpected number of refs: %d != %d", len(refs), expected)
 		}
 
 	})
@@ -180,15 +195,16 @@ func TestDirectoryResolver_FilesByGlobMultiple(t *testing.T) {
 
 func TestDirectoryResolver_FilesByGlobRecursive(t *testing.T) {
 	t.Run("finds multiple matching files", func(t *testing.T) {
-		resolver := DirectoryResolver{"test-fixtures"}
+		resolver := DirectoryResolver{"test-fixtures/image-symlinks"}
 		refs, err := resolver.FilesByGlob("**/*.txt")
 
 		if err != nil {
 			t.Fatalf("could not use resolver: %+v, %+v", err, refs)
 		}
 
-		if len(refs) != 4 {
-			t.Errorf("unexpected number of refs: %d != 4", len(refs))
+		expected := 6
+		if len(refs) != expected {
+			t.Errorf("unexpected number of refs: %d != %d", len(refs), expected)
 		}
 
 	})
@@ -202,8 +218,9 @@ func TestDirectoryResolver_FilesByGlobSingle(t *testing.T) {
 			t.Fatalf("could not use resolver: %+v, %+v", err, refs)
 		}
 
-		if len(refs) != 1 {
-			t.Errorf("unexpected number of refs: %d != 1", len(refs))
+		expected := 1
+		if len(refs) != expected {
+			t.Errorf("unexpected number of refs: %d != %d", len(refs), expected)
 		}
 
 	})

syft/source/file_data.go

@@ -0,0 +1,8 @@
+package source
+
+import "io"
+
+type FileData struct {
+	Location Location
+	Contents io.ReadCloser
+}

syft/source/image_metadata.go

@@ -5,13 +5,16 @@ import "github.com/anchore/stereoscope/pkg/image"
 // ImageMetadata represents all static metadata that defines what a container image is. This is useful to later describe
 // "what" was cataloged without needing the more complicated stereoscope Image objects or Resolver objects.
 type ImageMetadata struct {
-	UserInput string          `json:"userInput"`
-	Scope     Scope           `json:"scope"` // specific perspective to catalog
-	Layers    []LayerMetadata `json:"layers"`
-	Size      int64           `json:"size"`
-	Digest    string          `json:"digest"`
-	MediaType string          `json:"mediaType"`
-	Tags      []string        `json:"tags"`
+	UserInput      string          `json:"userInput"`
+	ID             string          `json:"imageID"`
+	ManifestDigest string          `json:"manifestDigest"`
+	MediaType      string          `json:"mediaType"`
+	Tags           []string        `json:"tags"`
+	Size           int64           `json:"imageSize"`
+	Scope          Scope           `json:"scope"` // specific perspective to catalog
+	Layers         []LayerMetadata `json:"layers"`
+	RawManifest    []byte          `json:"manifest"`
+	RawConfig      []byte          `json:"config"`
 }
 
 // LayerMetadata represents all static metadata that defines what a container image layer is.
@@ -29,13 +32,16 @@ func NewImageMetadata(img *image.Image, userInput string, scope Scope) ImageMeta
 		tags[idx] = tag.String()
 	}
 	theImg := ImageMetadata{
-		UserInput: userInput,
-		Scope:     scope,
-		Digest:    img.Metadata.Digest,
-		Size:      img.Metadata.Size,
-		MediaType: string(img.Metadata.MediaType),
-		Tags:      tags,
-		Layers:    make([]LayerMetadata, len(img.Layers)),
+		ID:             img.Metadata.ID,
+		UserInput:      userInput,
+		Scope:          scope,
+		ManifestDigest: img.Metadata.ManifestDigest,
+		Size:           img.Metadata.Size,
+		MediaType:      string(img.Metadata.MediaType),
+		Tags:           tags,
+		Layers:         make([]LayerMetadata, len(img.Layers)),
+		RawConfig:      img.Metadata.RawConfig,
+		RawManifest:    img.Metadata.RawManifest,
 	}
 
 	// populate image metadata

syft/source/image_squash_resolver.go

@@ -2,8 +2,10 @@ package source
 
 import (
 	"fmt"
+	"io"
 
 	"github.com/anchore/stereoscope/pkg/file"
+	"github.com/anchore/stereoscope/pkg/filetree"
 	"github.com/anchore/stereoscope/pkg/image"
 )
 
@@ -22,6 +24,11 @@ func NewImageSquashResolver(img *image.Image) (*ImageSquashResolver, error) {
 	return &ImageSquashResolver{img: img}, nil
 }
 
+// HasPath indicates if the given path exists in the underlying source.
+func (r *ImageSquashResolver) HasPath(path string) bool {
+	return r.img.SquashedTree().HasPath(file.Path(path))
+}
+
 // FilesByPath returns all file.References that match the given paths within the squashed representation of the image.
 func (r *ImageSquashResolver) FilesByPath(paths ...string) ([]Location, error) {
 	uniqueFileIDs := file.NewFileReferenceSet()
@@ -29,19 +36,22 @@ func (r *ImageSquashResolver) FilesByPath(paths ...string) ([]Location, error) {
 
 	for _, path := range paths {
 		tree := r.img.SquashedTree()
-		ref := tree.File(file.Path(path))
+		_, ref, err := tree.File(file.Path(path), filetree.FollowBasenameLinks)
+		if err != nil {
+			return nil, err
+		}
 		if ref == nil {
 			// no file found, keep looking through layers
 			continue
 		}
 
 		// don't consider directories (special case: there is no path information for /)
-		if ref.Path == "/" {
+		if ref.RealPath == "/" {
 			continue
 		} else if r.img.FileCatalog.Exists(*ref) {
 			metadata, err := r.img.FileCatalog.Get(*ref)
 			if err != nil {
-				return nil, fmt.Errorf("unable to get file metadata for path=%q: %w", ref.Path, err)
+				return nil, fmt.Errorf("unable to get file metadata for path=%q: %w", ref.RealPath, err)
 			}
 			if metadata.Metadata.IsDir {
 				continue
@@ -56,7 +66,7 @@ func (r *ImageSquashResolver) FilesByPath(paths ...string) ([]Location, error) {
 
 		if resolvedRef != nil && !uniqueFileIDs.Contains(*resolvedRef) {
 			uniqueFileIDs.Add(*resolvedRef)
-			uniqueLocations = append(uniqueLocations, NewLocationFromImage(*resolvedRef, r.img))
+			uniqueLocations = append(uniqueLocations, NewLocationFromImage(path, *resolvedRef, r.img))
 		}
 	}
 
@@ -69,28 +79,28 @@ func (r *ImageSquashResolver) FilesByGlob(patterns ...string) ([]Location, error
 	uniqueLocations := make([]Location, 0)
 
 	for _, pattern := range patterns {
-		refs, err := r.img.SquashedTree().FilesByGlob(pattern)
+		results, err := r.img.SquashedTree().FilesByGlob(pattern)
 		if err != nil {
 			return nil, fmt.Errorf("failed to resolve files by glob (%s): %w", pattern, err)
 		}
 
-		for _, ref := range refs {
+		for _, result := range results {
 			// don't consider directories (special case: there is no path information for /)
-			if ref.Path == "/" {
+			if result.MatchPath == "/" {
 				continue
-			} else if r.img.FileCatalog.Exists(ref) {
-				metadata, err := r.img.FileCatalog.Get(ref)
+			} else if r.img.FileCatalog.Exists(result.Reference) {
+				metadata, err := r.img.FileCatalog.Get(result.Reference)
 				if err != nil {
-					return nil, fmt.Errorf("unable to get file metadata for path=%q: %w", ref.Path, err)
+					return nil, fmt.Errorf("unable to get file metadata for path=%q: %w", result.MatchPath, err)
 				}
 				if metadata.Metadata.IsDir {
 					continue
 				}
 			}
 
-			resolvedLocations, err := r.FilesByPath(string(ref.Path))
+			resolvedLocations, err := r.FilesByPath(string(result.MatchPath))
 			if err != nil {
-				return nil, fmt.Errorf("failed to find files by path (ref=%+v): %w", ref, err)
+				return nil, fmt.Errorf("failed to find files by path (result=%+v): %w", result, err)
 			}
 			for _, resolvedLocation := range resolvedLocations {
 				if !uniqueFileIDs.Contains(resolvedLocation.ref) {
@@ -121,12 +131,12 @@ func (r *ImageSquashResolver) RelativeFileByPath(_ Location, path string) *Locat
 
 // MultipleFileContentsByLocation returns the file contents for all file.References relative to the image. Note that a
 // file.Reference is a path relative to a particular layer, in this case only from the squashed representation.
-func (r *ImageSquashResolver) MultipleFileContentsByLocation(locations []Location) (map[Location]string, error) {
+func (r *ImageSquashResolver) MultipleFileContentsByLocation(locations []Location) (map[Location]io.ReadCloser, error) {
 	return mapLocationRefs(r.img.MultipleFileContentsByRef, locations)
 }
 
 // FileContentsByLocation fetches file contents for a single file reference, irregardless of the source layer.
 // If the path does not exist an error is returned.
-func (r *ImageSquashResolver) FileContentsByLocation(location Location) (string, error) {
+func (r *ImageSquashResolver) FileContentsByLocation(location Location) (io.ReadCloser, error) {
 	return r.img.FileContentsByRef(location.ref)
 }

syft/source/image_squash_resolver_test.go

@@ -8,10 +8,11 @@ import (
 
 func TestImageSquashResolver_FilesByPath(t *testing.T) {
 	cases := []struct {
-		name         string
-		linkPath     string
-		resolveLayer uint
-		resolvePath  string
+		name                 string
+		linkPath             string
+		resolveLayer         uint
+		resolvePath          string
+		forcePositiveHasPath bool
 	}{
 		{
 			name:         "link with previous data",
@@ -41,12 +42,22 @@ func TestImageSquashResolver_FilesByPath(t *testing.T) {
 			name:         "dead link",
 			linkPath:     "/link-dead",
 			resolveLayer: 8,
-			resolvePath:  "/link-dead",
+			resolvePath:  "",
+			// the path should exist, even if the link is dead
+			forcePositiveHasPath: true,
 		},
 		{
 			name:        "ignore directories",
 			linkPath:    "/bin",
 			resolvePath: "",
+			// the path should exist, even if we ignore it
+			forcePositiveHasPath: true,
+		},
+		{
+			name:         "parent is a link (with overridden data)",
+			linkPath:     "/parent-link/file-4.txt",
+			resolveLayer: 11,
+			resolvePath:  "/parent/file-4.txt",
 		},
 	}
 	for _, c := range cases {
@@ -59,6 +70,17 @@ func TestImageSquashResolver_FilesByPath(t *testing.T) {
 				t.Fatalf("could not create resolver: %+v", err)
 			}
 
+			hasPath := resolver.HasPath(c.linkPath)
+			if !c.forcePositiveHasPath {
+				if c.resolvePath != "" && !hasPath {
+					t.Errorf("expected HasPath() to indicate existance, but did not")
+				} else if c.resolvePath == "" && hasPath {
+					t.Errorf("expeced HasPath() to NOT indicate existance, but does")
+				}
+			} else if !hasPath {
+				t.Errorf("expected HasPath() to indicate existance, but did not (force path)")
+			}
+
 			refs, err := resolver.FilesByPath(c.linkPath)
 			if err != nil {
 				t.Fatalf("could not use resolver: %+v", err)
@@ -80,8 +102,12 @@ func TestImageSquashResolver_FilesByPath(t *testing.T) {
 
 			actual := refs[0]
 
-			if actual.Path != c.resolvePath {
-				t.Errorf("bad resolve path: '%s'!='%s'", actual.Path, c.resolvePath)
+			if string(actual.ref.RealPath) != c.resolvePath {
+				t.Errorf("bad resolve path: '%s'!='%s'", string(actual.ref.RealPath), c.resolvePath)
+			}
+
+			if c.resolvePath != "" && string(actual.ref.RealPath) != actual.RealPath {
+				t.Errorf("we should always prefer real paths over ones with links")
 			}
 
 			entry, err := img.FileCatalog.Get(actual.ref)
@@ -89,8 +115,8 @@ func TestImageSquashResolver_FilesByPath(t *testing.T) {
 				t.Fatalf("failed to get metadata: %+v", err)
 			}
 
-			if entry.Source.Metadata.Index != c.resolveLayer {
-				t.Errorf("bad resolve layer: '%d'!='%d'", entry.Source.Metadata.Index, c.resolveLayer)
+			if entry.Layer.Metadata.Index != c.resolveLayer {
+				t.Errorf("bad resolve layer: '%d'!='%d'", entry.Layer.Metadata.Index, c.resolveLayer)
 			}
 		})
 	}
@@ -105,39 +131,51 @@ func TestImageSquashResolver_FilesByGlob(t *testing.T) {
 	}{
 		{
 			name:         "link with previous data",
-			glob:         "**link-1",
+			glob:         "**/link-1",
 			resolveLayer: 1,
 			resolvePath:  "/file-1.txt",
 		},
 		{
 			name:         "link with in layer data",
-			glob:         "**link-within",
+			glob:         "**/link-within",
 			resolveLayer: 5,
 			resolvePath:  "/file-3.txt",
 		},
 		{
 			name:         "link with overridden data",
-			glob:         "**link-2",
+			glob:         "**/link-2",
 			resolveLayer: 7,
 			resolvePath:  "/file-2.txt",
 		},
 		{
 			name:         "indirect link (with overridden data)",
-			glob:         "**link-indirect",
+			glob:         "**/link-indirect",
 			resolveLayer: 7,
 			resolvePath:  "/file-2.txt",
 		},
 		{
-			name:         "dead link",
-			glob:         "**link-dead",
-			resolveLayer: 8,
-			resolvePath:  "/link-dead",
+			name: "dead link",
+			glob: "**/link-dead",
+			// dead links are dead! they shouldn't match on globs
+			resolvePath: "",
 		},
 		{
 			name:        "ignore directories",
 			glob:        "**/bin",
 			resolvePath: "",
 		},
+		{
+			name:         "parent without link",
+			glob:         "**/parent/*.txt",
+			resolveLayer: 11,
+			resolvePath:  "/parent/file-4.txt",
+		},
+		{
+			name:         "parent is a link (override)",
+			glob:         "**/parent-link/file-4.txt",
+			resolveLayer: 11,
+			resolvePath:  "/parent/file-4.txt",
+		},
 	}
 	for _, c := range cases {
 		t.Run(c.name, func(t *testing.T) {
@@ -170,8 +208,12 @@ func TestImageSquashResolver_FilesByGlob(t *testing.T) {
 
 			actual := refs[0]
 
-			if actual.Path != c.resolvePath {
-				t.Errorf("bad resolve path: '%s'!='%s'", actual.Path, c.resolvePath)
+			if string(actual.ref.RealPath) != c.resolvePath {
+				t.Errorf("bad resolve path: '%s'!='%s'", string(actual.ref.RealPath), c.resolvePath)
+			}
+
+			if c.resolvePath != "" && string(actual.ref.RealPath) != actual.RealPath {
+				t.Errorf("we should always prefer real paths over ones with links")
 			}
 
 			entry, err := img.FileCatalog.Get(actual.ref)
@@ -179,8 +221,8 @@ func TestImageSquashResolver_FilesByGlob(t *testing.T) {
 				t.Fatalf("failed to get metadata: %+v", err)
 			}
 
-			if entry.Source.Metadata.Index != c.resolveLayer {
-				t.Errorf("bad resolve layer: '%d'!='%d'", entry.Source.Metadata.Index, c.resolveLayer)
+			if entry.Layer.Metadata.Index != c.resolveLayer {
+				t.Errorf("bad resolve layer: '%d'!='%d'", entry.Layer.Metadata.Index, c.resolveLayer)
 			}
 		})
 	}

syft/source/location.go

@@ -7,9 +7,11 @@ import (
 	"github.com/anchore/stereoscope/pkg/image"
 )
 
-// Location represents a path relative to a particular filesystem.
+// Location represents a path relative to a particular filesystem resolved to a specific file.Reference. This struct is used as a key
+// in content fetching to uniquely identify a file relative to a request (the VirtualPath).
 type Location struct {
-	Path         string         `json:"path"`              // The string path of the location (e.g. /etc/hosts)
+	RealPath     string         `json:"path"`              // The path where all path ancestors have no hardlinks / symlinks
+	VirtualPath  string         `json:"-"`                 // The path to the file which may or may not have hardlinks / symlinks
 	FileSystemID string         `json:"layerID,omitempty"` // An ID representing the filesystem. For container images this is a layer digest, directories or root filesystem this is blank.
 	ref          file.Reference // The file reference relative to the stereoscope.FileCatalog that has more information about this location.
 }
@@ -17,24 +19,26 @@ type Location struct {
 // NewLocation creates a new Location representing a path without denoting a filesystem or FileCatalog reference.
 func NewLocation(path string) Location {
 	return Location{
-		Path: path,
+		RealPath: path,
 	}
 }
 
 // NewLocationFromImage creates a new Location representing the given path (extracted from the ref) relative to the given image.
-func NewLocationFromImage(ref file.Reference, img *image.Image) Location {
+func NewLocationFromImage(virtualPath string, ref file.Reference, img *image.Image) Location {
 	entry, err := img.FileCatalog.Get(ref)
 	if err != nil {
 		log.Warnf("unable to find file catalog entry for ref=%+v", ref)
 		return Location{
-			Path: string(ref.Path),
-			ref:  ref,
+			VirtualPath: virtualPath,
+			RealPath:    string(ref.RealPath),
+			ref:         ref,
 		}
 	}
 
 	return Location{
-		Path:         string(ref.Path),
-		FileSystemID: entry.Source.Metadata.Digest,
+		VirtualPath:  virtualPath,
+		RealPath:     string(ref.RealPath),
+		FileSystemID: entry.Layer.Metadata.Digest,
 		ref:          ref,
 	}
 }

syft/source/mock_resolver.go

@@ -0,0 +1,112 @@
+package source
+
+import (
+	"fmt"
+	"io"
+	"os"
+
+	"github.com/anchore/syft/internal/file"
+)
+
+var _ Resolver = (*MockResolver)(nil)
+
+// MockResolver implements the Resolver interface and is intended for use *only in test code*.
+// It provides an implementation that can resolve local filesystem paths using only a provided discrete list of file
+// paths, which are typically paths to test fixtures.
+type MockResolver struct {
+	Locations []Location
+}
+
+// NewMockResolverForPaths creates a new MockResolver, where the only resolvable
+// files are those specified by the supplied paths.
+func NewMockResolverForPaths(paths ...string) *MockResolver {
+	var locations []Location
+	for _, p := range paths {
+		locations = append(locations, NewLocation(p))
+	}
+
+	return &MockResolver{Locations: locations}
+}
+
+// HasPath indicates if the given path exists in the underlying source.
+func (r MockResolver) HasPath(path string) bool {
+	for _, l := range r.Locations {
+		if l.RealPath == path {
+			return true
+		}
+	}
+	return false
+}
+
+// String returns the string representation of the MockResolver.
+func (r MockResolver) String() string {
+	return fmt.Sprintf("mock:(%s,...)", r.Locations[0].RealPath)
+}
+
+// FileContentsByLocation fetches file contents for a single location. If the
+// path does not exist, an error is returned.
+func (r MockResolver) FileContentsByLocation(location Location) (io.ReadCloser, error) {
+	for _, l := range r.Locations {
+		if l == location {
+			return os.Open(location.RealPath)
+		}
+	}
+
+	return nil, fmt.Errorf("no file for location: %v", location)
+}
+
+// MultipleFileContentsByLocation returns the file contents for all specified Locations.
+func (r MockResolver) MultipleFileContentsByLocation(locations []Location) (map[Location]io.ReadCloser, error) {
+	results := make(map[Location]io.ReadCloser)
+	for _, l := range locations {
+		contents, err := r.FileContentsByLocation(l)
+		if err != nil {
+			return nil, err
+		}
+		results[l] = contents
+	}
+
+	return results, nil
+}
+
+// FilesByPath returns all Locations that match the given paths.
+func (r MockResolver) FilesByPath(paths ...string) ([]Location, error) {
+	var results []Location
+	for _, p := range paths {
+		for _, location := range r.Locations {
+			if p == location.RealPath {
+				results = append(results, NewLocation(p))
+			}
+		}
+	}
+
+	return results, nil
+}
+
+// FilesByGlob returns all Locations that match the given path glob pattern.
+func (r MockResolver) FilesByGlob(patterns ...string) ([]Location, error) {
+	var results []Location
+	for _, pattern := range patterns {
+		for _, location := range r.Locations {
+			if file.GlobMatch(pattern, location.RealPath) {
+				results = append(results, location)
+			}
+		}
+	}
+
+	return results, nil
+}
+
+// RelativeFileByPath returns a single Location for the given path.
+func (r MockResolver) RelativeFileByPath(_ Location, path string) *Location {
+	paths, err := r.FilesByPath(path)
+	if err != nil {
+		return nil
+	}
+
+	if len(paths) < 1 {
+		return nil
+	}
+
+	return &paths[0]
+}

syft/source/resolver.go

@@ -2,6 +2,7 @@ package source
 
 import (
 	"fmt"
+	"io"
 
 	"github.com/anchore/stereoscope/pkg/image"
 )
@@ -14,13 +15,15 @@ type Resolver interface {
 
 // ContentResolver knows how to get file content for given file.References
 type ContentResolver interface {
-	FileContentsByLocation(Location) (string, error)
-	MultipleFileContentsByLocation([]Location) (map[Location]string, error)
-	// TODO: we should consider refactoring to return a set of io.Readers or file.Openers instead of the full contents themselves (allow for optional buffering).
+	FileContentsByLocation(Location) (io.ReadCloser, error)
+	// TODO: it is possible to be given duplicate locations that will be overridden in the map (key), a subtle problem that coule easily be misued.
+	MultipleFileContentsByLocation([]Location) (map[Location]io.ReadCloser, error)
 }
 
-// FileResolver knows how to get file.References for given string paths and globs
+// FileResolver knows how to get a Location for given string paths and globs
 type FileResolver interface {
+	// HasPath indicates if the given path exists in the underlying source.
+	HasPath(path string) bool
 	// FilesByPath fetches a set of file references which have the given path (for an image, there may be multiple matches)
 	FilesByPath(paths ...string) ([]Location, error)
 	// FilesByGlob fetches a set of file references which the given glob matches

syft/source/source_test.go

@@ -1,6 +1,7 @@
 package source
 
 import (
+	"io/ioutil"
 	"os"
 	"testing"
 
@@ -137,9 +138,14 @@ func TestMultipleFileContentsByLocation(t *testing.T) {
 			location := locations[0]
 
 			contents, err := p.Resolver.MultipleFileContentsByLocation([]Location{location})
-			content := contents[location]
+			contentReader := contents[location]
 
-			if content != test.expected {
+			content, err := ioutil.ReadAll(contentReader)
+			if err != nil {
+				t.Fatalf("cannot read contents: %+v", err)
+			}
+
+			if string(content) != test.expected {
 				t.Errorf("unexpected contents from file: '%s' != '%s'", content, test.expected)
 			}
 

syft/source/test-fixtures/image-simple/Dockerfile

@@ -0,0 +1,6 @@
+# Note: changes to this file will result in updating several test values. Consider making a new image fixture instead of editing this one.
+FROM scratch
+ADD file-1.txt /somefile-1.txt
+ADD file-2.txt /somefile-2.txt
+# note: adding a directory will behave differently on docker engine v18 vs v19
+ADD target /

syft/source/test-fixtures/image-simple/file-1.txt

@@ -0,0 +1 @@
+this file has contents

syft/source/test-fixtures/image-simple/file-2.txt

@@ -0,0 +1 @@
+file-2 contents!

syft/source/test-fixtures/image-simple/target/really/nested/file-3.txt

@@ -0,0 +1,2 @@
+another file!
+with lines...

syft/source/test-fixtures/image-symlinks/Dockerfile

@@ -21,4 +21,12 @@ RUN ln -s ./link-2 link-indirect
 ADD new-file-2.txt file-2.txt
 
 # LAYER 8: dead link
-RUN ln -s ./i-dont-exist.txt link-dead
+RUN ln -s ./i-dont-exist.txt link-dead
+
+# LAYER 9: add the parent dir
+ADD parent /parent
+# LAYER 10: parent is a symlink
+RUN ln -s /parent parent-link
+
+# LAYER 11: parent is a symlink and the child target is overridden
+COPY new-file-4.txt /parent-link/file-4.txt

syft/source/test-fixtures/image-symlinks/new-file-4.txt

@@ -0,0 +1 @@
+override file 4!

syft/source/test-fixtures/image-symlinks/parent/file-4.txt

@@ -0,0 +1 @@
+the 4th file!

test/integration/distro_test.go

@@ -16,7 +16,7 @@ func TestDistroImage(t *testing.T) {
 	tarPath := imagetest.GetFixtureImageTarPath(t, fixtureImageName)
 	defer cleanup()
 
-	_, _, actualDistro, err := syft.Catalog("docker-archive:"+tarPath, source.AllLayersScope)
+	_, _, actualDistro, err := syft.Catalog("docker-archive:"+tarPath, source.SquashedScope)
 	if err != nil {
 		t.Fatalf("failed to catalog image: %+v", err)
 	}

test/integration/document_import_test.go

@@ -31,7 +31,7 @@ func TestCatalogFromJSON(t *testing.T) {
 			tarPath := imagetest.GetFixtureImageTarPath(t, test.fixture)
 			defer cleanup()
 
-			expectedSource, expectedCatalog, expectedDistro, err := syft.Catalog("docker-archive:"+tarPath, source.AllLayersScope)
+			expectedSource, expectedCatalog, expectedDistro, err := syft.Catalog("docker-archive:"+tarPath, source.SquashedScope)
 			if err != nil {
 				t.Fatalf("failed to catalog image: %+v", err)
 			}
@@ -81,6 +81,12 @@ func TestCatalogFromJSON(t *testing.T) {
 					e.Metadata = metadata
 				}
 
+				// ignore the virtual path on the location for now
+				for l := range a.Locations {
+					a.Locations[l].VirtualPath = ""
+					e.Locations[l].VirtualPath = ""
+				}
+
 				for _, d := range deep.Equal(a, e) {
 					// ignore errors for empty collections vs nil for select fields
 					// TODO: this is brittle, but not dangerously so. We should still find a better way to do this.

test/integration/json_schema_test.go

@@ -9,11 +9,13 @@ import (
 	"strings"
 	"testing"
 
+	"github.com/anchore/syft/internal"
+
+	"github.com/anchore/syft/syft/distro"
+	"github.com/anchore/syft/syft/presenter"
+
 	"github.com/anchore/stereoscope/pkg/imagetest"
 	"github.com/anchore/syft/syft"
-	"github.com/anchore/syft/syft/distro"
-	"github.com/anchore/syft/syft/pkg"
-	"github.com/anchore/syft/syft/presenter"
 	"github.com/anchore/syft/syft/source"
 	"github.com/xeipuuv/gojsonschema"
 )
@@ -34,7 +36,7 @@ func repoRoot(t *testing.T) string {
 }
 
 func validateAgainstV1Schema(t *testing.T, json string) {
-	fullSchemaPath := path.Join(repoRoot(t), jsonSchemaPath, "schema.json")
+	fullSchemaPath := path.Join(repoRoot(t), jsonSchemaPath, fmt.Sprintf("schema-%s.json", internal.JSONSchemaVersion))
 	schemaLoader := gojsonschema.NewReferenceLoader(fmt.Sprintf("file://%s", fullSchemaPath))
 	documentLoader := gojsonschema.NewStringLoader(json)
 
@@ -45,13 +47,23 @@ func validateAgainstV1Schema(t *testing.T, json string) {
 
 	if !result.Valid() {
 		t.Errorf("failed json schema validation:")
+		t.Errorf("JSON:\n%s\n", json)
 		for _, desc := range result.Errors() {
 			t.Errorf("  - %s\n", desc)
 		}
 	}
 }
 
-func testJsonSchema(t *testing.T, catalog *pkg.Catalog, theScope source.Source, prefix string) {
+func TestJsonSchemaImg(t *testing.T) {
+	fixtureImageName := "image-pkg-coverage"
+	_, cleanup := imagetest.GetFixtureImage(t, "docker-archive", fixtureImageName)
+	tarPath := imagetest.GetFixtureImageTarPath(t, fixtureImageName)
+	defer cleanup()
+
+	src, catalog, _, err := syft.Catalog("docker-archive:"+tarPath, source.SquashedScope)
+	if err != nil {
+		t.Fatalf("failed to catalog image: %+v", err)
+	}
 
 	output := bytes.NewBufferString("")
 
@@ -60,7 +72,34 @@ func testJsonSchema(t *testing.T, catalog *pkg.Catalog, theScope source.Source,
 		t.Fatalf("bad distro: %+v", err)
 	}
 
-	p := presenter.GetPresenter(presenter.JSONPresenter, theScope.Metadata, catalog, &d)
+	p := presenter.GetPresenter(presenter.JSONPresenter, src.Metadata, catalog, &d)
+	if p == nil {
+		t.Fatal("unable to get presenter")
+	}
+
+	err = p.Present(output)
+	if err != nil {
+		t.Fatalf("unable to present: %+v", err)
+	}
+
+	validateAgainstV1Schema(t, output.String())
+
+}
+
+func TestJsonSchemaDirs(t *testing.T) {
+	src, catalog, _, err := syft.Catalog("dir:test-fixtures/image-pkg-coverage", source.SquashedScope)
+	if err != nil {
+		t.Errorf("unable to create source from dir: %+v", err)
+	}
+
+	output := bytes.NewBufferString("")
+
+	d, err := distro.NewDistro(distro.CentOS, "5", "rhel fedora")
+	if err != nil {
+		t.Fatalf("bad distro: %+v", err)
+	}
+
+	p := presenter.GetPresenter(presenter.JSONPresenter, src.Metadata, catalog, &d)
 	if p == nil {
 		t.Fatal("unable to get presenter")
 	}
@@ -72,42 +111,3 @@ func testJsonSchema(t *testing.T, catalog *pkg.Catalog, theScope source.Source,
 
 	validateAgainstV1Schema(t, output.String())
 }
-
-func TestJsonSchemaImg(t *testing.T) {
-	fixtureImageName := "image-pkg-coverage"
-	_, cleanup := imagetest.GetFixtureImage(t, "docker-archive", fixtureImageName)
-	tarPath := imagetest.GetFixtureImageTarPath(t, fixtureImageName)
-	defer cleanup()
-
-	src, catalog, _, err := syft.Catalog("docker-archive:"+tarPath, source.AllLayersScope)
-	if err != nil {
-		t.Fatalf("failed to catalog image: %+v", err)
-	}
-
-	var cases []testCase
-	cases = append(cases, commonTestCases...)
-	cases = append(cases, imageOnlyTestCases...)
-
-	for _, c := range cases {
-		t.Run(c.name, func(t *testing.T) {
-			testJsonSchema(t, catalog, src, "img")
-		})
-	}
-}
-
-func TestJsonSchemaDirs(t *testing.T) {
-	src, catalog, _, err := syft.Catalog("dir:test-fixtures/image-pkg-coverage", source.AllLayersScope)
-	if err != nil {
-		t.Errorf("unable to create source from dir: %+v", err)
-	}
-
-	var cases []testCase
-	cases = append(cases, commonTestCases...)
-	cases = append(cases, dirOnlyTestCases...)
-
-	for _, c := range cases {
-		t.Run(c.name, func(t *testing.T) {
-			testJsonSchema(t, catalog, src, "dir")
-		})
-	}
-}

test/integration/pkg_cases_test.go

@@ -15,7 +15,10 @@ var imageOnlyTestCases = []testCase{
 		pkgType:     pkg.GemPkg,
 		pkgLanguage: pkg.Ruby,
 		pkgInfo: map[string]string{
+			// specifications in the root specification directory
 			"bundler": "2.1.4",
+			// specifications in named directories
+			"unbundler": "3.1.4",
 		},
 	},
 	{

test/integration/pkg_coverage_test.go

@@ -18,7 +18,7 @@ func TestPkgCoverageImage(t *testing.T) {
 	tarPath := imagetest.GetFixtureImageTarPath(t, fixtureImageName)
 	defer cleanup()
 
-	_, catalog, _, err := syft.Catalog("docker-archive:"+tarPath, source.AllLayersScope)
+	_, catalog, _, err := syft.Catalog("docker-archive:"+tarPath, source.SquashedScope)
 	if err != nil {
 		t.Fatalf("failed to catalog image: %+v", err)
 	}
@@ -100,7 +100,7 @@ func TestPkgCoverageImage(t *testing.T) {
 }
 
 func TestPkgCoverageDirectory(t *testing.T) {
-	_, catalog, _, err := syft.Catalog("dir:test-fixtures/image-pkg-coverage", source.AllLayersScope)
+	_, catalog, _, err := syft.Catalog("dir:test-fixtures/image-pkg-coverage", source.SquashedScope)
 
 	if err != nil {
 		t.Errorf("unable to create source from dir: %+v", err)

test/integration/test-fixtures/image-pkg-coverage/ruby/specifications/default/unbundler.gemspec

@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+# -*- encoding: utf-8 -*-                                                                                                                                         
+# stub: unbundler 2.1.4 ruby lib                                                                                                                                    
+                                                                                                                                                                  
+Gem::Specification.new do |s|                                                                                                                                     
+    s.name = "unbundler".freeze
+    s.version = "3.1.4"
+                                                                                                                                                                    
+    s.required_rubygems_version = Gem::Requirement.new(">= 2.5.2".freeze) if s.respond_to? :required_rubygems_version=                                              
+    s.require_paths = ["lib".freeze]                                                                                                                                
+    s.authors = ["Andr\u00E9 Arko".freeze, "Samuel Giddins".freeze, "Colby Swandale".freeze, "Hiroshi Shibata".freeze, "David Rodr\u00EDguez".freeze, "Grey Baker".f
+    s.bindir = "exe".freeze                                                                                                                                         
+    s.date = "2020-01-05"                                                                                                                                           
+    s.description = "Bundler manages an application's dependencies through its entire life, across many machines, systematically and repeatably".freeze             
+    s.email = ["team@unbundler.io".freeze]                                                                                                                            
+    s.executables = ["unbundle".freeze, "unbundler".freeze]                                                                                                             
+    s.files = ["exe/unbundle".freeze, "exe/unbundler".freeze]                                                                                                           
+    s.homepage = "https://unbundler.io".freeze                                                                                                                        
+    s.licenses = ["MIT".freeze]                                                                                                                                     
+    s.required_ruby_version = Gem::Requirement.new(">= 2.3.0".freeze)                                                                                               
+    s.rubygems_version = "3.1.2".freeze                                                                                                                             
+    s.summary = "The best way to manage your application's dependencies".freeze                                                                                     
+                                                                                                                                                                    
+    s.installed_by_version = "3.1.2" if s.respond_to? :installed_by_version                                                                                         
+  end 

ui/event_handlers.go

@@ -181,6 +181,7 @@ func PullDockerImageHandler(ctx context.Context, fr *frame.Frame, event partybus
 }
 
 // FetchImageHandler periodically writes a the image save and write-to-disk process in the form of a progress bar.
+// nolint:dupl
 func FetchImageHandler(ctx context.Context, fr *frame.Frame, event partybus.Event, wg *sync.WaitGroup) error {
 	_, prog, err := stereoEventParsers.ParseFetchImage(event)
 	if err != nil {
@@ -307,3 +308,48 @@ func CatalogerStartedHandler(ctx context.Context, fr *frame.Frame, event partybu
 
 	return nil
 }
+
+// ImportStartedHandler shows the intermittent upload progress to Anchore Enterprise.
+// nolint:dupl
+func ImportStartedHandler(ctx context.Context, fr *frame.Frame, event partybus.Event, wg *sync.WaitGroup) error {
+	host, prog, err := syftEventParsers.ParseImportStarted(event)
+	if err != nil {
+		return fmt.Errorf("bad %s event: %w", event.Type, err)
+	}
+
+	line, err := fr.Append()
+	if err != nil {
+		return err
+	}
+	wg.Add(1)
+
+	formatter, spinner := startProcess()
+	stream := progress.Stream(ctx, prog, interval)
+	title := tileFormat.Sprint("Uploading image")
+
+	formatFn := func(p progress.Progress) {
+		progStr, err := formatter.Format(p)
+		spin := color.Magenta.Sprint(spinner.Next())
+		if err != nil {
+			_, _ = io.WriteString(line, fmt.Sprintf("Error: %+v", err))
+		} else {
+			auxInfo := auxInfoFormat.Sprintf("[%s]", prog.Stage())
+			_, _ = io.WriteString(line, fmt.Sprintf(statusTitleTemplate+"%s %s", spin, title, progStr, auxInfo))
+		}
+	}
+
+	go func() {
+		defer wg.Done()
+
+		formatFn(progress.Progress{})
+		for p := range stream {
+			formatFn(p)
+		}
+
+		spin := color.Green.Sprint(completedStatus)
+		title = tileFormat.Sprint("Uploaded image")
+		auxInfo := auxInfoFormat.Sprintf("[%s]", host)
+		_, _ = io.WriteString(line, fmt.Sprintf(statusTitleTemplate+"%s", spin, title, auxInfo))
+	}()
+	return err
+}

ui/handler.go

@@ -27,7 +27,7 @@ func NewHandler() *Handler {
 // RespondsTo indicates if the handler is capable of handling the given event.
 func (r *Handler) RespondsTo(event partybus.Event) bool {
 	switch event.Type {
-	case stereoscopeEvent.PullDockerImage, stereoscopeEvent.ReadImage, stereoscopeEvent.FetchImage, syftEvent.CatalogerStarted:
+	case stereoscopeEvent.PullDockerImage, stereoscopeEvent.ReadImage, stereoscopeEvent.FetchImage, syftEvent.CatalogerStarted, syftEvent.ImportStarted:
 		return true
 	default:
 		return false
@@ -48,6 +48,9 @@ func (r *Handler) Handle(ctx context.Context, fr *frame.Frame, event partybus.Ev
 
 	case syftEvent.CatalogerStarted:
 		return CatalogerStartedHandler(ctx, fr, event, wg)
+
+	case syftEvent.ImportStarted:
+		return ImportStartedHandler(ctx, fr, event, wg)
 	}
 	return nil
 }