Merge pull request #237 from anchore/update-future-plans

Remove CycloneDX from future plans

.github/workflows/acceptance-test.yaml

@@ -35,8 +35,11 @@ jobs:
             ${{ runner.os }}-go-${{ env.GO_VERSION }}-${{ hashFiles('**/go.sum') }}-
             ${{ runner.os }}-go-${{ env.GO_VERSION }}-
 
-      - name: Bootstrap dependencies
-        if: steps.cache.outputs.cache-hit != 'true'
+      - name: Bootstrap project dependencies
+        if: steps.bootstrap-cache.outputs.cache-hit != 'true'
+        run: make bootstrap
+
+      - name: Bootstrap CI dependencies
         run: make ci-bootstrap
 
       - name: Import GPG key

.github/workflows/codeql-analysis.yml

@@ -0,0 +1,71 @@
+# For most projects, this workflow file will not need changing; you simply need
+# to commit it to your repository.
+#
+# You may wish to alter this file to override the set of languages analyzed,
+# or to provide custom queries or build logic.
+name: "CodeQL Security Scan"
+
+on:
+  push:
+    branches: [main]
+  pull_request:
+    # The branches below must be a subset of the branches above
+    branches: [main]
+  schedule:
+    - cron: '0 0 * * 3'
+
+jobs:
+  analyze:
+    name: Analyze
+    runs-on: ubuntu-latest
+
+    strategy:
+      fail-fast: false
+      matrix:
+        # Override automatic language detection by changing the below list
+        # Supported options are ['csharp', 'cpp', 'go', 'java', 'javascript', 'python']
+        language: ['go', 'python']
+        # Learn more...
+        # https://docs.github.com/en/github/finding-security-vulnerabilities-and-errors-in-your-code/configuring-code-scanning#overriding-automatic-language-detection
+
+    steps:
+    - name: Checkout repository
+      uses: actions/checkout@v2
+      with:
+        # We must fetch at least the immediate parents so that if this is
+        # a pull request then we can checkout the head.
+        fetch-depth: 2
+
+    # If this run was triggered by a pull request event, then checkout
+    # the head of the pull request instead of the merge commit.
+    - run: git checkout HEAD^2
+      if: ${{ github.event_name == 'pull_request' }}
+
+    # Initializes the CodeQL tools for scanning.
+    - name: Initialize CodeQL
+      uses: github/codeql-action/init@v1
+      with:
+        languages: ${{ matrix.language }}
+        # If you wish to specify custom queries, you can do so here or in a config file.
+        # By default, queries listed here will override any specified in a config file. 
+        # Prefix the list here with "+" to use these queries and those in the config file.
+        # queries: ./path/to/local/query, your-org/your-repo/queries@main
+
+    # Autobuild attempts to build any compiled languages  (C/C++, C#, or Java).
+    # If this step fails, then you should remove it and run the build manually (see below)
+    - name: Autobuild
+      uses: github/codeql-action/autobuild@v1
+
+    # ℹ️ Command-line programs to run using the OS shell.
+    # 📚 https://git.io/JvXDl
+
+    # ✏️ If the Autobuild fails above, remove it and uncomment the following three lines
+    #    and modify them (or add more) to build your code if your project
+    #    uses a compiled language
+
+    #- run: |
+    #   make bootstrap
+    #   make release
+
+    - name: Perform CodeQL Analysis
+      uses: github/codeql-action/analyze@v1

Makefile

@@ -122,7 +122,7 @@ validate-cyclonedx-schema:
 .PHONY: unit
 unit: fixtures ## Run unit tests (with coverage)
 	$(call title,Running unit tests)
-	go test -coverprofile $(COVER_REPORT) ./...
+	go test -coverprofile $(COVER_REPORT) $(shell go list ./... | grep -v anchore/syft/test)
 	@go tool cover -func $(COVER_REPORT) | grep total |  awk '{print substr($$3, 1, length($$3)-1)}' > $(COVER_TOTAL)
 	@echo "Coverage: $$(cat $(COVER_TOTAL))"
 	@if [ $$(echo "$$(cat $(COVER_TOTAL)) >= $(COVERAGE_THRESHOLD)" | bc -l) -ne 1 ]; then echo "$(RED)$(BOLD)Failed coverage quality gate (> $(COVERAGE_THRESHOLD)%)$(RESET)" && false; fi
@@ -130,7 +130,9 @@ unit: fixtures ## Run unit tests (with coverage)
 .PHONY: integration
 integration: ## Run integration tests
 	$(call title,Running integration tests)
-	go test -tags=integration -v ./test/integration
+
+	go test -v ./test/integration
+
 
 # note: this is used by CI to determine if the integration test fixture cache (docker image tars) should be busted
 integration-fingerprint:

README.md

@@ -112,5 +112,4 @@ check-for-app-update: true
 ## Future plans
 
 The following areas of potential development are currently being investigated:
-- Add CycloneDX to list of output formats
 - Establish a stable interchange format w/Grype

internal/file/opener.go

@@ -0,0 +1,14 @@
+package file
+
+import (
+	"io"
+	"os"
+)
+
+type Opener struct {
+	path string
+}
+
+func (o Opener) Open() (io.ReadCloser, error) {
+	return os.Open(o.path)
+}

internal/file/zip_file_traversal.go

@@ -63,8 +63,8 @@ func TraverseFilesInZip(archivePath string, visitor func(*zip.File) error, paths
 	return nil
 }
 
-func ExtractFromZipToUniqueTempFile(archivePath, dir string, paths ...string) (map[string]io.Reader, error) {
-	results := make(map[string]io.Reader)
+func ExtractFromZipToUniqueTempFile(archivePath, dir string, paths ...string) (map[string]Opener, error) {
+	results := make(map[string]Opener)
 
 	// don't allow for full traversal, only select traversal from given paths
 	if len(paths) == 0 {
@@ -78,11 +78,21 @@ func ExtractFromZipToUniqueTempFile(archivePath, dir string, paths ...string) (m
 		if err != nil {
 			return fmt.Errorf("unable to create temp file: %w", err)
 		}
+		// we shouldn't try and keep the tempfile open as the returned result may have several files, which takes up
+		// resources (leading to "too many open files"). Instead we'll return a file opener to the caller which
+		// provides a ReadCloser. It is up to the caller to handle closing the file explicitly.
+		defer tempFile.Close()
 
 		zippedFile, err := file.Open()
 		if err != nil {
 			return fmt.Errorf("unable to read file=%q from zip=%q: %w", file.Name, archivePath, err)
 		}
+		defer func() {
+			err := zippedFile.Close()
+			if err != nil {
+				log.Errorf("unable to close source file=%q from zip=%q: %+v", file.Name, archivePath, err)
+			}
+		}()
 
 		if file.FileInfo().IsDir() {
 			return fmt.Errorf("unable to extract directories, only files: %s", file.Name)
@@ -103,12 +113,8 @@ func ExtractFromZipToUniqueTempFile(archivePath, dir string, paths ...string) (m
 			return fmt.Errorf("unable to reset file pointer (%s): %w", tempFile.Name(), err)
 		}
 
-		results[file.Name] = tempFile
+		results[file.Name] = Opener{path: tempFile.Name()}
 
-		err = zippedFile.Close()
-		if err != nil {
-			return fmt.Errorf("unable to close source file=%q from zip=%q: %w", file.Name, archivePath, err)
-		}
 		return nil
 	}
 

schema/README.md

@@ -0,0 +1,9 @@
+## Updating the JSON schema
+Today the JSON schema is generated from integration test data. Specifically, when integration tests are run, the `/schema/json/examples` directory is populated with syft JSON output data. This examples directory is used to drive automatically generating the JSON schema.
+The caveats with this approach is:
+1) the JSON schema is only as good as the examples provided
+2) there is an integration test that ensures that the JSON schema is valid relative to what the code currently generates.
+This means to update the JSON schema you need to
+1) Open up `test/integration/json_schema_test.go` and comment out invocations of the `validateAgainstV1Schema` function.
+2) From the root of the repo run `generate-json-schema`. Now there should be a new schema generated at `/schema/json/schema.json`
+3) Uncomment the `validateAgainstV1Schema` function.

schema/json/schema.json

@@ -40,6 +40,12 @@
                             "architecture": {
                                 "type": "string"
                             },
+                            "author": {
+                                "type": "string"
+                            },
+                            "authorEmail": {
+                                "type": "string"
+                            },
                             "description": {
                                 "type": "string"
                             },
@@ -47,41 +53,62 @@
                                 "type": "integer"
                             },
                             "files": {
-                                "items": {
-                                    "anyOf": [
-                                        {
-                                            "type": "string"
-                                        },
-                                        {
-                                            "properties": {
-                                                "checksum": {
+                                "anyOf": [
+                                    {
+                                        "type": "null"
+                                    },
+                                    {
+                                        "items": {
+                                            "anyOf": [
+                                                {
                                                     "type": "string"
                                                 },
-                                                "ownerGid": {
-                                                    "type": "string"
-                                                },
-                                                "ownerUid": {
-                                                    "type": "string"
-                                                },
-                                                "path": {
-                                                    "type": "string"
-                                                },
-                                                "permissions": {
-                                                    "type": "string"
+                                                {
+                                                    "properties": {
+                                                        "checksum": {
+                                                            "type": "string"
+                                                        },
+                                                        "digest": {
+                                                            "properties": {
+                                                                "algorithm": {
+                                                                    "type": "string"
+                                                                },
+                                                                "value": {
+                                                                    "type": "string"
+                                                                }
+                                                            },
+                                                            "required": [
+                                                                "algorithm",
+                                                                "value"
+                                                            ],
+                                                            "type": "object"
+                                                        },
+                                                        "ownerGid": {
+                                                            "type": "string"
+                                                        },
+                                                        "ownerUid": {
+                                                            "type": "string"
+                                                        },
+                                                        "path": {
+                                                            "type": "string"
+                                                        },
+                                                        "permissions": {
+                                                            "type": "string"
+                                                        },
+                                                        "size": {
+                                                            "type": "string"
+                                                        }
+                                                    },
+                                                    "required": [
+                                                        "path"
+                                                    ],
+                                                    "type": "object"
                                                 }
-                                            },
-                                            "required": [
-                                                "checksum",
-                                                "ownerGid",
-                                                "ownerUid",
-                                                "path",
-                                                "permissions"
-                                            ],
-                                            "type": "object"
-                                        }
-                                    ]
-                                },
-                                "type": "array"
+                                            ]
+                                        },
+                                        "type": "array"
+                                    }
+                                ]
                             },
                             "gitCommitOfApkPort": {
                                 "type": "string"
@@ -393,6 +420,9 @@
                                                 ],
                                                 "type": "object"
                                             },
+                                            "metadataType": {
+                                                "type": "string"
+                                            },
                                             "sources": {
                                                 "type": "null"
                                             },
@@ -409,6 +439,7 @@
                                             "licenses",
                                             "manifest",
                                             "metadata",
+                                            "metadataType",
                                             "sources",
                                             "type",
                                             "version"
@@ -417,6 +448,9 @@
                                     }
                                 ]
                             },
+                            "platform": {
+                                "type": "string"
+                            },
                             "pomProperties": {
                                 "properties": {
                                     "Path": {
@@ -457,6 +491,9 @@
                             "release": {
                                 "type": "string"
                             },
+                            "sitePackagesRootPath": {
+                                "type": "string"
+                            },
                             "size": {
                                 "type": "integer"
                             },
@@ -466,6 +503,12 @@
                             "sourceRpm": {
                                 "type": "string"
                             },
+                            "topLevelPackages": {
+                                "items": {
+                                    "type": "string"
+                                },
+                                "type": "array"
+                            },
                             "url": {
                                 "type": "string"
                             },

syft/cataloger/apkdb/parse_apk_db.go

@@ -49,11 +49,12 @@ func parseApkDB(_ string, reader io.Reader) ([]pkg.Package, error) {
 		}
 		if metadata != nil {
 			packages = append(packages, pkg.Package{
-				Name:     metadata.Package,
-				Version:  metadata.Version,
-				Licenses: strings.Split(metadata.License, " "),
-				Type:     pkg.ApkPkg,
-				Metadata: *metadata,
+				Name:         metadata.Package,
+				Version:      metadata.Version,
+				Licenses:     strings.Split(metadata.License, " "),
+				Type:         pkg.ApkPkg,
+				MetadataType: pkg.ApkMetadataType,
+				Metadata:     *metadata,
 			})
 		}
 	}

syft/cataloger/apkdb/parse_apk_db_test.go

@@ -160,10 +160,11 @@ func TestMultiplePackages(t *testing.T) {
 			fixture: "test-fixtures/multiple",
 			expected: []pkg.Package{
 				{
-					Name:     "libc-utils",
-					Version:  "0.7.2-r0",
-					Licenses: []string{"BSD"},
-					Type:     pkg.ApkPkg,
+					Name:         "libc-utils",
+					Version:      "0.7.2-r0",
+					Licenses:     []string{"BSD"},
+					Type:         pkg.ApkPkg,
+					MetadataType: pkg.ApkMetadataType,
 					Metadata: pkg.ApkMetadata{
 						Package:          "libc-utils",
 						OriginPackage:    "libc-dev",
@@ -182,10 +183,11 @@ func TestMultiplePackages(t *testing.T) {
 					},
 				},
 				{
-					Name:     "musl-utils",
-					Version:  "1.1.24-r2",
-					Licenses: []string{"MIT", "BSD", "GPL2+"},
-					Type:     pkg.ApkPkg,
+					Name:         "musl-utils",
+					Version:      "1.1.24-r2",
+					Licenses:     []string{"MIT", "BSD", "GPL2+"},
+					Type:         pkg.ApkPkg,
+					MetadataType: pkg.ApkMetadataType,
 					Metadata: pkg.ApkMetadata{
 						Package:          "musl-utils",
 						OriginPackage:    "musl",

syft/cataloger/catalog.go

@@ -1,7 +1,6 @@
 package cataloger
 
 import (
-	"github.com/anchore/stereoscope/pkg/file"
 	"github.com/anchore/syft/internal/bus"
 	"github.com/anchore/syft/internal/log"
 	"github.com/anchore/syft/syft/event"
@@ -39,36 +38,21 @@ func newMonitor() (*progress.Manual, *progress.Manual) {
 // request.
 func Catalog(resolver scope.Resolver, catalogers ...Cataloger) (*pkg.Catalog, error) {
 	catalog := pkg.NewCatalog()
-	fileSelection := make([]file.Reference, 0)
-
 	filesProcessed, packagesDiscovered := newMonitor()
 
-	// ask catalogers for files to extract from the image tar
-	for _, a := range catalogers {
-		fileSelection = append(fileSelection, a.SelectFiles(resolver)...)
-		log.Debugf("cataloger '%s' selected '%d' files", a.Name(), len(fileSelection))
-		filesProcessed.N += int64(len(fileSelection))
-	}
-
-	// fetch contents for requested selection by catalogers
-	// TODO: we should consider refactoring to return a set of io.Readers instead of the full contents themselves (allow for optional buffering).
-	contents, err := resolver.MultipleFileContentsByRef(fileSelection...)
-	if err != nil {
-		return nil, err
-	}
-
 	// perform analysis, accumulating errors for each failed analysis
 	var errs error
-	for _, a := range catalogers {
-		// TODO: check for multiple rounds of analyses by Iterate error
-		packages, err := a.Catalog(contents)
+	for _, theCataloger := range catalogers {
+		packages, err := theCataloger.Catalog(resolver)
 		if err != nil {
 			errs = multierror.Append(errs, err)
 			continue
 		}
 
-		log.Debugf("cataloger '%s' discovered '%d' packages", a.Name(), len(packages))
-		packagesDiscovered.N += int64(len(packages))
+		catalogedPackages := len(packages)
+
+		log.Debugf("cataloger '%s' discovered '%d' packages", theCataloger.Name(), catalogedPackages)
+		packagesDiscovered.N += int64(catalogedPackages)
 
 		for _, p := range packages {
 			catalog.Add(p)

syft/cataloger/cataloger.go

@@ -6,7 +6,6 @@ catalogers defined in child packages as well as the interface definition to impl
 package cataloger
 
 import (
-	"github.com/anchore/stereoscope/pkg/file"
 	"github.com/anchore/syft/syft/cataloger/apkdb"
 	"github.com/anchore/syft/syft/cataloger/deb"
 	"github.com/anchore/syft/syft/cataloger/golang"
@@ -25,20 +24,16 @@ import (
 type Cataloger interface {
 	// Name returns a string that uniquely describes a cataloger
 	Name() string
-	// SelectFiles discovers and returns specific files that the cataloger would like to inspect the contents of.
-	SelectFiles(scope.FileResolver) []file.Reference
-	// Catalog is given the file contents and should return any discovered Packages after analyzing the contents.
-	Catalog(map[file.Reference]string) ([]pkg.Package, error)
-	// TODO: add "IterationNeeded" error to indicate to the driver to continue with another Select/Catalog pass
-	// TODO: we should consider refactoring to return a set of io.Readers instead of the full contents themselves (allow for optional buffering).
+	// Catalog is given an object to resolve file references and content, this function returns any discovered Packages after analyzing the catalog source.
+	Catalog(resolver scope.Resolver) ([]pkg.Package, error)
 }
 
 // ImageCatalogers returns a slice of locally implemented catalogers that are fit for detecting installations of packages.
 func ImageCatalogers() []Cataloger {
 	return []Cataloger{
 		ruby.NewGemSpecCataloger(),
-		python.NewPythonCataloger(),         // TODO: split and replace me
-		javascript.NewJavascriptCataloger(), // TODO: split and replace me
+		python.NewPythonPackageCataloger(),
+		javascript.NewJavascriptPackageCataloger(),
 		deb.NewDpkgdbCataloger(),
 		rpmdb.NewRpmdbCataloger(),
 		java.NewJavaCataloger(),
@@ -51,8 +46,9 @@ func ImageCatalogers() []Cataloger {
 func DirectoryCatalogers() []Cataloger {
 	return []Cataloger{
 		ruby.NewGemFileLockCataloger(),
-		python.NewPythonCataloger(),         // TODO: split and replace me
-		javascript.NewJavascriptCataloger(), // TODO: split and replace me
+		python.NewPythonIndexCataloger(),
+		python.NewPythonPackageCataloger(),
+		javascript.NewJavascriptLockCataloger(),
 		deb.NewDpkgdbCataloger(),
 		rpmdb.NewRpmdbCataloger(),
 		java.NewJavaCataloger(),

syft/cataloger/common/generic_cataloger.go

@@ -34,73 +34,83 @@ func NewGenericCataloger(pathParsers map[string]ParserFn, globParsers map[string
 }
 
 // Name returns a string that uniquely describes the upstream cataloger that this Generic Cataloger represents.
-func (a *GenericCataloger) Name() string {
-	return a.upstreamCataloger
+func (c *GenericCataloger) Name() string {
+	return c.upstreamCataloger
 }
 
 // register pairs a set of file references with a parser function for future cataloging (when the file contents are resolved)
-func (a *GenericCataloger) register(files []file.Reference, parser ParserFn) {
-	a.selectedFiles = append(a.selectedFiles, files...)
+func (c *GenericCataloger) register(files []file.Reference, parser ParserFn) {
+	c.selectedFiles = append(c.selectedFiles, files...)
 	for _, f := range files {
-		a.parsers[f] = parser
+		c.parsers[f] = parser
 	}
 }
 
 // clear deletes all registered file-reference-to-parser-function pairings from former SelectFiles() and register() calls
-func (a *GenericCataloger) clear() {
-	a.selectedFiles = make([]file.Reference, 0)
-	a.parsers = make(map[file.Reference]ParserFn)
+func (c *GenericCataloger) clear() {
+	c.selectedFiles = make([]file.Reference, 0)
+	c.parsers = make(map[file.Reference]ParserFn)
+}
+
+// Catalog is given an object to resolve file references and content, this function returns any discovered Packages after analyzing the catalog source.
+func (c *GenericCataloger) Catalog(resolver scope.Resolver) ([]pkg.Package, error) {
+	fileSelection := c.selectFiles(resolver)
+	contents, err := resolver.MultipleFileContentsByRef(fileSelection...)
+	if err != nil {
+		return nil, err
+	}
+	return c.catalog(contents)
 }
 
 // SelectFiles takes a set of file trees and resolves and file references of interest for future cataloging
-func (a *GenericCataloger) SelectFiles(resolver scope.FileResolver) []file.Reference {
+func (c *GenericCataloger) selectFiles(resolver scope.FileResolver) []file.Reference {
 	// select by exact path
-	for path, parser := range a.pathParsers {
+	for path, parser := range c.pathParsers {
 		files, err := resolver.FilesByPath(file.Path(path))
 		if err != nil {
 			log.Errorf("cataloger failed to select files by path: %+v", err)
 		}
 		if files != nil {
-			a.register(files, parser)
+			c.register(files, parser)
 		}
 	}
 
 	// select by glob pattern
-	for globPattern, parser := range a.globParsers {
+	for globPattern, parser := range c.globParsers {
 		fileMatches, err := resolver.FilesByGlob(globPattern)
 		if err != nil {
 			log.Errorf("failed to find files by glob: %s", globPattern)
 		}
 		if fileMatches != nil {
-			a.register(fileMatches, parser)
+			c.register(fileMatches, parser)
 		}
 	}
 
-	return a.selectedFiles
+	return c.selectedFiles
 }
 
-// Catalog takes a set of file contents and uses any configured parser functions to resolve and return discovered packages
-func (a *GenericCataloger) Catalog(contents map[file.Reference]string) ([]pkg.Package, error) {
-	defer a.clear()
+// catalog takes a set of file contents and uses any configured parser functions to resolve and return discovered packages
+func (c *GenericCataloger) catalog(contents map[file.Reference]string) ([]pkg.Package, error) {
+	defer c.clear()
 
 	packages := make([]pkg.Package, 0)
 
-	for reference, parser := range a.parsers {
+	for reference, parser := range c.parsers {
 		content, ok := contents[reference]
 		if !ok {
-			log.Errorf("cataloger '%s' missing file content: %+v", a.upstreamCataloger, reference)
+			log.Errorf("cataloger '%s' missing file content: %+v", c.upstreamCataloger, reference)
 			continue
 		}
 
 		entries, err := parser(string(reference.Path), strings.NewReader(content))
 		if err != nil {
 			// TODO: should we fail? or only log?
-			log.Errorf("cataloger '%s' failed to parse entries (reference=%+v): %+v", a.upstreamCataloger, reference, err)
+			log.Errorf("cataloger '%s' failed to parse entries (reference=%+v): %+v", c.upstreamCataloger, reference, err)
 			continue
 		}
 
 		for _, entry := range entries {
-			entry.FoundBy = a.upstreamCataloger
+			entry.FoundBy = c.upstreamCataloger
 			entry.Source = []file.Reference{reference}
 
 			packages = append(packages, entry)

syft/cataloger/common/generic_cataloger_test.go

@@ -7,21 +7,28 @@ import (
 	"testing"
 
 	"github.com/anchore/stereoscope/pkg/file"
-	"github.com/anchore/syft/internal"
 	"github.com/anchore/syft/syft/pkg"
 )
 
-type testResolver struct {
+type testResolverMock struct {
 	contents map[file.Reference]string
 }
 
-func newTestResolver() *testResolver {
-	return &testResolver{
+func newTestResolver() *testResolverMock {
+	return &testResolverMock{
 		contents: make(map[file.Reference]string),
 	}
 }
 
-func (r *testResolver) FilesByPath(paths ...file.Path) ([]file.Reference, error) {
+func (r *testResolverMock) FileContentsByRef(_ file.Reference) (string, error) {
+	return "", fmt.Errorf("not implemented")
+}
+
+func (r *testResolverMock) MultipleFileContentsByRef(_ ...file.Reference) (map[file.Reference]string, error) {
+	return r.contents, nil
+}
+
+func (r *testResolverMock) FilesByPath(paths ...file.Path) ([]file.Reference, error) {
 	results := make([]file.Reference, len(paths))
 
 	for idx, p := range paths {
@@ -32,13 +39,17 @@ func (r *testResolver) FilesByPath(paths ...file.Path) ([]file.Reference, error)
 	return results, nil
 }
 
-func (r *testResolver) FilesByGlob(patterns ...string) ([]file.Reference, error) {
+func (r *testResolverMock) FilesByGlob(_ ...string) ([]file.Reference, error) {
 	path := "/a-path.txt"
 	ref := file.NewFileReference(file.Path(path))
 	r.contents[ref] = fmt.Sprintf("%s file contents!", path)
 	return []file.Reference{ref}, nil
 }
 
+func (r *testResolverMock) RelativeFileByPath(_ file.Reference, _ string) (*file.Reference, error) {
+	return nil, fmt.Errorf("not implemented")
+}
+
 func parser(_ string, reader io.Reader) ([]pkg.Package, error) {
 	contents, err := ioutil.ReadAll(reader)
 	if err != nil {
@@ -64,31 +75,16 @@ func TestGenericCataloger(t *testing.T) {
 	resolver := newTestResolver()
 	cataloger := NewGenericCataloger(pathParsers, globParsers, upstream)
 
-	selected := cataloger.SelectFiles(resolver)
-
-	if len(selected) != 3 {
-		t.Fatalf("unexpected selection length: %d", len(selected))
-	}
-
-	expectedSelection := internal.NewStringSetFromSlice([]string{"/last/path.txt", "/another-path.txt", "/a-path.txt"})
-	selectionByPath := make(map[string]file.Reference)
-	for _, s := range selected {
-		if !expectedSelection.Contains(string(s.Path)) {
-			t.Errorf("unexpected selection path: %+v", s.Path)
-		}
-		selectionByPath[string(s.Path)] = s
-	}
-
-	expectedPkgs := make(map[file.Reference]pkg.Package)
-	for path, ref := range selectionByPath {
-		expectedPkgs[ref] = pkg.Package{
+	expectedSelection := []string{"/last/path.txt", "/another-path.txt", "/a-path.txt"}
+	expectedPkgs := make(map[string]pkg.Package)
+	for _, path := range expectedSelection {
+		expectedPkgs[path] = pkg.Package{
 			FoundBy: upstream,
-			Source:  []file.Reference{ref},
 			Name:    fmt.Sprintf("%s file contents!", path),
 		}
 	}
 
-	actualPkgs, err := cataloger.Catalog(resolver.contents)
+	actualPkgs, err := cataloger.Catalog(resolver)
 	if err != nil {
 		t.Fatalf("cataloger catalog action failed: %+v", err)
 	}
@@ -99,7 +95,7 @@ func TestGenericCataloger(t *testing.T) {
 
 	for _, p := range actualPkgs {
 		ref := p.Source[0]
-		exP, ok := expectedPkgs[ref]
+		exP, ok := expectedPkgs[string(ref.Path)]
 		if !ok {
 			t.Errorf("missing expected pkg: ref=%+v", ref)
 			continue

syft/cataloger/deb/parse_dpkg_status.go

@@ -30,10 +30,11 @@ func parseDpkgStatus(_ string, reader io.Reader) ([]pkg.Package, error) {
 			return nil, err
 		}
 		packages = append(packages, pkg.Package{
-			Name:     entry.Package,
-			Version:  entry.Version,
-			Type:     pkg.DebPkg,
-			Metadata: entry,
+			Name:         entry.Package,
+			Version:      entry.Version,
+			Type:         pkg.DebPkg,
+			MetadataType: pkg.DpkgMetadataType,
+			Metadata:     entry,
 		})
 	}
 

syft/cataloger/java/archive_parser.go

@@ -5,6 +5,8 @@ import (
 	"io"
 	"strings"
 
+	"github.com/anchore/syft/internal/log"
+
 	"github.com/anchore/syft/internal"
 	"github.com/anchore/syft/internal/file"
 	"github.com/anchore/syft/syft/cataloger/common"
@@ -140,10 +142,11 @@ func (j *archiveParser) discoverMainPackage() (*pkg.Package, error) {
 	}
 
 	return &pkg.Package{
-		Name:     selectName(manifest, j.fileInfo),
-		Version:  selectVersion(manifest, j.fileInfo),
-		Language: pkg.Java,
-		Type:     pkg.JavaPkg,
+		Name:         selectName(manifest, j.fileInfo),
+		Version:      selectVersion(manifest, j.fileInfo),
+		Language:     pkg.Java,
+		Type:         pkg.JavaPkg,
+		MetadataType: pkg.JavaMetadataType,
 		Metadata: pkg.JavaMetadata{
 			Manifest: manifest,
 		},
@@ -175,10 +178,11 @@ func (j *archiveParser) discoverPkgsFromPomProperties(parentPkg *pkg.Package) ([
 
 				// discovered props = new package
 				p := pkg.Package{
-					Name:     propsObj.ArtifactID,
-					Version:  propsObj.Version,
-					Language: pkg.Java,
-					Type:     pkg.JavaPkg,
+					Name:         propsObj.ArtifactID,
+					Version:      propsObj.Version,
+					Language:     pkg.Java,
+					Type:         pkg.JavaPkg,
+					MetadataType: pkg.JavaMetadataType,
 					Metadata: pkg.JavaMetadata{
 						PomProperties: propsObj,
 						Parent:        parentPkg,
@@ -214,18 +218,28 @@ func (j *archiveParser) discoverPkgsFromNestedArchives(parentPkg *pkg.Package) (
 	}
 
 	// search and parse pom.properties files & fetch the contents
-	readers, err := file.ExtractFromZipToUniqueTempFile(j.archivePath, j.contentPath, j.fileManifest.GlobMatch(archiveFormatGlobs...)...)
+	openers, err := file.ExtractFromZipToUniqueTempFile(j.archivePath, j.contentPath, j.fileManifest.GlobMatch(archiveFormatGlobs...)...)
 	if err != nil {
 		return nil, fmt.Errorf("unable to extract files from zip: %w", err)
 	}
 
 	// discover nested artifacts
-	for archivePath, archiveReader := range readers {
-		nestedPath := fmt.Sprintf("%s:%s", j.virtualPath, archivePath)
-		nestedPkgs, err := parseJavaArchive(nestedPath, archiveReader)
+	for archivePath, archiveOpener := range openers {
+		archiveReadCloser, err := archiveOpener.Open()
 		if err != nil {
+			return nil, fmt.Errorf("unable to open archived file from tempdir: %w", err)
+		}
+		nestedPath := fmt.Sprintf("%s:%s", j.virtualPath, archivePath)
+		nestedPkgs, err := parseJavaArchive(nestedPath, archiveReadCloser)
+		if err != nil {
+			if closeErr := archiveReadCloser.Close(); closeErr != nil {
+				log.Errorf("unable to close archived file from tempdir: %+v", closeErr)
+			}
 			return nil, fmt.Errorf("unable to process nested java archive (%s): %w", archivePath, err)
 		}
+		if err = archiveReadCloser.Close(); err != nil {
+			return nil, fmt.Errorf("unable to close archived file from tempdir: %w", err)
+		}
 
 		// attach the parent package to all discovered packages that are not already associated with a java archive
 		for _, p := range nestedPkgs {

syft/cataloger/java/archive_parser_test.go

@@ -137,10 +137,11 @@ func TestParseJar(t *testing.T) {
 			},
 			expected: map[string]pkg.Package{
 				"example-jenkins-plugin": {
-					Name:     "example-jenkins-plugin",
-					Version:  "1.0-SNAPSHOT",
-					Language: pkg.Java,
-					Type:     pkg.JenkinsPluginPkg,
+					Name:         "example-jenkins-plugin",
+					Version:      "1.0-SNAPSHOT",
+					Language:     pkg.Java,
+					Type:         pkg.JenkinsPluginPkg,
+					MetadataType: pkg.JavaMetadataType,
 					Metadata: pkg.JavaMetadata{
 						Manifest: &pkg.JavaManifest{
 							ManifestVersion: "1.0",
@@ -181,10 +182,11 @@ func TestParseJar(t *testing.T) {
 			fixture: "test-fixtures/java-builds/packages/example-java-app-gradle-0.1.0.jar",
 			expected: map[string]pkg.Package{
 				"example-java-app-gradle": {
-					Name:     "example-java-app-gradle",
-					Version:  "0.1.0",
-					Language: pkg.Java,
-					Type:     pkg.JavaPkg,
+					Name:         "example-java-app-gradle",
+					Version:      "0.1.0",
+					Language:     pkg.Java,
+					Type:         pkg.JavaPkg,
+					MetadataType: pkg.JavaMetadataType,
 					Metadata: pkg.JavaMetadata{
 						Manifest: &pkg.JavaManifest{
 							ManifestVersion: "1.0",
@@ -200,10 +202,11 @@ func TestParseJar(t *testing.T) {
 			},
 			expected: map[string]pkg.Package{
 				"example-java-app-maven": {
-					Name:     "example-java-app-maven",
-					Version:  "0.1.0",
-					Language: pkg.Java,
-					Type:     pkg.JavaPkg,
+					Name:         "example-java-app-maven",
+					Version:      "0.1.0",
+					Language:     pkg.Java,
+					Type:         pkg.JavaPkg,
+					MetadataType: pkg.JavaMetadataType,
 					Metadata: pkg.JavaMetadata{
 						Manifest: &pkg.JavaManifest{
 							ManifestVersion: "1.0",
@@ -224,10 +227,11 @@ func TestParseJar(t *testing.T) {
 					},
 				},
 				"joda-time": {
-					Name:     "joda-time",
-					Version:  "2.9.2",
-					Language: pkg.Java,
-					Type:     pkg.JavaPkg,
+					Name:         "joda-time",
+					Version:      "2.9.2",
+					Language:     pkg.Java,
+					Type:         pkg.JavaPkg,
+					MetadataType: pkg.JavaMetadataType,
 					Metadata: pkg.JavaMetadata{
 						PomProperties: &pkg.PomProperties{
 							Path:       "META-INF/maven/joda-time/joda-time/pom.properties",

syft/cataloger/java/save_archive_to_tmp.go

@@ -35,6 +35,7 @@ func saveArchiveToTmp(reader io.Reader) (string, string, func(), error) {
 	if err != nil {
 		return contentDir, "", cleanupFn, fmt.Errorf("unable to create archive: %w", err)
 	}
+	defer archiveFile.Close()
 
 	_, err = io.Copy(archiveFile, reader)
 	if err != nil {

syft/cataloger/javascript/cataloger.go

@@ -7,12 +7,21 @@ import (
 	"github.com/anchore/syft/syft/cataloger/common"
 )
 
-// NewJavascriptCataloger returns a new JavaScript cataloger object.
-func NewJavascriptCataloger() *common.GenericCataloger {
+// NewJavascriptPackageCataloger returns a new JavaScript cataloger object based on detection of npm based packages.
+func NewJavascriptPackageCataloger() *common.GenericCataloger {
+	globParsers := map[string]common.ParserFn{
+		"**/package.json": parsePackageJSON,
+	}
+
+	return common.NewGenericCataloger(nil, globParsers, "javascript-package-cataloger")
+}
+
+// NewJavascriptLockCataloger returns a new Javascript cataloger object base on package lock files.
+func NewJavascriptLockCataloger() *common.GenericCataloger {
 	globParsers := map[string]common.ParserFn{
 		"**/package-lock.json": parsePackageLock,
 		"**/yarn.lock":         parseYarnLock,
 	}
 
-	return common.NewGenericCataloger(nil, globParsers, "javascript-cataloger")
+	return common.NewGenericCataloger(nil, globParsers, "javascript-lock-cataloger")
 }

syft/cataloger/javascript/parse_package_json.go

@@ -0,0 +1,55 @@
+package javascript
+
+import (
+	"encoding/json"
+	"fmt"
+	"io"
+
+	"github.com/anchore/syft/syft/cataloger/common"
+	"github.com/anchore/syft/syft/pkg"
+)
+
+// integrity check
+var _ common.ParserFn = parsePackageLock
+
+// PackageJSON represents a JavaScript package.json file
+type PackageJSON struct {
+	Version      string            `json:"version"`
+	Latest       []string          `json:"latest"`
+	Author       string            `json:"author"`
+	License      string            `json:"license"`
+	Name         string            `json:"name"`
+	Homepage     string            `json:"homepage"`
+	Description  string            `json:"description"`
+	Dependencies map[string]string `json:"dependencies"`
+}
+
+// parsePackageJson parses a package.json and returns the discovered JavaScript packages.
+func parsePackageJSON(_ string, reader io.Reader) ([]pkg.Package, error) {
+	packages := make([]pkg.Package, 0)
+	dec := json.NewDecoder(reader)
+
+	for {
+		var p PackageJSON
+		if err := dec.Decode(&p); err == io.EOF {
+			break
+		} else if err != nil {
+			return nil, fmt.Errorf("failed to parse package.json file: %w", err)
+		}
+
+		packages = append(packages, pkg.Package{
+			Name:         p.Name,
+			Version:      p.Version,
+			Licenses:     []string{p.License},
+			Language:     pkg.JavaScript,
+			Type:         pkg.NpmPkg,
+			MetadataType: pkg.NpmPackageJSONMetadataType,
+			Metadata: pkg.NpmPackageJSONMetadata{
+				Author:   p.Author,
+				Homepage: p.Homepage,
+			},
+		})
+	}
+
+	return packages, nil
+}

syft/cataloger/javascript/parse_package_json_test.go

@@ -0,0 +1,44 @@
+package javascript
+
+import (
+	"os"
+	"testing"
+
+	"github.com/anchore/syft/syft/pkg"
+	"github.com/go-test/deep"
+)
+
+func TestParsePackageJSON(t *testing.T) {
+	expected := pkg.Package{
+		Name:         "npm",
+		Version:      "6.14.6",
+		Type:         pkg.NpmPkg,
+		Licenses:     []string{"Artistic-2.0"},
+		Language:     pkg.JavaScript,
+		MetadataType: pkg.NpmPackageJSONMetadataType,
+		Metadata: pkg.NpmPackageJSONMetadata{
+			Author:   "Isaac Z. Schlueter <i@izs.me> (http://blog.izs.me)",
+			Homepage: "https://docs.npmjs.com/",
+		},
+	}
+	fixture, err := os.Open("test-fixtures/pkg-json/package.json")
+	if err != nil {
+		t.Fatalf("failed to open fixture: %+v", err)
+	}
+
+	actual, err := parsePackageJSON(fixture.Name(), fixture)
+	if err != nil {
+		t.Fatalf("failed to parse package-lock.json: %+v", err)
+	}
+	if len(actual) != 1 {
+		for _, a := range actual {
+			t.Log("   ", a)
+		}
+		t.Fatalf("unexpected package count: %d!=1", len(actual))
+	}
+
+	for _, d := range deep.Equal(actual[0], expected) {
+		t.Errorf("diff: %+v", d)
+	}
+
+}

syft/cataloger/javascript/parse_yarn_lock.go

@@ -66,7 +66,7 @@ func parseYarnLock(_ string, reader io.Reader) ([]pkg.Package, error) {
 				Name:     currentName,
 				Version:  strings.Trim(version, "\""),
 				Language: pkg.JavaScript,
-				Type:     pkg.YarnPkg,
+				Type:     pkg.NpmPkg,
 			})
 		}
 	}

syft/cataloger/javascript/parse_yarn_lock_test.go

@@ -13,43 +13,43 @@ func TestParseYarnLock(t *testing.T) {
 			Name:     "@babel/code-frame",
 			Version:  "7.10.4",
 			Language: pkg.JavaScript,
-			Type:     pkg.YarnPkg,
+			Type:     pkg.NpmPkg,
 		},
 		"@types/minimatch": {
 			Name:     "@types/minimatch",
 			Version:  "3.0.3",
 			Language: pkg.JavaScript,
-			Type:     pkg.YarnPkg,
+			Type:     pkg.NpmPkg,
 		},
 		"@types/qs": {
 			Name:     "@types/qs",
 			Version:  "6.9.4",
 			Language: pkg.JavaScript,
-			Type:     pkg.YarnPkg,
+			Type:     pkg.NpmPkg,
 		},
 		"ajv": {
 			Name:     "ajv",
 			Version:  "6.12.3",
 			Language: pkg.JavaScript,
-			Type:     pkg.YarnPkg,
+			Type:     pkg.NpmPkg,
 		},
 		"atob": {
 			Name:     "atob",
 			Version:  "2.1.2",
 			Language: pkg.JavaScript,
-			Type:     pkg.YarnPkg,
+			Type:     pkg.NpmPkg,
 		},
 		"aws-sdk": {
 			Name:     "aws-sdk",
 			Version:  "2.706.0",
 			Language: pkg.JavaScript,
-			Type:     pkg.YarnPkg,
+			Type:     pkg.NpmPkg,
 		},
 		"jhipster-core": {
 			Name:     "jhipster-core",
 			Version:  "7.3.4",
 			Language: pkg.JavaScript,
-			Type:     pkg.YarnPkg,
+			Type:     pkg.NpmPkg,
 		},
 	}
 	fixture, err := os.Open("test-fixtures/yarn/yarn.lock")

syft/cataloger/javascript/test-fixtures/pkg-json/package.json

@@ -0,0 +1,314 @@
+{
+  "version": "6.14.6",
+  "name": "npm",
+  "description": "a package manager for JavaScript",
+  "keywords": [
+    "install",
+    "modules",
+    "package manager",
+    "package.json"
+  ],
+  "preferGlobal": true,
+  "config": {
+    "publishtest": false
+  },
+  "homepage": "https://docs.npmjs.com/",
+  "author": "Isaac Z. Schlueter <i@izs.me> (http://blog.izs.me)",
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/npm/cli"
+  },
+  "bugs": {
+    "url": "https://npm.community/c/bugs"
+  },
+  "directories": {
+    "bin": "./bin",
+    "doc": "./doc",
+    "lib": "./lib",
+    "man": "./man"
+  },
+  "main": "./lib/npm.js",
+  "bin": {
+    "npm": "./bin/npm-cli.js",
+    "npx": "./bin/npx-cli.js"
+  },
+  "dependencies": {
+    "JSONStream": "^1.3.5",
+    "abbrev": "~1.1.1",
+    "ansicolors": "~0.3.2",
+    "ansistyles": "~0.1.3",
+    "aproba": "^2.0.0",
+    "archy": "~1.0.0",
+    "bin-links": "^1.1.7",
+    "bluebird": "^3.5.5",
+    "byte-size": "^5.0.1",
+    "cacache": "^12.0.3",
+    "call-limit": "^1.1.1",
+    "chownr": "^1.1.4",
+    "ci-info": "^2.0.0",
+    "cli-columns": "^3.1.2",
+    "cli-table3": "^0.5.1",
+    "cmd-shim": "^3.0.3",
+    "columnify": "~1.5.4",
+    "config-chain": "^1.1.12",
+    "detect-indent": "~5.0.0",
+    "detect-newline": "^2.1.0",
+    "dezalgo": "~1.0.3",
+    "editor": "~1.0.0",
+    "figgy-pudding": "^3.5.1",
+    "find-npm-prefix": "^1.0.2",
+    "fs-vacuum": "~1.2.10",
+    "fs-write-stream-atomic": "~1.0.10",
+    "gentle-fs": "^2.3.0",
+    "glob": "^7.1.6",
+    "graceful-fs": "^4.2.4",
+    "has-unicode": "~2.0.1",
+    "hosted-git-info": "^2.8.8",
+    "iferr": "^1.0.2",
+    "infer-owner": "^1.0.4",
+    "inflight": "~1.0.6",
+    "inherits": "^2.0.4",
+    "ini": "^1.3.5",
+    "init-package-json": "^1.10.3",
+    "is-cidr": "^3.0.0",
+    "json-parse-better-errors": "^1.0.2",
+    "lazy-property": "~1.0.0",
+    "libcipm": "^4.0.7",
+    "libnpm": "^3.0.1",
+    "libnpmaccess": "^3.0.2",
+    "libnpmhook": "^5.0.3",
+    "libnpmorg": "^1.0.1",
+    "libnpmsearch": "^2.0.2",
+    "libnpmteam": "^1.0.2",
+    "libnpx": "^10.2.2",
+    "lock-verify": "^2.1.0",
+    "lockfile": "^1.0.4",
+    "lodash._baseuniq": "~4.6.0",
+    "lodash.clonedeep": "~4.5.0",
+    "lodash.union": "~4.6.0",
+    "lodash.uniq": "~4.5.0",
+    "lodash.without": "~4.4.0",
+    "lru-cache": "^5.1.1",
+    "meant": "~1.0.1",
+    "mississippi": "^3.0.0",
+    "mkdirp": "^0.5.5",
+    "move-concurrently": "^1.0.1",
+    "node-gyp": "^5.1.0",
+    "nopt": "^4.0.3",
+    "normalize-package-data": "^2.5.0",
+    "npm-audit-report": "^1.3.2",
+    "npm-cache-filename": "~1.0.2",
+    "npm-install-checks": "^3.0.2",
+    "npm-lifecycle": "^3.1.4",
+    "npm-package-arg": "^6.1.1",
+    "npm-packlist": "^1.4.8",
+    "npm-pick-manifest": "^3.0.2",
+    "npm-profile": "^4.0.4",
+    "npm-registry-fetch": "^4.0.5",
+    "npm-user-validate": "~1.0.0",
+    "npmlog": "~4.1.2",
+    "once": "~1.4.0",
+    "opener": "^1.5.1",
+    "osenv": "^0.1.5",
+    "pacote": "^9.5.12",
+    "path-is-inside": "~1.0.2",
+    "promise-inflight": "~1.0.1",
+    "qrcode-terminal": "^0.12.0",
+    "query-string": "^6.8.2",
+    "qw": "~1.0.1",
+    "read": "~1.0.7",
+    "read-cmd-shim": "^1.0.5",
+    "read-installed": "~4.0.3",
+    "read-package-json": "^2.1.1",
+    "read-package-tree": "^5.3.1",
+    "readable-stream": "^3.6.0",
+    "readdir-scoped-modules": "^1.1.0",
+    "request": "^2.88.0",
+    "retry": "^0.12.0",
+    "rimraf": "^2.7.1",
+    "safe-buffer": "^5.1.2",
+    "semver": "^5.7.1",
+    "sha": "^3.0.0",
+    "slide": "~1.1.6",
+    "sorted-object": "~2.0.1",
+    "sorted-union-stream": "~2.1.3",
+    "ssri": "^6.0.1",
+    "stringify-package": "^1.0.1",
+    "tar": "^4.4.13",
+    "text-table": "~0.2.0",
+    "tiny-relative-date": "^1.3.0",
+    "uid-number": "0.0.6",
+    "umask": "~1.1.0",
+    "unique-filename": "^1.1.1",
+    "unpipe": "~1.0.0",
+    "update-notifier": "^2.5.0",
+    "uuid": "^3.3.3",
+    "validate-npm-package-license": "^3.0.4",
+    "validate-npm-package-name": "~3.0.0",
+    "which": "^1.3.1",
+    "worker-farm": "^1.7.0",
+    "write-file-atomic": "^2.4.3"
+  },
+  "bundleDependencies": [
+    "abbrev",
+    "ansicolors",
+    "ansistyles",
+    "aproba",
+    "archy",
+    "bin-links",
+    "bluebird",
+    "byte-size",
+    "cacache",
+    "call-limit",
+    "chownr",
+    "ci-info",
+    "cli-columns",
+    "cli-table3",
+    "cmd-shim",
+    "columnify",
+    "config-chain",
+    "debuglog",
+    "detect-indent",
+    "detect-newline",
+    "dezalgo",
+    "editor",
+    "figgy-pudding",
+    "find-npm-prefix",
+    "fs-vacuum",
+    "fs-write-stream-atomic",
+    "gentle-fs",
+    "glob",
+    "graceful-fs",
+    "has-unicode",
+    "hosted-git-info",
+    "iferr",
+    "imurmurhash",
+    "infer-owner",
+    "inflight",
+    "inherits",
+    "ini",
+    "init-package-json",
+    "is-cidr",
+    "json-parse-better-errors",
+    "JSONStream",
+    "lazy-property",
+    "libcipm",
+    "libnpm",
+    "libnpmaccess",
+    "libnpmhook",
+    "libnpmorg",
+    "libnpmsearch",
+    "libnpmteam",
+    "libnpx",
+    "lock-verify",
+    "lockfile",
+    "lodash._baseindexof",
+    "lodash._baseuniq",
+    "lodash._bindcallback",
+    "lodash._cacheindexof",
+    "lodash._createcache",
+    "lodash._getnative",
+    "lodash.clonedeep",
+    "lodash.restparam",
+    "lodash.union",
+    "lodash.uniq",
+    "lodash.without",
+    "lru-cache",
+    "meant",
+    "mississippi",
+    "mkdirp",
+    "move-concurrently",
+    "node-gyp",
+    "nopt",
+    "normalize-package-data",
+    "npm-audit-report",
+    "npm-cache-filename",
+    "npm-install-checks",
+    "npm-lifecycle",
+    "npm-package-arg",
+    "npm-packlist",
+    "npm-pick-manifest",
+    "npm-profile",
+    "npm-registry-fetch",
+    "npm-user-validate",
+    "npmlog",
+    "once",
+    "opener",
+    "osenv",
+    "pacote",
+    "path-is-inside",
+    "promise-inflight",
+    "qrcode-terminal",
+    "query-string",
+    "qw",
+    "read-cmd-shim",
+    "read-installed",
+    "read-package-json",
+    "read-package-tree",
+    "read",
+    "readable-stream",
+    "readdir-scoped-modules",
+    "request",
+    "retry",
+    "rimraf",
+    "safe-buffer",
+    "semver",
+    "sha",
+    "slide",
+    "sorted-object",
+    "sorted-union-stream",
+    "ssri",
+    "stringify-package",
+    "tar",
+    "text-table",
+    "tiny-relative-date",
+    "uid-number",
+    "umask",
+    "unique-filename",
+    "unpipe",
+    "update-notifier",
+    "uuid",
+    "validate-npm-package-license",
+    "validate-npm-package-name",
+    "which",
+    "worker-farm",
+    "write-file-atomic"
+  ],
+  "devDependencies": {
+    "deep-equal": "^1.0.1",
+    "get-stream": "^4.1.0",
+    "licensee": "^7.0.3",
+    "marked": "^0.6.3",
+    "marked-man": "^0.6.0",
+    "npm-registry-couchapp": "^2.7.4",
+    "npm-registry-mock": "^1.3.1",
+    "require-inject": "^1.4.4",
+    "sprintf-js": "^1.1.2",
+    "standard": "^11.0.1",
+    "tacks": "^1.3.0",
+    "tap": "^12.7.0",
+    "tar-stream": "^2.1.0"
+  },
+  "scripts": {
+    "dumpconf": "env | grep npm | sort | uniq",
+    "prepare": "node bin/npm-cli.js rebuild && node bin/npm-cli.js --no-audit --no-timing prune --prefix=. --no-global && rimraf test/*/*/node_modules && make -j4 mandocs",
+    "preversion": "bash scripts/update-authors.sh && git add AUTHORS && git commit -m \"update AUTHORS\" || true",
+    "licenses": "licensee --production --errors-only",
+    "tap": "tap -J --timeout 300 --no-esm",
+    "tap-cover": "tap -J --nyc-arg=--cache --coverage --timeout 600 --no-esm",
+    "lint": "standard",
+    "pretest": "npm run lint",
+    "test": "npm run test-tap --",
+    "test:nocleanup": "NO_TEST_CLEANUP=1 npm run test --",
+    "sudotest": "sudo npm run tap -- \"test/tap/*.js\"",
+    "sudotest:nocleanup": "sudo NO_TEST_CLEANUP=1 npm run tap -- \"test/tap/*.js\"",
+    "posttest": "rimraf test/npm_cache*",
+    "test-coverage": "npm run tap-cover -- \"test/tap/*.js\" \"test/network/*.js\"",
+    "test-tap": "npm run tap -- \"test/tap/*.js\" \"test/network/*.js\"",
+    "test-node": "tap --timeout 240 \"test/tap/*.js\" \"test/network/*.js\""
+  },
+  "license": "Artistic-2.0",
+  "engines": {
+    "node": "6 >=6.2.0 || 8 || >=9.3.0"
+  }
+}

syft/cataloger/python/cataloger.go

@@ -1,21 +0,0 @@
-/*
-Package python provides a concrete Cataloger implementation for Python ecosystem files (egg, wheel, requirements.txt).
-*/
-package python
-
-import (
-	"github.com/anchore/syft/syft/cataloger/common"
-)
-
-// NewPythonCataloger returns a new Python cataloger object.
-func NewPythonCataloger() *common.GenericCataloger {
-	globParsers := map[string]common.ParserFn{
-		"**/*egg-info/PKG-INFO":  parseEggMetadata,
-		"**/*dist-info/METADATA": parseWheelMetadata,
-		"**/*requirements*.txt":  parseRequirementsTxt,
-		"**/poetry.lock":         parsePoetryLock,
-		"**/setup.py":            parseSetup,
-	}
-
-	return common.NewGenericCataloger(nil, globParsers, "python-cataloger")
-}

syft/cataloger/python/index_cataloger.go

@@ -0,0 +1,19 @@
+/*
+Package python provides a concrete Cataloger implementation for Python ecosystem files (egg, wheel, requirements.txt).
+*/
+package python
+
+import (
+	"github.com/anchore/syft/syft/cataloger/common"
+)
+
+// NewPythonIndexCataloger returns a new cataloger for python packages referenced from poetry lock files, requirements.txt files, and setup.py files.
+func NewPythonIndexCataloger() *common.GenericCataloger {
+	globParsers := map[string]common.ParserFn{
+		"**/*requirements*.txt": parseRequirementsTxt,
+		"**/poetry.lock":        parsePoetryLock,
+		"**/setup.py":           parseSetup,
+	}
+
+	return common.NewGenericCataloger(nil, globParsers, "python-index-cataloger")
+}

syft/cataloger/python/package_cataloger.go

@@ -0,0 +1,179 @@
+package python
+
+import (
+	"bufio"
+	"fmt"
+	"path/filepath"
+	"strings"
+
+	"github.com/anchore/stereoscope/pkg/file"
+
+	"github.com/anchore/syft/syft/pkg"
+
+	"github.com/anchore/syft/syft/scope"
+)
+
+const (
+	eggMetadataGlob   = "**/*egg-info/PKG-INFO"
+	wheelMetadataGlob = "**/*dist-info/METADATA"
+)
+
+type PackageCataloger struct{}
+
+// NewPythonPackageCataloger returns a new cataloger for python packages within egg or wheel installation directories.
+func NewPythonPackageCataloger() *PackageCataloger {
+	return &PackageCataloger{}
+}
+
+// Name returns a string that uniquely describes a cataloger
+func (c *PackageCataloger) Name() string {
+	return "python-package-cataloger"
+}
+
+// Catalog is given an object to resolve file references and content, this function returns any discovered Packages after analyzing python egg and wheel installations.
+func (c *PackageCataloger) Catalog(resolver scope.Resolver) ([]pkg.Package, error) {
+	// nolint:prealloc
+	var fileMatches []file.Reference
+
+	for _, glob := range []string{eggMetadataGlob, wheelMetadataGlob} {
+		matches, err := resolver.FilesByGlob(glob)
+		if err != nil {
+			return nil, fmt.Errorf("failed to find files by glob: %s", glob)
+		}
+		fileMatches = append(fileMatches, matches...)
+	}
+
+	var pkgs []pkg.Package
+	for _, ref := range fileMatches {
+		p, err := c.catalogEggOrWheel(resolver, ref)
+		if err != nil {
+			return nil, fmt.Errorf("unable to catalog python package=%+v: %w", ref.Path, err)
+		}
+		if p != nil {
+			pkgs = append(pkgs, *p)
+		}
+	}
+	return pkgs, nil
+}
+
+// catalogEggOrWheel takes the primary metadata file reference and returns the python package it represents.
+func (c *PackageCataloger) catalogEggOrWheel(resolver scope.Resolver, metadataRef file.Reference) (*pkg.Package, error) {
+	metadata, sources, err := c.assembleEggOrWheelMetadata(resolver, metadataRef)
+	if err != nil {
+		return nil, err
+	}
+
+	var licenses []string
+	if metadata.License != "" {
+		licenses = []string{metadata.License}
+	}
+
+	return &pkg.Package{
+		Name:         metadata.Name,
+		Version:      metadata.Version,
+		FoundBy:      c.Name(),
+		Source:       sources,
+		Licenses:     licenses,
+		Language:     pkg.Python,
+		Type:         pkg.PythonPkg,
+		MetadataType: pkg.PythonPackageMetadataType,
+		Metadata:     *metadata,
+	}, nil
+}
+
+// fetchRecordFiles finds a corresponding RECORD file for the given python package metadata file and returns the set of file records contained.
+func (c *PackageCataloger) fetchRecordFiles(resolver scope.Resolver, metadataRef file.Reference) (files []pkg.PythonFileRecord, sources []file.Reference, err error) {
+	// we've been given a file reference to a specific wheel METADATA file. note: this may be for a directory
+	// or for an image... for an image the METADATA file may be present within multiple layers, so it is important
+	// to reconcile the RECORD path to the same layer (or the next adjacent lower layer).
+
+	// lets find the RECORD file relative to the directory where the METADATA file resides (in path AND layer structure)
+	recordPath := filepath.Join(filepath.Dir(string(metadataRef.Path)), "RECORD")
+	recordRef, err := resolver.RelativeFileByPath(metadataRef, recordPath)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	if recordRef != nil {
+		sources = append(sources, *recordRef)
+
+		recordContents, err := resolver.FileContentsByRef(*recordRef)
+		if err != nil {
+			return nil, nil, err
+		}
+
+		// parse the record contents
+		records, err := parseWheelOrEggRecord(strings.NewReader(recordContents))
+		if err != nil {
+			return nil, nil, err
+		}
+
+		files = append(files, records...)
+	}
+	return files, sources, nil
+}
+
+// fetchTopLevelPackages finds a corresponding top_level.txt file for the given python package metadata file and returns the set of package names contained.
+func (c *PackageCataloger) fetchTopLevelPackages(resolver scope.Resolver, metadataRef file.Reference) (pkgs []string, sources []file.Reference, err error) {
+	// a top_level.txt file specifies the python top-level packages (provided by this python package) installed into site-packages
+	parentDir := filepath.Dir(string(metadataRef.Path))
+	topLevelPath := filepath.Join(parentDir, "top_level.txt")
+	topLevelRef, err := resolver.RelativeFileByPath(metadataRef, topLevelPath)
+	if err != nil {
+		return nil, nil, err
+	}
+	if topLevelRef == nil {
+		return nil, nil, fmt.Errorf("missing python package top_level.txt (package=%q)", string(metadataRef.Path))
+	}
+
+	sources = append(sources, *topLevelRef)
+
+	topLevelContents, err := resolver.FileContentsByRef(*topLevelRef)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	scanner := bufio.NewScanner(strings.NewReader(topLevelContents))
+	for scanner.Scan() {
+		pkgs = append(pkgs, scanner.Text())
+	}
+
+	if err := scanner.Err(); err != nil {
+		return nil, nil, fmt.Errorf("could not read python package top_level.txt: %w", err)
+	}
+
+	return pkgs, sources, nil
+}
+
+// assembleEggOrWheelMetadata discovers and accumulates python package metadata from multiple file sources and returns a single metadata object as well as a list of files where the metadata was derived from.
+func (c *PackageCataloger) assembleEggOrWheelMetadata(resolver scope.Resolver, metadataRef file.Reference) (*pkg.PythonPackageMetadata, []file.Reference, error) {
+	var sources = []file.Reference{metadataRef}
+
+	metadataContents, err := resolver.FileContentsByRef(metadataRef)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	metadata, err := parseWheelOrEggMetadata(metadataRef.Path, strings.NewReader(metadataContents))
+	if err != nil {
+		return nil, nil, err
+	}
+
+	// attach any python files found for the given wheel/egg installation
+	r, s, err := c.fetchRecordFiles(resolver, metadataRef)
+	if err != nil {
+		return nil, nil, err
+	}
+	sources = append(sources, s...)
+	metadata.Files = r
+
+	// attach any top-level package names found for the given wheel/egg installation
+	p, s, err := c.fetchTopLevelPackages(resolver, metadataRef)
+	if err != nil {
+		return nil, nil, err
+	}
+	sources = append(sources, s...)
+	metadata.TopLevelPackages = p
+
+	return &metadata, sources, nil
+}

syft/cataloger/python/package_cataloger_test.go

@@ -0,0 +1,241 @@
+package python
+
+import (
+	"fmt"
+	"io"
+	"io/ioutil"
+	"os"
+	"strings"
+	"testing"
+
+	"github.com/anchore/stereoscope/pkg/file"
+
+	"github.com/anchore/syft/syft/pkg"
+	"github.com/go-test/deep"
+)
+
+type pythonTestResolverMock struct {
+	metadataReader io.Reader
+	recordReader   io.Reader
+	topLevelReader io.Reader
+	metadataRef    *file.Reference
+	recordRef      *file.Reference
+	topLevelRef    *file.Reference
+	contents       map[file.Reference]string
+}
+
+func newTestResolver(metaPath, recordPath, topPath string) *pythonTestResolverMock {
+	metadataReader, err := os.Open(metaPath)
+	if err != nil {
+		panic(fmt.Errorf("failed to open metadata: %+v", err))
+	}
+
+	var recordReader io.Reader
+	if recordPath != "" {
+		recordReader, err = os.Open(recordPath)
+		if err != nil {
+			panic(fmt.Errorf("failed to open record: %+v", err))
+		}
+	}
+
+	var topLevelReader io.Reader
+	if topPath != "" {
+		topLevelReader, err = os.Open(topPath)
+		if err != nil {
+			panic(fmt.Errorf("failed to open top level: %+v", err))
+		}
+	}
+
+	var recordRef *file.Reference
+	if recordReader != nil {
+		ref := file.NewFileReference("test-fixtures/dist-info/RECORD")
+		recordRef = &ref
+	}
+	var topLevelRef *file.Reference
+	if topLevelReader != nil {
+		ref := file.NewFileReference("test-fixtures/dist-info/top_level.txt")
+		topLevelRef = &ref
+	}
+	metadataRef := file.NewFileReference("test-fixtures/dist-info/METADATA")
+	return &pythonTestResolverMock{
+		recordReader:   recordReader,
+		metadataReader: metadataReader,
+		topLevelReader: topLevelReader,
+		metadataRef:    &metadataRef,
+		recordRef:      recordRef,
+		topLevelRef:    topLevelRef,
+		contents:       make(map[file.Reference]string),
+	}
+}
+
+func (r *pythonTestResolverMock) FileContentsByRef(ref file.Reference) (string, error) {
+	switch ref.Path {
+	case r.topLevelRef.Path:
+		b, err := ioutil.ReadAll(r.topLevelReader)
+		if err != nil {
+			return "", err
+		}
+		return string(b), nil
+	case r.metadataRef.Path:
+		b, err := ioutil.ReadAll(r.metadataReader)
+		if err != nil {
+			return "", err
+		}
+		return string(b), nil
+	case r.recordRef.Path:
+		b, err := ioutil.ReadAll(r.recordReader)
+		if err != nil {
+			return "", err
+		}
+		return string(b), nil
+	}
+	return "", fmt.Errorf("invalid value given")
+}
+
+func (r *pythonTestResolverMock) MultipleFileContentsByRef(_ ...file.Reference) (map[file.Reference]string, error) {
+	return nil, fmt.Errorf("not implemented")
+}
+
+func (r *pythonTestResolverMock) FilesByPath(_ ...file.Path) ([]file.Reference, error) {
+	return nil, fmt.Errorf("not implemented")
+}
+
+func (r *pythonTestResolverMock) FilesByGlob(_ ...string) ([]file.Reference, error) {
+	return nil, fmt.Errorf("not implemented")
+}
+func (r *pythonTestResolverMock) RelativeFileByPath(_ file.Reference, path string) (*file.Reference, error) {
+	switch {
+	case strings.Contains(path, "RECORD"):
+		return r.recordRef, nil
+	case strings.Contains(path, "top_level.txt"):
+		return r.topLevelRef, nil
+	default:
+		return nil, fmt.Errorf("invalid RelativeFileByPath value given: %q", path)
+	}
+}
+
+func TestPythonPackageWheelCataloger(t *testing.T) {
+	tests := []struct {
+		MetadataFixture string
+		RecordFixture   string
+		TopLevelFixture string
+		ExpectedPackage pkg.Package
+	}{
+		{
+			MetadataFixture: "test-fixtures/egg-info/PKG-INFO",
+			RecordFixture:   "test-fixtures/egg-info/RECORD",
+			TopLevelFixture: "test-fixtures/egg-info/top_level.txt",
+			ExpectedPackage: pkg.Package{
+				Name:         "requests",
+				Version:      "2.22.0",
+				Type:         pkg.PythonPkg,
+				Language:     pkg.Python,
+				Licenses:     []string{"Apache 2.0"},
+				FoundBy:      "python-package-cataloger",
+				MetadataType: pkg.PythonPackageMetadataType,
+				Metadata: pkg.PythonPackageMetadata{
+					Name:                 "requests",
+					Version:              "2.22.0",
+					License:              "Apache 2.0",
+					Platform:             "UNKNOWN",
+					Author:               "Kenneth Reitz",
+					AuthorEmail:          "me@kennethreitz.org",
+					SitePackagesRootPath: "test-fixtures",
+					Files: []pkg.PythonFileRecord{
+						{Path: "requests-2.22.0.dist-info/INSTALLER", Digest: &pkg.Digest{"sha256", "zuuue4knoyJ-UwPPXg8fezS7VCrXJQrAP7zeNuwvFQg"}, Size: "4"},
+						{Path: "requests/__init__.py", Digest: &pkg.Digest{"sha256", "PnKCgjcTq44LaAMzB-7--B2FdewRrE8F_vjZeaG9NhA"}, Size: "3921"},
+						{Path: "requests/__pycache__/__version__.cpython-38.pyc"},
+						{Path: "requests/__pycache__/utils.cpython-38.pyc"},
+						{Path: "requests/__version__.py", Digest: &pkg.Digest{"sha256", "Bm-GFstQaFezsFlnmEMrJDe8JNROz9n2XXYtODdvjjc"}, Size: "436"},
+						{Path: "requests/utils.py", Digest: &pkg.Digest{"sha256", "LtPJ1db6mJff2TJSJWKi7rBpzjPS3mSOrjC9zRhoD3A"}, Size: "30049"},
+					},
+					TopLevelPackages: []string{"requests"},
+				},
+			},
+		},
+		{
+			MetadataFixture: "test-fixtures/dist-info/METADATA",
+			RecordFixture:   "test-fixtures/dist-info/RECORD",
+			TopLevelFixture: "test-fixtures/dist-info/top_level.txt",
+			ExpectedPackage: pkg.Package{
+				Name:         "Pygments",
+				Version:      "2.6.1",
+				Type:         pkg.PythonPkg,
+				Language:     pkg.Python,
+				Licenses:     []string{"BSD License"},
+				FoundBy:      "python-package-cataloger",
+				MetadataType: pkg.PythonPackageMetadataType,
+				Metadata: pkg.PythonPackageMetadata{
+					Name:                 "Pygments",
+					Version:              "2.6.1",
+					License:              "BSD License",
+					Platform:             "any",
+					Author:               "Georg Brandl",
+					AuthorEmail:          "georg@python.org",
+					SitePackagesRootPath: "test-fixtures",
+					Files: []pkg.PythonFileRecord{
+						{Path: "../../../bin/pygmentize", Digest: &pkg.Digest{"sha256", "dDhv_U2jiCpmFQwIRHpFRLAHUO4R1jIJPEvT_QYTFp8"}, Size: "220"},
+						{Path: "Pygments-2.6.1.dist-info/AUTHORS", Digest: &pkg.Digest{"sha256", "PVpa2_Oku6BGuiUvutvuPnWGpzxqFy2I8-NIrqCvqUY"}, Size: "8449"},
+						{Path: "Pygments-2.6.1.dist-info/RECORD"},
+						{Path: "pygments/__pycache__/__init__.cpython-38.pyc"},
+						{Path: "pygments/util.py", Digest: &pkg.Digest{"sha256", "586xXHiJGGZxqk5PMBu3vBhE68DLuAe5MBARWrSPGxA"}, Size: "10778"},
+					},
+					TopLevelPackages: []string{"pygments", "something_else"},
+				},
+			},
+		},
+		{
+			// in cases where the metadata file is available and the record is not we should still record there is a package
+			// additionally empty top_level.txt files should not result in an error
+			MetadataFixture: "test-fixtures/partial.dist-info/METADATA",
+			TopLevelFixture: "test-fixtures/partial.dist-info/top_level.txt",
+			ExpectedPackage: pkg.Package{
+				Name:         "Pygments",
+				Version:      "2.6.1",
+				Type:         pkg.PythonPkg,
+				Language:     pkg.Python,
+				Licenses:     []string{"BSD License"},
+				FoundBy:      "python-package-cataloger",
+				MetadataType: pkg.PythonPackageMetadataType,
+				Metadata: pkg.PythonPackageMetadata{
+					Name:                 "Pygments",
+					Version:              "2.6.1",
+					License:              "BSD License",
+					Platform:             "any",
+					Author:               "Georg Brandl",
+					AuthorEmail:          "georg@python.org",
+					SitePackagesRootPath: "test-fixtures",
+				},
+			},
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.MetadataFixture, func(t *testing.T) {
+			resolver := newTestResolver(test.MetadataFixture, test.RecordFixture, test.TopLevelFixture)
+
+			// note that the source is the record ref created by the resolver mock... attach the expected values
+			test.ExpectedPackage.Source = []file.Reference{*resolver.metadataRef}
+			if resolver.recordRef != nil {
+				test.ExpectedPackage.Source = append(test.ExpectedPackage.Source, *resolver.recordRef)
+			}
+
+			if resolver.topLevelRef != nil {
+				test.ExpectedPackage.Source = append(test.ExpectedPackage.Source, *resolver.topLevelRef)
+			}
+			// end patching expected values with runtime data...
+
+			pyPkgCataloger := NewPythonPackageCataloger()
+
+			actual, err := pyPkgCataloger.catalogEggOrWheel(resolver, *resolver.metadataRef)
+			if err != nil {
+				t.Fatalf("failed to catalog python package: %+v", err)
+			}
+
+			for _, d := range deep.Equal(actual, &test.ExpectedPackage) {
+				t.Errorf("diff: %+v", d)
+			}
+		})
+	}
+
+}

syft/cataloger/python/parse_poetry_lock_test.go

@@ -1,10 +1,11 @@
 package python
 
 import (
-	"github.com/anchore/syft/syft/pkg"
-	"github.com/go-test/deep"
 	"os"
 	"testing"
+
+	"github.com/anchore/syft/syft/pkg"
+	"github.com/go-test/deep"
 )
 
 func TestParsePoetryLock(t *testing.T) {
@@ -13,28 +14,28 @@ func TestParsePoetryLock(t *testing.T) {
 			Name:     "added-value",
 			Version:  "0.14.2",
 			Language: pkg.Python,
-			Type:     pkg.PoetryPkg,
+			Type:     pkg.PythonPkg,
 			Licenses: nil,
 		},
 		{
 			Name:     "alabaster",
 			Version:  "0.7.12",
 			Language: pkg.Python,
-			Type:     pkg.PoetryPkg,
+			Type:     pkg.PythonPkg,
 			Licenses: nil,
 		},
 		{
 			Name:     "appnope",
 			Version:  "0.1.0",
 			Language: pkg.Python,
-			Type:     pkg.PoetryPkg,
+			Type:     pkg.PythonPkg,
 			Licenses: nil,
 		},
 		{
 			Name:     "asciitree",
 			Version:  "0.3.3",
 			Language: pkg.Python,
-			Type:     pkg.PoetryPkg,
+			Type:     pkg.PythonPkg,
 			Licenses: nil,
 		},
 	}

syft/cataloger/python/parse_requirements.go

@@ -47,7 +47,7 @@ func parseRequirementsTxt(_ string, reader io.Reader) ([]pkg.Package, error) {
 				Name:     name,
 				Version:  version,
 				Language: pkg.Python,
-				Type:     pkg.PythonRequirementsPkg,
+				Type:     pkg.PythonPkg,
 			})
 		default:
 			continue

syft/cataloger/python/parse_requirements_test.go

@@ -4,24 +4,45 @@ import (
 	"os"
 	"testing"
 
+	"github.com/go-test/deep"
+
 	"github.com/anchore/syft/syft/pkg"
 )
 
+func assertPackagesEqual(t *testing.T, actual []pkg.Package, expected map[string]pkg.Package) {
+	t.Helper()
+	if len(actual) != len(expected) {
+		for _, a := range actual {
+			t.Log("   ", a)
+		}
+		t.Fatalf("unexpected package count: %d!=%d", len(actual), len(expected))
+	}
+
+	for _, a := range actual {
+		expectedPkg, ok := expected[a.Name]
+		if !ok {
+			t.Errorf("unexpected package found: '%s'", a.Name)
+		}
+
+		for _, d := range deep.Equal(a, expectedPkg) {
+			t.Errorf("diff: %+v", d)
+		}
+	}
+}
+
 func TestParseRequirementsTxt(t *testing.T) {
 	expected := map[string]pkg.Package{
 		"foo": {
 			Name:     "foo",
 			Version:  "1.0.0",
 			Language: pkg.Python,
-			Type:     pkg.PythonRequirementsPkg,
-			Licenses: []string{},
+			Type:     pkg.PythonPkg,
 		},
 		"flask": {
 			Name:     "flask",
 			Version:  "4.0.0",
 			Language: pkg.Python,
-			Type:     pkg.PythonRequirementsPkg,
-			Licenses: []string{},
+			Type:     pkg.PythonPkg,
 		},
 	}
 	fixture, err := os.Open("test-fixtures/requires/requirements.txt")
@@ -34,6 +55,6 @@ func TestParseRequirementsTxt(t *testing.T) {
 		t.Fatalf("failed to parse requirements: %+v", err)
 	}
 
-	assertPkgsEqual(t, actual, expected)
+	assertPackagesEqual(t, actual, expected)
 
 }

syft/cataloger/python/parse_setup.go

@@ -41,7 +41,7 @@ func parseSetup(_ string, reader io.Reader) ([]pkg.Package, error) {
 				Name:     strings.Trim(name, "'\""),
 				Version:  strings.Trim(version, "'\""),
 				Language: pkg.Python,
-				Type:     pkg.PythonSetupPkg,
+				Type:     pkg.PythonPkg,
 			})
 		}
 	}

syft/cataloger/python/parse_setup_test.go

@@ -13,36 +13,31 @@ func TestParseSetup(t *testing.T) {
 			Name:     "pathlib3",
 			Version:  "2.2.0",
 			Language: pkg.Python,
-			Type:     pkg.PythonSetupPkg,
-			Licenses: []string{},
+			Type:     pkg.PythonPkg,
 		},
 		"mypy": {
 			Name:     "mypy",
 			Version:  "v0.770",
 			Language: pkg.Python,
-			Type:     pkg.PythonSetupPkg,
-			Licenses: []string{},
+			Type:     pkg.PythonPkg,
 		},
 		"mypy1": {
 			Name:     "mypy1",
 			Version:  "v0.770",
 			Language: pkg.Python,
-			Type:     pkg.PythonSetupPkg,
-			Licenses: []string{},
+			Type:     pkg.PythonPkg,
 		},
 		"mypy2": {
 			Name:     "mypy2",
 			Version:  "v0.770",
 			Language: pkg.Python,
-			Type:     pkg.PythonSetupPkg,
-			Licenses: []string{},
+			Type:     pkg.PythonPkg,
 		},
 		"mypy3": {
 			Name:     "mypy3",
 			Version:  "v0.770",
 			Language: pkg.Python,
-			Type:     pkg.PythonSetupPkg,
-			Licenses: []string{},
+			Type:     pkg.PythonPkg,
 		},
 	}
 	fixture, err := os.Open("test-fixtures/setup/setup.py")
@@ -55,6 +50,6 @@ func TestParseSetup(t *testing.T) {
 		t.Fatalf("failed to parse requirements: %+v", err)
 	}
 
-	assertPkgsEqual(t, actual, expected)
+	assertPackagesEqual(t, actual, expected)
 
 }

syft/cataloger/python/parse_wheel_egg.go

@@ -1,100 +0,0 @@
-package python
-
-import (
-	"bufio"
-	"fmt"
-	"io"
-	"strings"
-
-	"github.com/anchore/syft/syft/cataloger/common"
-	"github.com/anchore/syft/syft/pkg"
-)
-
-// integrity check
-var _ common.ParserFn = parseWheelMetadata
-var _ common.ParserFn = parseEggMetadata
-
-// parseWheelMetadata is a parser function for individual Python Wheel metadata file contents, returning all Python
-// packages listed.
-func parseWheelMetadata(_ string, reader io.Reader) ([]pkg.Package, error) {
-	packages, err := parseWheelOrEggMetadata(reader)
-	for idx := range packages {
-		packages[idx].Type = pkg.WheelPkg
-	}
-	return packages, err
-}
-
-// parseEggMetadata is a parser function for individual Python Egg metadata file contents, returning all Python
-// packages listed.
-func parseEggMetadata(_ string, reader io.Reader) ([]pkg.Package, error) {
-	packages, err := parseWheelOrEggMetadata(reader)
-	for idx := range packages {
-		packages[idx].Type = pkg.EggPkg
-	}
-	return packages, err
-}
-
-// parseWheelOrEggMetadata takes a Python Egg or Wheel (which share the same format and values for our purposes),
-// returning all Python packages listed.
-func parseWheelOrEggMetadata(reader io.Reader) ([]pkg.Package, error) {
-	fields := make(map[string]string)
-	var key string
-
-	scanner := bufio.NewScanner(reader)
-	for scanner.Scan() {
-		line := scanner.Text()
-
-		line = strings.TrimRight(line, "\n")
-
-		// empty line indicates end of entry
-		if len(line) == 0 {
-			// if the entry has not started, keep parsing lines
-			if len(fields) == 0 {
-				continue
-			}
-			break
-		}
-
-		switch {
-		case strings.HasPrefix(line, " "):
-			// a field-body continuation
-			if len(key) == 0 {
-				return nil, fmt.Errorf("no match for continuation: line: '%s'", line)
-			}
-
-			val, ok := fields[key]
-			if !ok {
-				return nil, fmt.Errorf("no previous key exists, expecting: %s", key)
-			}
-			// concatenate onto previous value
-			val = fmt.Sprintf("%s\n %s", val, strings.TrimSpace(line))
-			fields[key] = val
-		default:
-			// parse a new key (note, duplicate keys are overridden)
-			if i := strings.Index(line, ":"); i > 0 {
-				key = strings.TrimSpace(line[0:i])
-				val := strings.TrimSpace(line[i+1:])
-
-				fields[key] = val
-			} else {
-				return nil, fmt.Errorf("cannot parse field from line: '%s'", line)
-			}
-		}
-	}
-
-	if err := scanner.Err(); err != nil {
-		return nil, fmt.Errorf("failed to parse python wheel/egg: %w", err)
-	}
-
-	p := pkg.Package{
-		Name:     fields["Name"],
-		Version:  fields["Version"],
-		Language: pkg.Python,
-	}
-
-	if license, ok := fields["License"]; ok && license != "" {
-		p.Licenses = []string{license}
-	}
-
-	return []pkg.Package{p}, nil
-}

syft/cataloger/python/parse_wheel_egg_metadata.go

@@ -0,0 +1,80 @@
+package python
+
+import (
+	"bufio"
+	"fmt"
+	"io"
+	"path/filepath"
+	"strings"
+
+	"github.com/anchore/stereoscope/pkg/file"
+
+	"github.com/mitchellh/mapstructure"
+
+	"github.com/anchore/syft/syft/pkg"
+)
+
+// parseWheelOrEggMetadata takes a Python Egg or Wheel (which share the same format and values for our purposes),
+// returning all Python packages listed.
+func parseWheelOrEggMetadata(path file.Path, reader io.Reader) (pkg.PythonPackageMetadata, error) {
+	fields := make(map[string]string)
+	var key string
+
+	scanner := bufio.NewScanner(reader)
+	for scanner.Scan() {
+		line := scanner.Text()
+		line = strings.TrimRight(line, "\n")
+
+		// empty line indicates end of entry
+		if len(line) == 0 {
+			// if the entry has not started, keep parsing lines
+			if len(fields) == 0 {
+				continue
+			}
+			break
+		}
+
+		switch {
+		case strings.HasPrefix(line, " "):
+			// a field-body continuation
+			if len(key) == 0 {
+				return pkg.PythonPackageMetadata{}, fmt.Errorf("no match for continuation: line: '%s'", line)
+			}
+
+			val, ok := fields[key]
+			if !ok {
+				return pkg.PythonPackageMetadata{}, fmt.Errorf("no previous key exists, expecting: %s", key)
+			}
+			// concatenate onto previous value
+			val = fmt.Sprintf("%s\n %s", val, strings.TrimSpace(line))
+			fields[key] = val
+		default:
+			// parse a new key (note, duplicate keys are overridden)
+			if i := strings.Index(line, ":"); i > 0 {
+				// mapstruct cannot map keys with dashes, and we are expected to persist the "Author-email" field
+				key = strings.ReplaceAll(strings.TrimSpace(line[0:i]), "-", "")
+				val := strings.TrimSpace(line[i+1:])
+
+				fields[key] = val
+			} else {
+				return pkg.PythonPackageMetadata{}, fmt.Errorf("cannot parse field from line: '%s'", line)
+			}
+		}
+	}
+
+	if err := scanner.Err(); err != nil {
+		return pkg.PythonPackageMetadata{}, fmt.Errorf("failed to parse python wheel/egg: %w", err)
+	}
+
+	var metadata pkg.PythonPackageMetadata
+	if err := mapstructure.Decode(fields, &metadata); err != nil {
+		return pkg.PythonPackageMetadata{}, fmt.Errorf("unable to parse APK metadata: %w", err)
+	}
+
+	// add additional metadata not stored in the egg/wheel metadata file
+
+	sitePackagesRoot := filepath.Clean(filepath.Join(filepath.Dir(string(path)), ".."))
+	metadata.SitePackagesRootPath = sitePackagesRoot
+
+	return metadata, nil
+}

syft/cataloger/python/parse_wheel_egg_metadata_test.go

@@ -0,0 +1,62 @@
+package python
+
+import (
+	"os"
+	"testing"
+
+	"github.com/anchore/stereoscope/pkg/file"
+
+	"github.com/anchore/syft/syft/pkg"
+	"github.com/go-test/deep"
+)
+
+func TestParseWheelEggMetadata(t *testing.T) {
+	tests := []struct {
+		Fixture          string
+		ExpectedMetadata pkg.PythonPackageMetadata
+	}{
+		{
+			Fixture: "test-fixtures/egg-info/PKG-INFO",
+			ExpectedMetadata: pkg.PythonPackageMetadata{
+				Name:                 "requests",
+				Version:              "2.22.0",
+				License:              "Apache 2.0",
+				Platform:             "UNKNOWN",
+				Author:               "Kenneth Reitz",
+				AuthorEmail:          "me@kennethreitz.org",
+				SitePackagesRootPath: "test-fixtures",
+			},
+		},
+		{
+			Fixture: "test-fixtures/dist-info/METADATA",
+			ExpectedMetadata: pkg.PythonPackageMetadata{
+				Name:                 "Pygments",
+				Version:              "2.6.1",
+				License:              "BSD License",
+				Platform:             "any",
+				Author:               "Georg Brandl",
+				AuthorEmail:          "georg@python.org",
+				SitePackagesRootPath: "test-fixtures",
+			},
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.Fixture, func(t *testing.T) {
+			fixture, err := os.Open(test.Fixture)
+			if err != nil {
+				t.Fatalf("failed to open fixture: %+v", err)
+			}
+
+			actual, err := parseWheelOrEggMetadata(file.Path(test.Fixture), fixture)
+			if err != nil {
+				t.Fatalf("failed to parse: %+v", err)
+			}
+
+			for _, d := range deep.Equal(actual, test.ExpectedMetadata) {
+				t.Errorf("diff: %+v", d)
+			}
+		})
+	}
+
+}

syft/cataloger/python/parse_wheel_egg_record.go

@@ -0,0 +1,60 @@
+package python
+
+import (
+	"encoding/csv"
+	"fmt"
+	"io"
+	"strings"
+
+	"github.com/anchore/syft/syft/pkg"
+)
+
+// parseWheelOrEggMetadata takes a Python Egg or Wheel (which share the same format and values for our purposes),
+// returning all Python packages listed.
+func parseWheelOrEggRecord(reader io.Reader) ([]pkg.PythonFileRecord, error) {
+	var records []pkg.PythonFileRecord
+	r := csv.NewReader(reader)
+
+	for {
+		recordList, err := r.Read()
+		if err == io.EOF {
+			break
+		}
+		if err != nil {
+			return nil, fmt.Errorf("unable to read python record file: %w", err)
+		}
+
+		if len(recordList) != 3 {
+			return nil, fmt.Errorf("python record an unexpected length=%d: %q", len(recordList), recordList)
+		}
+
+		var record pkg.PythonFileRecord
+
+		for idx, item := range recordList {
+			switch idx {
+			case 0:
+				record.Path = item
+			case 1:
+				if item == "" {
+					continue
+				}
+				fields := strings.Split(item, "=")
+
+				if len(fields) != 2 {
+					return nil, fmt.Errorf("unexpected python record digest: %q", item)
+				}
+
+				record.Digest = &pkg.Digest{
+					Algorithm: fields[0],
+					Value:     fields[1],
+				}
+			case 2:
+				record.Size = item
+			}
+		}
+
+		records = append(records, record)
+	}
+
+	return records, nil
+}

syft/cataloger/python/parse_wheel_egg_record_test.go

@@ -0,0 +1,57 @@
+package python
+
+import (
+	"os"
+	"testing"
+
+	"github.com/anchore/syft/syft/pkg"
+	"github.com/go-test/deep"
+)
+
+func TestParseWheelEggRecord(t *testing.T) {
+	tests := []struct {
+		Fixture          string
+		ExpectedMetadata []pkg.PythonFileRecord
+	}{
+		{
+			Fixture: "test-fixtures/egg-info/RECORD",
+			ExpectedMetadata: []pkg.PythonFileRecord{
+				{Path: "requests-2.22.0.dist-info/INSTALLER", Digest: &pkg.Digest{"sha256", "zuuue4knoyJ-UwPPXg8fezS7VCrXJQrAP7zeNuwvFQg"}, Size: "4"},
+				{Path: "requests/__init__.py", Digest: &pkg.Digest{"sha256", "PnKCgjcTq44LaAMzB-7--B2FdewRrE8F_vjZeaG9NhA"}, Size: "3921"},
+				{Path: "requests/__pycache__/__version__.cpython-38.pyc"},
+				{Path: "requests/__pycache__/utils.cpython-38.pyc"},
+				{Path: "requests/__version__.py", Digest: &pkg.Digest{"sha256", "Bm-GFstQaFezsFlnmEMrJDe8JNROz9n2XXYtODdvjjc"}, Size: "436"},
+				{Path: "requests/utils.py", Digest: &pkg.Digest{"sha256", "LtPJ1db6mJff2TJSJWKi7rBpzjPS3mSOrjC9zRhoD3A"}, Size: "30049"},
+			},
+		},
+		{
+			Fixture: "test-fixtures/dist-info/RECORD",
+			ExpectedMetadata: []pkg.PythonFileRecord{
+				{Path: "../../../bin/pygmentize", Digest: &pkg.Digest{"sha256", "dDhv_U2jiCpmFQwIRHpFRLAHUO4R1jIJPEvT_QYTFp8"}, Size: "220"},
+				{Path: "Pygments-2.6.1.dist-info/AUTHORS", Digest: &pkg.Digest{"sha256", "PVpa2_Oku6BGuiUvutvuPnWGpzxqFy2I8-NIrqCvqUY"}, Size: "8449"},
+				{Path: "Pygments-2.6.1.dist-info/RECORD"},
+				{Path: "pygments/__pycache__/__init__.cpython-38.pyc"},
+				{Path: "pygments/util.py", Digest: &pkg.Digest{"sha256", "586xXHiJGGZxqk5PMBu3vBhE68DLuAe5MBARWrSPGxA"}, Size: "10778"},
+			},
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.Fixture, func(t *testing.T) {
+			fixture, err := os.Open(test.Fixture)
+			if err != nil {
+				t.Fatalf("failed to open fixture: %+v", err)
+			}
+
+			actual, err := parseWheelOrEggRecord(fixture)
+			if err != nil {
+				t.Fatalf("failed to parse: %+v", err)
+			}
+
+			for _, d := range deep.Equal(actual, test.ExpectedMetadata) {
+				t.Errorf("diff: %+v", d)
+			}
+		})
+	}
+
+}

syft/cataloger/python/parse_wheel_egg_test.go

@@ -1,93 +0,0 @@
-package python
-
-import (
-	"os"
-	"testing"
-
-	"github.com/anchore/syft/syft/pkg"
-)
-
-func assertPkgsEqual(t *testing.T, actual []pkg.Package, expected map[string]pkg.Package) {
-	t.Helper()
-	if len(actual) != len(expected) {
-		for _, a := range actual {
-			t.Log("   ", a)
-		}
-		t.Fatalf("unexpected package count: %d!=%d", len(actual), len(expected))
-	}
-
-	for _, a := range actual {
-		expectedPkg, ok := expected[a.Name]
-		if !ok {
-			t.Errorf("unexpected package found: '%s'", a.Name)
-		}
-
-		if expectedPkg.Version != a.Version {
-			t.Errorf("unexpected package version: '%s'", a.Version)
-		}
-
-		if a.Language != expectedPkg.Language {
-			t.Errorf("bad language: '%+v'", a.Language)
-		}
-
-		if a.Type != expectedPkg.Type {
-			t.Errorf("bad package type: %+v", a.Type)
-		}
-
-		if len(a.Licenses) < len(expectedPkg.Licenses) {
-			t.Errorf("bad package licenses count: '%+v'", a.Licenses)
-		}
-		if len(a.Licenses) > 0 {
-			if a.Licenses[0] != expectedPkg.Licenses[0] {
-				t.Errorf("bad package licenses: '%+v'", a.Licenses)
-			}
-		}
-
-	}
-}
-
-func TestParseEggMetadata(t *testing.T) {
-	expected := map[string]pkg.Package{
-		"requests": {
-			Name:     "requests",
-			Version:  "2.22.0",
-			Language: pkg.Python,
-			Type:     pkg.EggPkg,
-			Licenses: []string{"Apache 2.0"},
-		},
-	}
-	fixture, err := os.Open("test-fixtures/egg-info/PKG-INFO")
-	if err != nil {
-		t.Fatalf("failed to open fixture: %+v", err)
-	}
-
-	actual, err := parseEggMetadata(fixture.Name(), fixture)
-	if err != nil {
-		t.Fatalf("failed to parse egg-info: %+v", err)
-	}
-
-	assertPkgsEqual(t, actual, expected)
-}
-
-func TestParseWheelMetadata(t *testing.T) {
-	expected := map[string]pkg.Package{
-		"Pygments": {
-			Name:     "Pygments",
-			Version:  "2.6.1",
-			Language: pkg.Python,
-			Type:     pkg.WheelPkg,
-			Licenses: []string{"BSD License"},
-		},
-	}
-	fixture, err := os.Open("test-fixtures/dist-info/METADATA")
-	if err != nil {
-		t.Fatalf("failed to open fixture: %+v", err)
-	}
-
-	actual, err := parseWheelMetadata(fixture.Name(), fixture)
-	if err != nil {
-		t.Fatalf("failed to parse dist-info: %+v", err)
-	}
-
-	assertPkgsEqual(t, actual, expected)
-}

syft/cataloger/python/poetry_metadata_package.go

@@ -16,6 +16,6 @@ func (p PoetryMetadataPackage) Pkg() pkg.Package {
 		Name:     p.Name,
 		Version:  p.Version,
 		Language: pkg.Python,
-		Type:     pkg.PoetryPkg,
+		Type:     pkg.PythonPkg,
 	}
 }

syft/cataloger/python/test-fixtures/dist-info/RECORD

@@ -0,0 +1,5 @@
+../../../bin/pygmentize,sha256=dDhv_U2jiCpmFQwIRHpFRLAHUO4R1jIJPEvT_QYTFp8,220
+Pygments-2.6.1.dist-info/AUTHORS,sha256=PVpa2_Oku6BGuiUvutvuPnWGpzxqFy2I8-NIrqCvqUY,8449
+Pygments-2.6.1.dist-info/RECORD,,
+pygments/__pycache__/__init__.cpython-38.pyc,,
+pygments/util.py,sha256=586xXHiJGGZxqk5PMBu3vBhE68DLuAe5MBARWrSPGxA,10778

syft/cataloger/python/test-fixtures/dist-info/top_level.txt

@@ -0,0 +1,2 @@
+pygments
+something_else

syft/cataloger/python/test-fixtures/egg-info/RECORD

@@ -0,0 +1,6 @@
+requests-2.22.0.dist-info/INSTALLER,sha256=zuuue4knoyJ-UwPPXg8fezS7VCrXJQrAP7zeNuwvFQg,4
+requests/__init__.py,sha256=PnKCgjcTq44LaAMzB-7--B2FdewRrE8F_vjZeaG9NhA,3921
+requests/__pycache__/__version__.cpython-38.pyc,,
+requests/__pycache__/utils.cpython-38.pyc,,
+requests/__version__.py,sha256=Bm-GFstQaFezsFlnmEMrJDe8JNROz9n2XXYtODdvjjc,436
+requests/utils.py,sha256=LtPJ1db6mJff2TJSJWKi7rBpzjPS3mSOrjC9zRhoD3A,30049

syft/cataloger/python/test-fixtures/egg-info/top_level.txt

@@ -0,0 +1 @@
+requests

syft/cataloger/python/test-fixtures/partial.dist-info/METADATA

@@ -0,0 +1,47 @@
+Metadata-Version: 2.1
+Name: Pygments
+Version: 2.6.1
+Summary: Pygments is a syntax highlighting package written in Python.
+Home-page: https://pygments.org/
+Author: Georg Brandl
+Author-email: georg@python.org
+License: BSD License
+Keywords: syntax highlighting
+Platform: any
+Classifier: License :: OSI Approved :: BSD License
+Classifier: Intended Audience :: Developers
+Classifier: Intended Audience :: End Users/Desktop
+Classifier: Intended Audience :: System Administrators
+Classifier: Development Status :: 6 - Mature
+Classifier: Programming Language :: Python
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.5
+Classifier: Programming Language :: Python :: 3.6
+Classifier: Programming Language :: Python :: 3.7
+Classifier: Programming Language :: Python :: 3.8
+Classifier: Programming Language :: Python :: Implementation :: CPython
+Classifier: Programming Language :: Python :: Implementation :: PyPy
+Classifier: Operating System :: OS Independent
+Classifier: Topic :: Text Processing :: Filters
+Classifier: Topic :: Utilities
+Requires-Python: >=3.5
+
+
+Pygments
+~~~~~~~~
+
+Pygments is a syntax highlighting package written in Python.
+
+It is a generic syntax highlighter suitable for use in code hosting, forums,
+wikis or other applications that need to prettify source code.  Highlights
+are:
+
+* a wide range of over 500 languages and other text formats is supported
+* special attention is paid to details, increasing quality by a fair amount
+* support for new languages and formats are added easily
+* a number of output formats, presently HTML, LaTeX, RTF, SVG, all image     formats that PIL supports and ANSI sequences
+* it is usable as a command-line tool and as a library
+
+:copyright: Copyright 2006-2019 by the Pygments team, see AUTHORS.
+:license: BSD, see LICENSE for details.
+

syft/cataloger/rpmdb/parse_rpmdb.go

@@ -52,8 +52,9 @@ func parseRpmDB(_ string, reader io.Reader) ([]pkg.Package, error) {
 			Name:    entry.Name,
 			Version: fmt.Sprintf("%s-%s", entry.Version, entry.Release), // this is what engine does
 			//Version: fmt.Sprintf("%d:%s-%s.%s", entry.Epoch, entry.Version, entry.Release, entry.Arch),
-			Type: pkg.RpmPkg,
-			Metadata: pkg.RpmMetadata{
+			Type:         pkg.RpmPkg,
+			MetadataType: pkg.RpmdbMetadataType,
+			Metadata: pkg.RpmdbMetadata{
 				Name:      entry.Name,
 				Version:   entry.Version,
 				Epoch:     entry.Epoch,

syft/cataloger/rpmdb/parse_rpmdb_test.go

@@ -1,19 +1,21 @@
 package rpmdb
 
 import (
-	"github.com/anchore/syft/syft/pkg"
-	"github.com/go-test/deep"
 	"os"
 	"testing"
+
+	"github.com/anchore/syft/syft/pkg"
+	"github.com/go-test/deep"
 )
 
 func TestParseRpmDB(t *testing.T) {
 	expected := map[string]pkg.Package{
 		"dive": {
-			Name:    "dive",
-			Version: "0.9.2-1",
-			Type:    pkg.RpmPkg,
-			Metadata: pkg.RpmMetadata{
+			Name:         "dive",
+			Version:      "0.9.2-1",
+			Type:         pkg.RpmPkg,
+			MetadataType: pkg.RpmdbMetadataType,
+			Metadata: pkg.RpmdbMetadata{
 				Name:      "dive",
 				Epoch:     0,
 				Arch:      "x86_64",

syft/cataloger/ruby/parse_gemspec.go

@@ -96,12 +96,13 @@ func parseGemSpecEntries(_ string, reader io.Reader) ([]pkg.Package, error) {
 		}
 
 		pkgs = append(pkgs, pkg.Package{
-			Name:     metadata.Name,
-			Version:  metadata.Version,
-			Licenses: metadata.Licenses,
-			Language: pkg.Ruby,
-			Type:     pkg.GemPkg,
-			Metadata: metadata,
+			Name:         metadata.Name,
+			Version:      metadata.Version,
+			Licenses:     metadata.Licenses,
+			Language:     pkg.Ruby,
+			Type:         pkg.GemPkg,
+			MetadataType: pkg.GemMetadataType,
+			Metadata:     metadata,
 		})
 	}
 

syft/cataloger/ruby/parse_gemspec_test.go

@@ -10,11 +10,12 @@ import (
 
 func TestParseGemspec(t *testing.T) {
 	var expectedPkg = pkg.Package{
-		Name:     "bundler",
-		Version:  "2.1.4",
-		Type:     pkg.GemPkg,
-		Licenses: []string{"MIT"},
-		Language: pkg.Ruby,
+		Name:         "bundler",
+		Version:      "2.1.4",
+		Type:         pkg.GemPkg,
+		Licenses:     []string{"MIT"},
+		Language:     pkg.Ruby,
+		MetadataType: pkg.GemMetadataType,
 		Metadata: pkg.GemMetadata{
 			Name:     "bundler",
 			Version:  "2.1.4",

syft/pkg/metadata.go

@@ -0,0 +1,14 @@
+package pkg
+
+type MetadataType string
+
+const (
+	UnknownMetadataType        MetadataType = "UnknownMetadata"
+	ApkMetadataType            MetadataType = "apk-metadata"
+	DpkgMetadataType           MetadataType = "dpkg-metadata"
+	GemMetadataType            MetadataType = "gem-metadata"
+	JavaMetadataType           MetadataType = "java-metadata"
+	NpmPackageJSONMetadataType MetadataType = "npm-package-json-metadata"
+	RpmdbMetadataType          MetadataType = "rpmdb-metadata"
+	PythonPackageMetadataType  MetadataType = "python-package-metadata"
+)

syft/pkg/npm_metadata.go

@@ -0,0 +1,12 @@
+package pkg
+
+// NpmPackageJSONMetadata holds extra information that is used in pkg.Package
+type NpmPackageJSONMetadata struct {
+	Name        string   `mapstructure:"name" json:"name"`
+	Version     string   `mapstructure:"version" json:"version"`
+	Files       []string `mapstructure:"files" json:"files"`
+	Author      string   `mapstructure:"author" json:"author"`
+	License     string   `mapstructure:"license" json:"license"`
+	Homepage    string   `mapstructure:"homepage" json:"homepage"`
+	Description string   `mapstructure:"description" json:"description"`
+}

syft/pkg/package.go

@@ -23,10 +23,11 @@ type Package struct {
 	FoundBy string           `json:"foundBy"`  // the specific cataloger that discovered this package
 	Source  []file.Reference `json:"sources"`  // the locations that lead to the discovery of this package (note: this is not necessarily the locations that make up this package)
 	// TODO: should we move licenses into metadata?
-	Licenses []string    `json:"licenses"`           // licenses discovered with the package metadata
-	Language Language    `json:"language"`           // the language ecosystem this package belongs to (e.g. JavaScript, Python, etc)
-	Type     Type        `json:"type"`               // the package type (e.g. Npm, Yarn, Egg, Wheel, Rpm, Deb, etc)
-	Metadata interface{} `json:"metadata,omitempty"` // additional data found while parsing the package source
+	Licenses     []string     `json:"licenses"`           // licenses discovered with the package metadata
+	Language     Language     `json:"language"`           // the language ecosystem this package belongs to (e.g. JavaScript, Python, etc)
+	Type         Type         `json:"type"`               // the package type (e.g. Npm, Yarn, Python, Rpm, Deb, etc)
+	MetadataType MetadataType `json:"metadataType"`       // the shape of the additional data in the "metadata" field
+	Metadata     interface{}  `json:"metadata,omitempty"` // additional data found while parsing the package source
 }
 
 // ID returns the package ID, which is unique relative to a package catalog.

syft/pkg/package_test.go

@@ -25,7 +25,7 @@ func TestPackage_pURL(t *testing.T) {
 			pkg: Package{
 				Name:    "name",
 				Version: "v0.1.0",
-				Type:    WheelPkg,
+				Type:    PythonPkg,
 			},
 			expected: "pkg:pypi/name@v0.1.0",
 		},
@@ -33,7 +33,7 @@ func TestPackage_pURL(t *testing.T) {
 			pkg: Package{
 				Name:    "name",
 				Version: "v0.1.0",
-				Type:    EggPkg,
+				Type:    PythonPkg,
 			},
 			expected: "pkg:pypi/name@v0.1.0",
 		},
@@ -41,7 +41,7 @@ func TestPackage_pURL(t *testing.T) {
 			pkg: Package{
 				Name:    "name",
 				Version: "v0.1.0",
-				Type:    PythonSetupPkg,
+				Type:    PythonPkg,
 			},
 			expected: "pkg:pypi/name@v0.1.0",
 		},
@@ -49,7 +49,7 @@ func TestPackage_pURL(t *testing.T) {
 			pkg: Package{
 				Name:    "name",
 				Version: "v0.1.0",
-				Type:    PythonRequirementsPkg,
+				Type:    PythonPkg,
 			},
 			expected: "pkg:pypi/name@v0.1.0",
 		},
@@ -69,14 +69,6 @@ func TestPackage_pURL(t *testing.T) {
 			},
 			expected: "pkg:npm/name@v0.1.0",
 		},
-		{
-			pkg: Package{
-				Name:    "name",
-				Version: "v0.1.0",
-				Type:    YarnPkg,
-			},
-			expected: "pkg:npm/name@v0.1.0",
-		},
 		{
 			distro: distro.Distro{
 				Type: distro.Ubuntu,
@@ -101,7 +93,7 @@ func TestPackage_pURL(t *testing.T) {
 				Name:    "bad-name",
 				Version: "bad-v0.1.0",
 				Type:    RpmPkg,
-				Metadata: RpmMetadata{
+				Metadata: RpmdbMetadata{
 					Name:    "name",
 					Version: "v0.1.0",
 					Epoch:   2,

syft/pkg/python_package_metadata.go

@@ -0,0 +1,26 @@
+package pkg
+
+type Digest struct {
+	Algorithm string `json:"algorithm"`
+	Value     string `json:"value"`
+}
+
+// PythonFileRecord represents a single entry within a RECORD file for a python wheel or egg package
+type PythonFileRecord struct {
+	Path   string  `json:"path"`
+	Digest *Digest `json:"digest,omitempty"`
+	Size   string  `json:"size,omitempty"`
+}
+
+// PythonPackageMetadata represents all captured data for a python egg or wheel package.
+type PythonPackageMetadata struct {
+	Name                 string             `json:"name" mapstruct:"Name"`
+	Version              string             `json:"version" mapstruct:"Version"`
+	License              string             `json:"license" mapstruct:"License"`
+	Author               string             `json:"author" mapstruct:"Author"`
+	AuthorEmail          string             `json:"authorEmail" mapstruct:"Authoremail"`
+	Platform             string             `json:"platform" mapstruct:"Platform"`
+	Files                []PythonFileRecord `json:"files,omitempty"`
+	SitePackagesRootPath string             `json:"sitePackagesRootPath"`
+	TopLevelPackages     []string           `json:"topLevelPackages,omitempty"`
+}

syft/pkg/rpmdb_metadata.go

@@ -7,8 +7,8 @@ import (
 	"github.com/package-url/packageurl-go"
 )
 
-// RpmMetadata represents all captured data for a RPM DB package entry.
-type RpmMetadata struct {
+// RpmdbMetadata represents all captured data for a RPM DB package entry.
+type RpmdbMetadata struct {
 	Name      string `json:"name"`
 	Version   string `json:"version"`
 	Epoch     int    `json:"epoch"`
@@ -20,7 +20,7 @@ type RpmMetadata struct {
 	Vendor    string `json:"vendor"`
 }
 
-func (m RpmMetadata) PackageURL(d distro.Distro) string {
+func (m RpmdbMetadata) PackageURL(d distro.Distro) string {
 	pURL := packageurl.NewPackageURL(
 		packageurl.TypeRPM,
 		d.Type.String(),

syft/pkg/rpmdb_metadata_test.go

@@ -1,22 +1,23 @@
 package pkg
 
 import (
+	"testing"
+
 	"github.com/anchore/syft/syft/distro"
 	"github.com/sergi/go-diff/diffmatchpatch"
-	"testing"
 )
 
 func TestRpmMetadata_pURL(t *testing.T) {
 	tests := []struct {
 		distro   distro.Distro
-		metadata RpmMetadata
+		metadata RpmdbMetadata
 		expected string
 	}{
 		{
 			distro: distro.Distro{
 				Type: distro.CentOS,
 			},
-			metadata: RpmMetadata{
+			metadata: RpmdbMetadata{
 				Name:    "p",
 				Version: "v",
 				Arch:    "a",
@@ -29,7 +30,7 @@ func TestRpmMetadata_pURL(t *testing.T) {
 			distro: distro.Distro{
 				Type: distro.RedHat,
 			},
-			metadata: RpmMetadata{
+			metadata: RpmdbMetadata{
 				Name:    "p",
 				Version: "v",
 				Arch:    "a",

syft/pkg/type.go

@@ -6,36 +6,25 @@ import "github.com/package-url/packageurl-go"
 type Type string
 
 const (
-	UnknownPkg Type = "UnknownPackage"
-	ApkPkg     Type = "apk"
-	GemPkg     Type = "gem"
-	DebPkg     Type = "deb"
-	EggPkg     Type = "egg"
-	// PacmanPkg Type = "pacman"
-	RpmPkg                Type = "rpm"
-	WheelPkg              Type = "wheel"
-	PoetryPkg             Type = "poetry"
-	NpmPkg                Type = "npm"
-	YarnPkg               Type = "yarn"
-	PythonRequirementsPkg Type = "python-requirements"
-	PythonSetupPkg        Type = "python-setup"
-	JavaPkg               Type = "java-archive"
-	JenkinsPluginPkg      Type = "jenkins-plugin"
-	GoModulePkg           Type = "go-module"
+	UnknownPkg       Type = "UnknownPackage"
+	ApkPkg           Type = "apk"
+	GemPkg           Type = "gem"
+	DebPkg           Type = "deb"
+	RpmPkg           Type = "rpm"
+	NpmPkg           Type = "npm"
+	PythonPkg        Type = "python"
+	JavaPkg          Type = "java-archive"
+	JenkinsPluginPkg Type = "jenkins-plugin"
+	GoModulePkg      Type = "go-module"
 )
 
 var AllPkgs = []Type{
 	ApkPkg,
 	GemPkg,
 	DebPkg,
-	EggPkg,
-	// PacmanPkg,
 	RpmPkg,
-	WheelPkg,
 	NpmPkg,
-	YarnPkg,
-	PythonRequirementsPkg,
-	PythonSetupPkg,
+	PythonPkg,
 	JavaPkg,
 	JenkinsPluginPkg,
 	GoModulePkg,
@@ -49,9 +38,9 @@ func (t Type) PackageURLType() string {
 		return packageurl.TypeGem
 	case DebPkg:
 		return "deb"
-	case EggPkg, WheelPkg, PythonRequirementsPkg, PythonSetupPkg:
+	case PythonPkg:
 		return packageurl.TypePyPi
-	case NpmPkg, YarnPkg:
+	case NpmPkg:
 		return packageurl.TypeNPM
 	case JavaPkg, JenkinsPluginPkg:
 		return packageurl.TypeMaven

syft/presenter/cyclonedx/presenter_test.go

@@ -3,11 +3,12 @@ package cyclonedx
 import (
 	"bytes"
 	"flag"
-	"github.com/anchore/stereoscope/pkg/imagetest"
-	"github.com/anchore/syft/syft/distro"
 	"regexp"
 	"testing"
 
+	"github.com/anchore/stereoscope/pkg/imagetest"
+	"github.com/anchore/syft/syft/distro"
+
 	"github.com/anchore/go-testutils"
 	"github.com/anchore/stereoscope/pkg/file"
 	"github.com/anchore/syft/syft/pkg"
@@ -109,7 +110,7 @@ func TestCycloneDxImgsPresenter(t *testing.T) {
 		},
 		Type:    pkg.RpmPkg,
 		FoundBy: "the-cataloger-1",
-		Metadata: pkg.RpmMetadata{
+		Metadata: pkg.RpmdbMetadata{
 			Name:      "package1",
 			Epoch:     0,
 			Arch:      "x86_64",
@@ -133,7 +134,7 @@ func TestCycloneDxImgsPresenter(t *testing.T) {
 			"MIT",
 			"Apache-v2",
 		},
-		Metadata: pkg.RpmMetadata{
+		Metadata: pkg.RpmdbMetadata{
 			Name:      "package2",
 			Epoch:     0,
 			Arch:      "x86_64",

syft/scope/resolver.go

@@ -16,13 +16,20 @@ type Resolver interface {
 
 // ContentResolver knows how to get file content for given file.References
 type ContentResolver interface {
+	FileContentsByRef(ref file.Reference) (string, error)
 	MultipleFileContentsByRef(f ...file.Reference) (map[file.Reference]string, error)
+	// TODO: we should consider refactoring to return a set of io.Readers or file.Openers instead of the full contents themselves (allow for optional buffering).
 }
 
 //  FileResolver knows how to get file.References for given string paths and globs
 type FileResolver interface {
+	// FilesByPath fetches a set of file references which have the given path (for an image, there may be multiple matches)
 	FilesByPath(paths ...file.Path) ([]file.Reference, error)
+	// FilesByGlob fetches a set of file references which the given glob matches
 	FilesByGlob(patterns ...string) ([]file.Reference, error)
+	// RelativeFileByPath fetches a single file at the given path relative to the layer squash of the given reference.
+	// This is helpful when attempting to find a file that is in the same layer or lower as another file.
+	RelativeFileByPath(reference file.Reference, path string) (*file.Reference, error)
 }
 
 // getImageResolver returns the appropriate resolve for a container image given the scope option

syft/scope/resolvers/all_layers_resolver.go

@@ -109,8 +109,23 @@ func (r *AllLayersResolver) FilesByGlob(patterns ...string) ([]file.Reference, e
 	return uniqueFiles, nil
 }
 
+func (r *AllLayersResolver) RelativeFileByPath(reference file.Reference, path string) (*file.Reference, error) {
+	entry, err := r.img.FileCatalog.Get(reference)
+	if err != nil {
+		return nil, err
+	}
+
+	return entry.Source.SquashedTree.File(file.Path(path)), nil
+}
+
 // MultipleFileContentsByRef returns the file contents for all file.References relative to the image. Note that a
 // file.Reference is a path relative to a particular layer.
 func (r *AllLayersResolver) MultipleFileContentsByRef(f ...file.Reference) (map[file.Reference]string, error) {
 	return r.img.MultipleFileContentsByRef(f...)
 }
+
+// FileContentsByRef fetches file contents for a single file reference, irregardless of the source layer.
+// If the path does not exist an error is returned.
+func (r *AllLayersResolver) FileContentsByRef(ref file.Reference) (string, error) {
+	return r.img.FileContentsByRef(ref)
+}

syft/scope/resolvers/directory_resolver.go

@@ -5,6 +5,7 @@ import (
 	"io/ioutil"
 	"os"
 	"path"
+	"path/filepath"
 
 	"github.com/anchore/stereoscope/pkg/file"
 	"github.com/anchore/syft/internal/log"
@@ -18,7 +19,7 @@ type DirectoryResolver struct {
 
 // Stringer to represent a directory path data source
 func (s DirectoryResolver) String() string {
-	return fmt.Sprintf("dir://%s", s.Path)
+	return fmt.Sprintf("dir:%s", s.Path)
 }
 
 // FilesByPath returns all file.References that match the given paths from the directory.
@@ -26,15 +27,19 @@ func (s DirectoryResolver) FilesByPath(userPaths ...file.Path) ([]file.Reference
 	var references = make([]file.Reference, 0)
 
 	for _, userPath := range userPaths {
-		resolvedPath := path.Join(s.Path, string(userPath))
-		_, err := os.Stat(resolvedPath)
+		userStrPath := string(userPath)
+
+		if filepath.IsAbs(userStrPath) {
+			// a path relative to root should be prefixed with the resolvers directory path, otherwise it should be left as is
+			userStrPath = path.Join(s.Path, userStrPath)
+		}
+		_, err := os.Stat(userStrPath)
 		if os.IsNotExist(err) {
 			continue
 		} else if err != nil {
-			log.Errorf("path (%s) is not valid: %v", resolvedPath, err)
+			log.Errorf("path (%s) is not valid: %v", userStrPath, err)
 		}
-		filePath := file.Path(resolvedPath)
-		references = append(references, file.NewFileReference(filePath))
+		references = append(references, file.NewFileReference(file.Path(userStrPath)))
 	}
 
 	return references, nil
@@ -75,6 +80,18 @@ func (s DirectoryResolver) FilesByGlob(patterns ...string) ([]file.Reference, er
 	return result, nil
 }
 
+func (s *DirectoryResolver) RelativeFileByPath(_ file.Reference, path string) (*file.Reference, error) {
+	paths, err := s.FilesByPath(file.Path(path))
+	if err != nil {
+		return nil, err
+	}
+	if len(paths) == 0 {
+		return nil, nil
+	}
+
+	return &paths[0], nil
+}
+
 // MultipleFileContentsByRef returns the file contents for all file.References relative a directory.
 func (s DirectoryResolver) MultipleFileContentsByRef(f ...file.Reference) (map[file.Reference]string, error) {
 	refContents := make(map[file.Reference]string)
@@ -82,9 +99,20 @@ func (s DirectoryResolver) MultipleFileContentsByRef(f ...file.Reference) (map[f
 		contents, err := fileContents(fileRef.Path)
 
 		if err != nil {
-			return refContents, fmt.Errorf("could not read contents of file: %s", fileRef.Path)
+			return nil, fmt.Errorf("could not read contents of file: %s", fileRef.Path)
 		}
 		refContents[fileRef] = string(contents)
 	}
 	return refContents, nil
 }
+
+// FileContentsByRef fetches file contents for a single file reference relative to a directory.
+// If the path does not exist an error is returned.
+func (s DirectoryResolver) FileContentsByRef(reference file.Reference) (string, error) {
+	contents, err := fileContents(reference.Path)
+	if err != nil {
+		return "", fmt.Errorf("could not read contents of file: %s", reference.Path)
+	}
+
+	return string(contents), nil
+}

syft/scope/resolvers/directory_resolver_test.go

@@ -1,7 +1,6 @@
 package resolvers
 
 import (
-	"path"
 	"testing"
 
 	"github.com/anchore/stereoscope/pkg/file"
@@ -10,24 +9,49 @@ import (
 func TestDirectoryResolver_FilesByPath(t *testing.T) {
 	cases := []struct {
 		name     string
+		root     string
 		input    string
+		expected string
 		refCount int
 	}{
 		{
-			name:     "finds a file",
-			input:    "image-symlinks/file-1.txt",
+			name:     "finds a file (relative)",
+			root:     "./test-fixtures/",
+			input:    "test-fixtures/image-symlinks/file-1.txt",
+			expected: "test-fixtures/image-symlinks/file-1.txt",
 			refCount: 1,
 		},
 		{
-			name:     "managed non-existing files",
-			input:    "image-symlinks/bogus.txt",
+			name:     "finds a file with relative indirection",
+			root:     "./test-fixtures/../test-fixtures",
+			input:    "test-fixtures/image-symlinks/file-1.txt",
+			expected: "test-fixtures/image-symlinks/file-1.txt",
+			refCount: 1,
+		},
+		{
+			// note: this is asserting the old behavior is not supported
+			name:     "relative lookup with wrong path fails",
+			root:     "./test-fixtures/",
+			input:    "image-symlinks/file-1.txt",
 			refCount: 0,
 		},
+		{
+			name:     "managed non-existing files (relative)",
+			root:     "./test-fixtures/",
+			input:    "test-fixtures/image-symlinks/bogus.txt",
+			refCount: 0,
+		},
+		{
+			name:     "finds a file (absolute)",
+			root:     "./test-fixtures/",
+			input:    "/image-symlinks/file-1.txt",
+			expected: "test-fixtures/image-symlinks/file-1.txt",
+			refCount: 1,
+		},
 	}
 	for _, c := range cases {
 		t.Run(c.name, func(t *testing.T) {
-			resolver := DirectoryResolver{"test-fixtures"}
-			expected := path.Join("test-fixtures", c.input)
+			resolver := DirectoryResolver{c.root}
 			refs, err := resolver.FilesByPath(file.Path(c.input))
 			if err != nil {
 				t.Fatalf("could not use resolver: %+v, %+v", err, refs)
@@ -38,8 +62,8 @@ func TestDirectoryResolver_FilesByPath(t *testing.T) {
 			}
 
 			for _, actual := range refs {
-				if actual.Path != file.Path(expected) {
-					t.Errorf("bad resolve path: '%s'!='%s'", actual.Path, c.input)
+				if actual.Path != file.Path(c.expected) {
+					t.Errorf("bad resolve path: '%s'!='%s'", actual.Path, c.expected)
 				}
 			}
 		})
@@ -54,17 +78,17 @@ func TestDirectoryResolver_MultipleFilesByPath(t *testing.T) {
 	}{
 		{
 			name:     "finds multiple files",
-			input:    []file.Path{file.Path("image-symlinks/file-1.txt"), file.Path("image-symlinks/file-2.txt")},
+			input:    []file.Path{file.Path("test-fixtures/image-symlinks/file-1.txt"), file.Path("test-fixtures/image-symlinks/file-2.txt")},
 			refCount: 2,
 		},
 		{
 			name:     "skips non-existing files",
-			input:    []file.Path{file.Path("image-symlinks/bogus.txt"), file.Path("image-symlinks/file-1.txt")},
+			input:    []file.Path{file.Path("test-fixtures/image-symlinks/bogus.txt"), file.Path("test-fixtures/image-symlinks/file-1.txt")},
 			refCount: 1,
 		},
 		{
 			name:     "does not return anything for non-existing directories",
-			input:    []file.Path{file.Path("non-existing/bogus.txt"), file.Path("non-existing/file-1.txt")},
+			input:    []file.Path{file.Path("test-fixtures/non-existing/bogus.txt"), file.Path("test-fixtures/non-existing/file-1.txt")},
 			refCount: 0,
 		},
 	}
@@ -93,17 +117,17 @@ func TestDirectoryResolver_MultipleFileContentsByRef(t *testing.T) {
 	}{
 		{
 			name:     "gets multiple file contents",
-			input:    []file.Path{file.Path("image-symlinks/file-1.txt"), file.Path("image-symlinks/file-2.txt")},
+			input:    []file.Path{file.Path("test-fixtures/image-symlinks/file-1.txt"), file.Path("test-fixtures/image-symlinks/file-2.txt")},
 			refCount: 2,
 		},
 		{
 			name:     "skips non-existing files",
-			input:    []file.Path{file.Path("image-symlinks/bogus.txt"), file.Path("image-symlinks/file-1.txt")},
+			input:    []file.Path{file.Path("test-fixtures/image-symlinks/bogus.txt"), file.Path("test-fixtures/image-symlinks/file-1.txt")},
 			refCount: 1,
 		},
 		{
 			name:     "does not return anything for non-existing directories",
-			input:    []file.Path{file.Path("non-existing/bogus.txt"), file.Path("non-existing/file-1.txt")},
+			input:    []file.Path{file.Path("test-fixtures/non-existing/bogus.txt"), file.Path("test-fixtures/non-existing/file-1.txt")},
 			refCount: 0,
 		},
 	}

syft/scope/resolvers/image_squash_resolver.go

@@ -73,8 +73,26 @@ func (r *ImageSquashResolver) FilesByGlob(patterns ...string) ([]file.Reference,
 	return uniqueFiles, nil
 }
 
+func (r *ImageSquashResolver) RelativeFileByPath(reference file.Reference, path string) (*file.Reference, error) {
+	paths, err := r.FilesByPath(file.Path(path))
+	if err != nil {
+		return nil, err
+	}
+	if len(paths) == 0 {
+		return nil, nil
+	}
+
+	return &paths[0], nil
+}
+
 // MultipleFileContentsByRef returns the file contents for all file.References relative to the image. Note that a
 // file.Reference is a path relative to a particular layer, in this case only from the squashed representation.
 func (r *ImageSquashResolver) MultipleFileContentsByRef(f ...file.Reference) (map[file.Reference]string, error) {
 	return r.img.MultipleFileContentsByRef(f...)
 }
+
+// FileContentsByRef fetches file contents for a single file reference, irregardless of the source layer.
+// If the path does not exist an error is returned.
+func (r *ImageSquashResolver) FileContentsByRef(ref file.Reference) (string, error) {
+	return r.img.FileContentsByRef(ref)
+}

syft/scope/scope_test.go

@@ -61,13 +61,13 @@ func TestDirectoryScope(t *testing.T) {
 		{
 			desc:       "path detected",
 			input:      "test-fixtures",
-			inputPaths: []file.Path{file.Path("path-detected")},
+			inputPaths: []file.Path{file.Path("test-fixtures/path-detected")},
 			expRefs:    1,
 		},
 		{
 			desc:       "no files-by-path detected",
 			input:      "test-fixtures",
-			inputPaths: []file.Path{file.Path("no-path-detected")},
+			inputPaths: []file.Path{file.Path("test-fixtures/no-path-detected")},
 			expRefs:    0,
 		},
 	}
@@ -105,13 +105,13 @@ func TestMultipleFileContentsByRefContents(t *testing.T) {
 		{
 			input:    "test-fixtures/path-detected",
 			desc:     "empty file",
-			path:     "empty",
+			path:     "test-fixtures/path-detected/empty",
 			expected: "",
 		},
 		{
 			input:    "test-fixtures/path-detected",
 			desc:     "file has contents",
-			path:     ".vimrc",
+			path:     "test-fixtures/path-detected/.vimrc",
 			expected: "\" A .vimrc file\n",
 		},
 	}
@@ -127,7 +127,7 @@ func TestMultipleFileContentsByRefContents(t *testing.T) {
 			}
 
 			if len(refs) != 1 {
-				t.Errorf("expected a single ref to be generated but got: %d", len(refs))
+				t.Fatalf("expected a single ref to be generated but got: %d", len(refs))
 			}
 			ref := refs[0]
 

test/integration/distro_test.go

@@ -1,5 +1,3 @@
-// +build integration
-
 package integration
 
 import (

test/integration/json_schema_test.go

@@ -1,5 +1,3 @@
-// +build integration
-
 package integration
 
 import (

test/integration/pkg_cases_test.go

@@ -1,5 +1,3 @@
-// +build integration
-
 package integration
 
 import "github.com/anchore/syft/syft/pkg"
@@ -20,6 +18,25 @@ var imageOnlyTestCases = []testCase{
 			"bundler": "2.1.4",
 		},
 	},
+	{
+		name:        "find npm package",
+		pkgType:     pkg.NpmPkg,
+		pkgLanguage: pkg.JavaScript,
+		pkgInfo: map[string]string{
+			"npm": "6.14.6",
+		},
+	},
+	{
+		name:        "find python egg & wheel packages",
+		pkgType:     pkg.PythonPkg,
+		pkgLanguage: pkg.Python,
+		pkgInfo: map[string]string{
+			"Pygments":     "2.6.1",
+			"requests":     "2.22.0",
+			"somerequests": "3.22.0",
+			"someotherpkg": "3.19.0",
+		},
+	},
 }
 
 var dirOnlyTestCases = []testCase{
@@ -81,6 +98,35 @@ var dirOnlyTestCases = []testCase{
 			"unicorn":              "4.8.3",
 		},
 	},
+	{
+		name:        "find javascript npm packages (yarn.lock & package-lock.json)",
+		pkgType:     pkg.NpmPkg,
+		pkgLanguage: pkg.JavaScript,
+		pkgInfo: map[string]string{
+			"@babel/code-frame": "7.10.4",
+			"get-stdin":         "8.0.0",
+		},
+	},
+	{
+		name:        "find python requirements.txt & setup.py package references",
+		pkgType:     pkg.PythonPkg,
+		pkgLanguage: pkg.Python,
+		pkgInfo: map[string]string{
+			// dir specific test cases
+			"flask":              "4.0.0",
+			"python-dateutil":    "2.8.1",
+			"python-swiftclient": "3.8.1",
+			"pytz":               "2019.3",
+			"jsonschema":         "2.6.0",
+			"passlib":            "1.7.2",
+			"mypy":               "v0.770",
+			// common to image and directory
+			"Pygments":     "2.6.1",
+			"requests":     "2.22.0",
+			"somerequests": "3.22.0",
+			"someotherpkg": "3.19.0",
+		},
+	},
 }
 
 var commonTestCases = []testCase{
@@ -116,62 +162,6 @@ var commonTestCases = []testCase{
 			"example-jenkins-plugin": "1.0-SNAPSHOT",
 		},
 	},
-	{
-		name:        "find python wheel packages",
-		pkgType:     pkg.WheelPkg,
-		pkgLanguage: pkg.Python,
-		pkgInfo: map[string]string{
-			"Pygments": "2.6.1",
-			"requests": "2.10.0",
-		},
-	},
-	{
-		name:        "find javascript npm packages",
-		pkgType:     pkg.NpmPkg,
-		pkgLanguage: pkg.JavaScript,
-		pkgInfo: map[string]string{
-			"get-stdin": "8.0.0",
-		},
-	},
-	{
-		name:        "find javascript yarn packages",
-		pkgType:     pkg.YarnPkg,
-		pkgLanguage: pkg.JavaScript,
-		pkgInfo: map[string]string{
-			"@babel/code-frame": "7.10.4",
-		},
-	},
-	{
-		name:        "find python egg packages",
-		pkgType:     pkg.EggPkg,
-		pkgLanguage: pkg.Python,
-		pkgInfo: map[string]string{
-			"requests": "2.22.0",
-			"otherpkg": "2.19.0",
-		},
-	},
-	{
-		name:        "find python requirements.txt packages",
-		pkgType:     pkg.PythonRequirementsPkg,
-		pkgLanguage: pkg.Python,
-		pkgInfo: map[string]string{
-			"flask":              "4.0.0",
-			"python-dateutil":    "2.8.1",
-			"python-swiftclient": "3.8.1",
-			"pytz":               "2019.3",
-			"jsonschema":         "2.6.0",
-			"passlib":            "1.7.2",
-			"pathlib":            "1.0.1",
-		},
-	},
-	{
-		name:        "find python setup.py packages",
-		pkgType:     pkg.PythonSetupPkg,
-		pkgLanguage: pkg.Python,
-		pkgInfo: map[string]string{
-			"mypy": "v0.770",
-		},
-	},
 	{
 
 		name:    "find apkdb packages",

test/integration/pkg_coverage_test.go

@@ -1,5 +1,3 @@
-// +build integration
-
 package integration
 
 import (
@@ -70,6 +68,7 @@ func TestPkgCoverageImage(t *testing.T) {
 			}
 
 			if pkgCount != len(c.pkgInfo) {
+				t.Logf("Discovered packages of type %+v", c.pkgType)
 				for a := range catalog.Enumerate(c.pkgType) {
 					t.Log("   ", a)
 				}
@@ -123,39 +122,39 @@ func TestPkgCoverageDirectory(t *testing.T) {
 	cases = append(cases, commonTestCases...)
 	cases = append(cases, dirOnlyTestCases...)
 
-	for _, c := range cases {
-		t.Run(c.name, func(t *testing.T) {
-			pkgCount := 0
+	for _, test := range cases {
+		t.Run(test.name, func(t *testing.T) {
+			actualPkgCount := 0
 
-			for a := range catalog.Enumerate(c.pkgType) {
+			for actualPkg := range catalog.Enumerate(test.pkgType) {
 
-				observedLanguages.Add(a.Language.String())
-				observedPkgs.Add(string(a.Type))
+				observedLanguages.Add(actualPkg.Language.String())
+				observedPkgs.Add(string(actualPkg.Type))
 
-				expectedVersion, ok := c.pkgInfo[a.Name]
+				expectedVersion, ok := test.pkgInfo[actualPkg.Name]
 				if !ok {
-					t.Errorf("unexpected package found: %s", a.Name)
+					t.Errorf("unexpected package found: %s", actualPkg.Name)
 				}
 
-				if expectedVersion != a.Version {
-					t.Errorf("unexpected package version (pkg=%s): %s", a.Name, a.Version)
+				if expectedVersion != actualPkg.Version {
+					t.Errorf("unexpected package version (pkg=%s): %s", actualPkg.Name, actualPkg.Version)
 				}
 
-				if a.Language != c.pkgLanguage {
-					t.Errorf("bad language (pkg=%+v): %+v", a.Name, a.Language)
+				if actualPkg.Language != test.pkgLanguage {
+					t.Errorf("bad language (pkg=%+v): %+v", actualPkg.Name, actualPkg.Language)
 				}
 
-				if a.Type != c.pkgType {
-					t.Errorf("bad package type (pkg=%+v): %+v", a.Name, a.Type)
+				if actualPkg.Type != test.pkgType {
+					t.Errorf("bad package type (pkg=%+v): %+v", actualPkg.Name, actualPkg.Type)
 				}
-				pkgCount++
+				actualPkgCount++
 			}
 
-			if pkgCount != len(c.pkgInfo) {
-				for a := range catalog.Enumerate(c.pkgType) {
-					t.Log("   ", a)
+			if actualPkgCount != len(test.pkgInfo) {
+				for actualPkg := range catalog.Enumerate(test.pkgType) {
+					t.Log("   ", actualPkg)
 				}
-				t.Fatalf("unexpected package count: %d!=%d", pkgCount, len(c.pkgInfo))
+				t.Fatalf("unexpected package count: %d!=%d", actualPkgCount, len(test.pkgInfo))
 			}
 
 		})

test/integration/test-fixtures/image-pkg-coverage/javascript/package-json/package.json

@@ -0,0 +1,314 @@
+{
+  "version": "6.14.6",
+  "name": "npm",
+  "description": "a package manager for JavaScript",
+  "keywords": [
+    "install",
+    "modules",
+    "package manager",
+    "package.json"
+  ],
+  "preferGlobal": true,
+  "config": {
+    "publishtest": false
+  },
+  "homepage": "https://docs.npmjs.com/",
+  "author": "Isaac Z. Schlueter <i@izs.me> (http://blog.izs.me)",
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/npm/cli"
+  },
+  "bugs": {
+    "url": "https://npm.community/c/bugs"
+  },
+  "directories": {
+    "bin": "./bin",
+    "doc": "./doc",
+    "lib": "./lib",
+    "man": "./man"
+  },
+  "main": "./lib/npm.js",
+  "bin": {
+    "npm": "./bin/npm-cli.js",
+    "npx": "./bin/npx-cli.js"
+  },
+  "dependencies": {
+    "JSONStream": "^1.3.5",
+    "abbrev": "~1.1.1",
+    "ansicolors": "~0.3.2",
+    "ansistyles": "~0.1.3",
+    "aproba": "^2.0.0",
+    "archy": "~1.0.0",
+    "bin-links": "^1.1.7",
+    "bluebird": "^3.5.5",
+    "byte-size": "^5.0.1",
+    "cacache": "^12.0.3",
+    "call-limit": "^1.1.1",
+    "chownr": "^1.1.4",
+    "ci-info": "^2.0.0",
+    "cli-columns": "^3.1.2",
+    "cli-table3": "^0.5.1",
+    "cmd-shim": "^3.0.3",
+    "columnify": "~1.5.4",
+    "config-chain": "^1.1.12",
+    "detect-indent": "~5.0.0",
+    "detect-newline": "^2.1.0",
+    "dezalgo": "~1.0.3",
+    "editor": "~1.0.0",
+    "figgy-pudding": "^3.5.1",
+    "find-npm-prefix": "^1.0.2",
+    "fs-vacuum": "~1.2.10",
+    "fs-write-stream-atomic": "~1.0.10",
+    "gentle-fs": "^2.3.0",
+    "glob": "^7.1.6",
+    "graceful-fs": "^4.2.4",
+    "has-unicode": "~2.0.1",
+    "hosted-git-info": "^2.8.8",
+    "iferr": "^1.0.2",
+    "infer-owner": "^1.0.4",
+    "inflight": "~1.0.6",
+    "inherits": "^2.0.4",
+    "ini": "^1.3.5",
+    "init-package-json": "^1.10.3",
+    "is-cidr": "^3.0.0",
+    "json-parse-better-errors": "^1.0.2",
+    "lazy-property": "~1.0.0",
+    "libcipm": "^4.0.7",
+    "libnpm": "^3.0.1",
+    "libnpmaccess": "^3.0.2",
+    "libnpmhook": "^5.0.3",
+    "libnpmorg": "^1.0.1",
+    "libnpmsearch": "^2.0.2",
+    "libnpmteam": "^1.0.2",
+    "libnpx": "^10.2.2",
+    "lock-verify": "^2.1.0",
+    "lockfile": "^1.0.4",
+    "lodash._baseuniq": "~4.6.0",
+    "lodash.clonedeep": "~4.5.0",
+    "lodash.union": "~4.6.0",
+    "lodash.uniq": "~4.5.0",
+    "lodash.without": "~4.4.0",
+    "lru-cache": "^5.1.1",
+    "meant": "~1.0.1",
+    "mississippi": "^3.0.0",
+    "mkdirp": "^0.5.5",
+    "move-concurrently": "^1.0.1",
+    "node-gyp": "^5.1.0",
+    "nopt": "^4.0.3",
+    "normalize-package-data": "^2.5.0",
+    "npm-audit-report": "^1.3.2",
+    "npm-cache-filename": "~1.0.2",
+    "npm-install-checks": "^3.0.2",
+    "npm-lifecycle": "^3.1.4",
+    "npm-package-arg": "^6.1.1",
+    "npm-packlist": "^1.4.8",
+    "npm-pick-manifest": "^3.0.2",
+    "npm-profile": "^4.0.4",
+    "npm-registry-fetch": "^4.0.5",
+    "npm-user-validate": "~1.0.0",
+    "npmlog": "~4.1.2",
+    "once": "~1.4.0",
+    "opener": "^1.5.1",
+    "osenv": "^0.1.5",
+    "pacote": "^9.5.12",
+    "path-is-inside": "~1.0.2",
+    "promise-inflight": "~1.0.1",
+    "qrcode-terminal": "^0.12.0",
+    "query-string": "^6.8.2",
+    "qw": "~1.0.1",
+    "read": "~1.0.7",
+    "read-cmd-shim": "^1.0.5",
+    "read-installed": "~4.0.3",
+    "read-package-json": "^2.1.1",
+    "read-package-tree": "^5.3.1",
+    "readable-stream": "^3.6.0",
+    "readdir-scoped-modules": "^1.1.0",
+    "request": "^2.88.0",
+    "retry": "^0.12.0",
+    "rimraf": "^2.7.1",
+    "safe-buffer": "^5.1.2",
+    "semver": "^5.7.1",
+    "sha": "^3.0.0",
+    "slide": "~1.1.6",
+    "sorted-object": "~2.0.1",
+    "sorted-union-stream": "~2.1.3",
+    "ssri": "^6.0.1",
+    "stringify-package": "^1.0.1",
+    "tar": "^4.4.13",
+    "text-table": "~0.2.0",
+    "tiny-relative-date": "^1.3.0",
+    "uid-number": "0.0.6",
+    "umask": "~1.1.0",
+    "unique-filename": "^1.1.1",
+    "unpipe": "~1.0.0",
+    "update-notifier": "^2.5.0",
+    "uuid": "^3.3.3",
+    "validate-npm-package-license": "^3.0.4",
+    "validate-npm-package-name": "~3.0.0",
+    "which": "^1.3.1",
+    "worker-farm": "^1.7.0",
+    "write-file-atomic": "^2.4.3"
+  },
+  "bundleDependencies": [
+    "abbrev",
+    "ansicolors",
+    "ansistyles",
+    "aproba",
+    "archy",
+    "bin-links",
+    "bluebird",
+    "byte-size",
+    "cacache",
+    "call-limit",
+    "chownr",
+    "ci-info",
+    "cli-columns",
+    "cli-table3",
+    "cmd-shim",
+    "columnify",
+    "config-chain",
+    "debuglog",
+    "detect-indent",
+    "detect-newline",
+    "dezalgo",
+    "editor",
+    "figgy-pudding",
+    "find-npm-prefix",
+    "fs-vacuum",
+    "fs-write-stream-atomic",
+    "gentle-fs",
+    "glob",
+    "graceful-fs",
+    "has-unicode",
+    "hosted-git-info",
+    "iferr",
+    "imurmurhash",
+    "infer-owner",
+    "inflight",
+    "inherits",
+    "ini",
+    "init-package-json",
+    "is-cidr",
+    "json-parse-better-errors",
+    "JSONStream",
+    "lazy-property",
+    "libcipm",
+    "libnpm",
+    "libnpmaccess",
+    "libnpmhook",
+    "libnpmorg",
+    "libnpmsearch",
+    "libnpmteam",
+    "libnpx",
+    "lock-verify",
+    "lockfile",
+    "lodash._baseindexof",
+    "lodash._baseuniq",
+    "lodash._bindcallback",
+    "lodash._cacheindexof",
+    "lodash._createcache",
+    "lodash._getnative",
+    "lodash.clonedeep",
+    "lodash.restparam",
+    "lodash.union",
+    "lodash.uniq",
+    "lodash.without",
+    "lru-cache",
+    "meant",
+    "mississippi",
+    "mkdirp",
+    "move-concurrently",
+    "node-gyp",
+    "nopt",
+    "normalize-package-data",
+    "npm-audit-report",
+    "npm-cache-filename",
+    "npm-install-checks",
+    "npm-lifecycle",
+    "npm-package-arg",
+    "npm-packlist",
+    "npm-pick-manifest",
+    "npm-profile",
+    "npm-registry-fetch",
+    "npm-user-validate",
+    "npmlog",
+    "once",
+    "opener",
+    "osenv",
+    "pacote",
+    "path-is-inside",
+    "promise-inflight",
+    "qrcode-terminal",
+    "query-string",
+    "qw",
+    "read-cmd-shim",
+    "read-installed",
+    "read-package-json",
+    "read-package-tree",
+    "read",
+    "readable-stream",
+    "readdir-scoped-modules",
+    "request",
+    "retry",
+    "rimraf",
+    "safe-buffer",
+    "semver",
+    "sha",
+    "slide",
+    "sorted-object",
+    "sorted-union-stream",
+    "ssri",
+    "stringify-package",
+    "tar",
+    "text-table",
+    "tiny-relative-date",
+    "uid-number",
+    "umask",
+    "unique-filename",
+    "unpipe",
+    "update-notifier",
+    "uuid",
+    "validate-npm-package-license",
+    "validate-npm-package-name",
+    "which",
+    "worker-farm",
+    "write-file-atomic"
+  ],
+  "devDependencies": {
+    "deep-equal": "^1.0.1",
+    "get-stream": "^4.1.0",
+    "licensee": "^7.0.3",
+    "marked": "^0.6.3",
+    "marked-man": "^0.6.0",
+    "npm-registry-couchapp": "^2.7.4",
+    "npm-registry-mock": "^1.3.1",
+    "require-inject": "^1.4.4",
+    "sprintf-js": "^1.1.2",
+    "standard": "^11.0.1",
+    "tacks": "^1.3.0",
+    "tap": "^12.7.0",
+    "tar-stream": "^2.1.0"
+  },
+  "scripts": {
+    "dumpconf": "env | grep npm | sort | uniq",
+    "prepare": "node bin/npm-cli.js rebuild && node bin/npm-cli.js --no-audit --no-timing prune --prefix=. --no-global && rimraf test/*/*/node_modules && make -j4 mandocs",
+    "preversion": "bash scripts/update-authors.sh && git add AUTHORS && git commit -m \"update AUTHORS\" || true",
+    "licenses": "licensee --production --errors-only",
+    "tap": "tap -J --timeout 300 --no-esm",
+    "tap-cover": "tap -J --nyc-arg=--cache --coverage --timeout 600 --no-esm",
+    "lint": "standard",
+    "pretest": "npm run lint",
+    "test": "npm run test-tap --",
+    "test:nocleanup": "NO_TEST_CLEANUP=1 npm run test --",
+    "sudotest": "sudo npm run tap -- \"test/tap/*.js\"",
+    "sudotest:nocleanup": "sudo NO_TEST_CLEANUP=1 npm run tap -- \"test/tap/*.js\"",
+    "posttest": "rimraf test/npm_cache*",
+    "test-coverage": "npm run tap-cover -- \"test/tap/*.js\" \"test/network/*.js\"",
+    "test-tap": "npm run tap -- \"test/tap/*.js\" \"test/network/*.js\"",
+    "test-node": "tap --timeout 240 \"test/tap/*.js\" \"test/network/*.js\""
+  },
+  "license": "Artistic-2.0",
+  "engines": {
+    "node": "6 >=6.2.0 || 8 || >=9.3.0"
+  }
+}

test/integration/test-fixtures/image-pkg-coverage/python/dist-info/RECORD

@@ -0,0 +1,5 @@
+../../../bin/pygmentize,sha256=dDhv_U2jiCpmFQwIRHpFRLAHUO4R1jIJPEvT_QYTFp8,220
+Pygments-2.6.1.dist-info/AUTHORS,sha256=PVpa2_Oku6BGuiUvutvuPnWGpzxqFy2I8-NIrqCvqUY,8449
+Pygments-2.6.1.dist-info/RECORD,,
+pygments/__pycache__/__init__.cpython-38.pyc,,
+pygments/util.py,sha256=586xXHiJGGZxqk5PMBu3vBhE68DLuAe5MBARWrSPGxA,10778

test/integration/test-fixtures/image-pkg-coverage/python/dist-info/top_level.txt

@@ -0,0 +1 @@
+top-level-pkg

test/integration/test-fixtures/image-pkg-coverage/python/egg-info/top_level.txt

@@ -0,0 +1 @@
+top-level-pkg

test/integration/test-fixtures/image-pkg-coverage/python/requires/requirements-dev.txt

@@ -1,3 +1,2 @@
 jsonschema==2.6.0
-passlib==1.7.2
-pathlib==1.0.1
+passlib==1.7.2

test/integration/test-fixtures/image-pkg-coverage/python/someotherpkg-3.19.0-py3.8.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
-Name: otherpkg
-Version: 2.19.0
+Name: someotherpkg
+Version: 3.19.0
 Summary: Python HTTP for Humans.
 Home-page: http://python-requests.org
 Author: Kenneth Reitz

test/integration/test-fixtures/image-pkg-coverage/python/someotherpkg-3.19.0-py3.8.egg-info/top_level.txt

@@ -0,0 +1 @@
+top-level-pkg

test/integration/test-fixtures/image-pkg-coverage/python/somerequests-3.22.0.dist-info/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
-Name: requests
-Version: 2.10.0
+Name: somerequests
+Version: 3.22.0
 Summary: stuff
 Home-page: stuff
 Author: Georg Brandl

test/integration/test-fixtures/image-pkg-coverage/python/somerequests-3.22.0.dist-info/top_level.txt

@@ -0,0 +1 @@
+top-level-pkg