rename analyzers to catalogers; fix generic cataloger path dispatcher

2026-11-04 16:36:40 +01:00 · 2020-06-15 10:52:55 -04:00
parent 812934c964
commit 294e267e47
29 changed files with 252 additions and 248 deletions
@@ -0,0 +1,34 @@
+package bundler
+
+import (
+	"github.com/anchore/imgbom/imgbom/cataloger/common"
+	"github.com/anchore/imgbom/imgbom/pkg"
+	"github.com/anchore/stereoscope/pkg/file"
+	"github.com/anchore/stereoscope/pkg/tree"
+)
+
+type Cataloger struct {
+	cataloger common.GenericCataloger
+}
+
+func NewCataloger() *Cataloger {
+	globParsers := map[string]common.ParserFn{
+		"*/Gemfile.lock": parseGemfileLockEntries,
+	}
+
+	return &Cataloger{
+		cataloger: common.NewGenericCataloger(nil, globParsers),
+	}
+}
+
+func (a *Cataloger) Name() string {
+	return "bundler-cataloger"
+}
+
+func (a *Cataloger) SelectFiles(trees []tree.FileTreeReader) []file.Reference {
+	return a.cataloger.SelectFiles(trees)
+}
+
+func (a *Cataloger) Catalog(contents map[file.Reference]string) ([]pkg.Package, error) {
+	return a.cataloger.Catalog(contents, a.Name())
+}
@@ -0,0 +1,57 @@
+package bundler
+
+import (
+	"bufio"
+	"io"
+	"strings"
+
+	"github.com/anchore/imgbom/imgbom/pkg"
+	"github.com/anchore/imgbom/internal"
+)
+
+var sectionsOfInterest = internal.NewStringSetFromSlice([]string{"GEM"})
+
+func parseGemfileLockEntries(reader io.Reader) ([]pkg.Package, error) {
+	pkgs := make([]pkg.Package, 0)
+	scanner := bufio.NewScanner(reader)
+
+	var currentSection string
+
+	for scanner.Scan() {
+		line := scanner.Text()
+		sanitizedLine := strings.TrimSpace(line)
+
+		if len(line) > 1 && line[0] != ' ' {
+			// start of section
+			currentSection = sanitizedLine
+			continue
+		} else if !sectionsOfInterest.Contains(currentSection) {
+			// skip this line, we're in the wrong section
+			continue
+		}
+
+		if isDependencyLine(line) {
+			candidate := strings.Fields(sanitizedLine)
+			if len(candidate) != 2 {
+				continue
+			}
+			pkgs = append(pkgs, pkg.Package{
+				Name:     candidate[0],
+				Version:  strings.Trim(candidate[1], "()"),
+				Language: pkg.Ruby,
+				Type:     pkg.BundlerPkg,
+			})
+		}
+	}
+	if err := scanner.Err(); err != nil {
+		return nil, err
+	}
+	return pkgs, nil
+}
+
+func isDependencyLine(line string) bool {
+	if len(line) < 5 {
+		return false
+	}
+	return strings.Count(line[:5], " ") == 4
+}
@@ -0,0 +1,100 @@
+package bundler
+
+import (
+	"os"
+	"testing"
+
+	"github.com/anchore/imgbom/imgbom/pkg"
+)
+
+var expected = map[string]string{
+	"actionmailer":         "4.1.1",
+	"actionpack":           "4.1.1",
+	"actionview":           "4.1.1",
+	"activemodel":          "4.1.1",
+	"activerecord":         "4.1.1",
+	"activesupport":        "4.1.1",
+	"arel":                 "5.0.1.20140414130214",
+	"bootstrap-sass":       "3.1.1.1",
+	"builder":              "3.2.2",
+	"coffee-rails":         "4.0.1",
+	"coffee-script":        "2.2.0",
+	"coffee-script-source": "1.7.0",
+	"erubis":               "2.7.0",
+	"execjs":               "2.0.2",
+	"hike":                 "1.2.3",
+	"i18n":                 "0.6.9",
+	"jbuilder":             "2.0.7",
+	"jquery-rails":         "3.1.0",
+	"json":                 "1.8.1",
+	"kgio":                 "2.9.2",
+	"libv8":                "3.16.14.3",
+	"mail":                 "2.5.4",
+	"mime-types":           "1.25.1",
+	"minitest":             "5.3.4",
+	"multi_json":           "1.10.1",
+	"mysql2":               "0.3.16",
+	"polyglot":             "0.3.4",
+	"rack":                 "1.5.2",
+	"rack-test":            "0.6.2",
+	"rails":                "4.1.1",
+	"railties":             "4.1.1",
+	"raindrops":            "0.13.0",
+	"rake":                 "10.3.2",
+	"rdoc":                 "4.1.1",
+	"ref":                  "1.0.5",
+	"sass":                 "3.2.19",
+	"sass-rails":           "4.0.3",
+	"sdoc":                 "0.4.0",
+	"spring":               "1.1.3",
+	"sprockets":            "2.11.0",
+	"sprockets-rails":      "2.1.3",
+	"sqlite3":              "1.3.9",
+	"therubyracer":         "0.12.1",
+	"thor":                 "0.19.1",
+	"thread_safe":          "0.3.3",
+	"tilt":                 "1.4.1",
+	"treetop":              "1.4.15",
+	"turbolinks":           "2.2.2",
+	"tzinfo":               "1.2.0",
+	"uglifier":             "2.5.0",
+	"unicorn":              "4.8.3",
+}
+
+func TestParseGemfileLockEntries(t *testing.T) {
+	fixture, err := os.Open("test-fixtures/Gemfile.lock")
+	if err != nil {
+		t.Fatalf("failed to open fixture: %+v", err)
+	}
+
+	actual, err := parseGemfileLockEntries(fixture)
+	if err != nil {
+		t.Fatalf("failed to parse gemfile lock: %+v", err)
+	}
+
+	if len(actual) != len(expected) {
+		for _, a := range actual {
+			t.Log("   ", a)
+		}
+		t.Fatalf("unexpected package count: %d!=%d", len(actual), len(expected))
+	}
+
+	for _, a := range actual {
+		expectedVersion, ok := expected[a.Name]
+		if !ok {
+			t.Errorf("unexpected package found: %s", a.Name)
+		}
+
+		if expectedVersion != a.Version {
+			t.Errorf("unexpected package version (pkg=%s): %s", a.Name, a.Version)
+		}
+
+		if a.Language != pkg.Ruby {
+			t.Errorf("bad language (pkg=%+v): %+v", a.Name, a.Language)
+		}
+
+		if a.Type != pkg.BundlerPkg {
+			t.Errorf("bad package type (pkg=%+v): %+v", a.Name, a.Type)
+		}
+	}
+}
@@ -0,0 +1,149 @@
+GEM
+  remote: https://rubygems.org/
+  specs:
+    actionmailer (4.1.1)
+      actionpack (= 4.1.1)
+      actionview (= 4.1.1)
+      mail (~> 2.5.4)
+    actionpack (4.1.1)
+      actionview (= 4.1.1)
+      activesupport (= 4.1.1)
+      rack (~> 1.5.2)
+      rack-test (~> 0.6.2)
+    actionview (4.1.1)
+      activesupport (= 4.1.1)
+      builder (~> 3.1)
+      erubis (~> 2.7.0)
+    activemodel (4.1.1)
+      activesupport (= 4.1.1)
+      builder (~> 3.1)
+    activerecord (4.1.1)
+      activemodel (= 4.1.1)
+      activesupport (= 4.1.1)
+      arel (~> 5.0.0)
+    activesupport (4.1.1)
+      i18n (~> 0.6, >= 0.6.9)
+      json (~> 1.7, >= 1.7.7)
+      minitest (~> 5.1)
+      thread_safe (~> 0.1)
+      tzinfo (~> 1.1)
+    arel (5.0.1.20140414130214)
+    bootstrap-sass (3.1.1.1)
+      sass (~> 3.2)
+    builder (3.2.2)
+    coffee-rails (4.0.1)
+      coffee-script (>= 2.2.0)
+      railties (>= 4.0.0, < 5.0)
+    coffee-script (2.2.0)
+      coffee-script-source
+      execjs
+    coffee-script-source (1.7.0)
+    erubis (2.7.0)
+    execjs (2.0.2)
+    hike (1.2.3)
+    i18n (0.6.9)
+    jbuilder (2.0.7)
+      activesupport (>= 3.0.0, < 5)
+      multi_json (~> 1.2)
+    jquery-rails (3.1.0)
+      railties (>= 3.0, < 5.0)
+      thor (>= 0.14, < 2.0)
+    json (1.8.1)
+    kgio (2.9.2)
+    libv8 (3.16.14.3)
+    mail (2.5.4)
+      mime-types (~> 1.16)
+      treetop (~> 1.4.8)
+    mime-types (1.25.1)
+    minitest (5.3.4)
+    multi_json (1.10.1)
+    mysql2 (0.3.16)
+    polyglot (0.3.4)
+    rack (1.5.2)
+    rack-test (0.6.2)
+      rack (>= 1.0)
+    rails (4.1.1)
+      actionmailer (= 4.1.1)
+      actionpack (= 4.1.1)
+      actionview (= 4.1.1)
+      activemodel (= 4.1.1)
+      activerecord (= 4.1.1)
+      activesupport (= 4.1.1)
+      bundler (>= 1.3.0, < 2.0)
+      railties (= 4.1.1)
+      sprockets-rails (~> 2.0)
+    railties (4.1.1)
+      actionpack (= 4.1.1)
+      activesupport (= 4.1.1)
+      rake (>= 0.8.7)
+      thor (>= 0.18.1, < 2.0)
+    raindrops (0.13.0)
+    rake (10.3.2)
+    rdoc (4.1.1)
+      json (~> 1.4)
+    ref (1.0.5)
+    sass (3.2.19)
+    sass-rails (4.0.3)
+      railties (>= 4.0.0, < 5.0)
+      sass (~> 3.2.0)
+      sprockets (~> 2.8, <= 2.11.0)
+      sprockets-rails (~> 2.0)
+    sdoc (0.4.0)
+      json (~> 1.8)
+      rdoc (~> 4.0, < 5.0)
+    spring (1.1.3)
+    sprockets (2.11.0)
+      hike (~> 1.2)
+      multi_json (~> 1.0)
+      rack (~> 1.0)
+      tilt (~> 1.1, != 1.3.0)
+    sprockets-rails (2.1.3)
+      actionpack (>= 3.0)
+      activesupport (>= 3.0)
+      sprockets (~> 2.8)
+    sqlite3 (1.3.9)
+    therubyracer (0.12.1)
+      libv8 (~> 3.16.14.0)
+      ref
+    thor (0.19.1)
+    thread_safe (0.3.3)
+    tilt (1.4.1)
+    treetop (1.4.15)
+      polyglot
+      polyglot (>= 0.3.1)
+    turbolinks (2.2.2)
+      coffee-rails
+    tzinfo (1.2.0)
+      thread_safe (~> 0.1)
+    uglifier (2.5.0)
+      execjs (>= 0.3.0)
+      json (>= 1.8.0)
+    unicorn (4.8.3)
+      kgio (~> 2.6)
+      rack
+      raindrops (~> 0.7)
+
+BAD-SECTION
+    bad-entry (5.5.5)
+      bad-kgio (~> 2.6)
+      bad-rack
+      bad-raindrops (~> 0.7)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  bootstrap-sass
+  coffee-rails (~> 4.0.0)
+  jbuilder (~> 2.0)
+  jquery-rails
+  mysql2 (~> 0.3.16)
+  rails (= 4.1.1)
+  sass-rails (~> 4.0.3)
+  sdoc (~> 0.4.0)
+  spring
+  sqlite3
+  therubyracer
+  turbolinks
+  uglifier (>= 1.3.0)
+  unicorn
@@ -0,0 +1,16 @@
+package cataloger
+
+import (
+	"github.com/anchore/imgbom/imgbom/pkg"
+	"github.com/anchore/stereoscope/pkg/file"
+	"github.com/anchore/stereoscope/pkg/tree"
+)
+
+type Cataloger interface {
+	Name() string
+	// TODO: add ID / Name for catalog for uniquely identifying this cataloger type
+	SelectFiles([]tree.FileTreeReader) []file.Reference
+	// NOTE: one of the errors which is returned is "IterationNeeded", which indicates to the driver to
+	// continue with another Select/Catalog pass
+	Catalog(map[file.Reference]string) ([]pkg.Package, error)
+}
@@ -0,0 +1,101 @@
+package common
+
+import (
+	"strings"
+
+	"github.com/anchore/imgbom/imgbom/pkg"
+	"github.com/anchore/imgbom/internal/log"
+	"github.com/anchore/stereoscope/pkg/file"
+	"github.com/anchore/stereoscope/pkg/tree"
+)
+
+// TODO: put under test...
+
+// GenericCataloger implements the Catalog interface and is responsible for dispatching the proper parser function for
+// a given path or glob pattern. This is intended to be reusable across many package cataloger types.
+type GenericCataloger struct {
+	globParsers   map[string]ParserFn
+	pathParsers   map[string]ParserFn
+	selectedFiles []file.Reference
+	parsers       map[file.Reference]ParserFn
+}
+
+// NewGenericCataloger if provided path-to-parser-function and glob-to-parser-function lookups creates a GenericCataloger
+func NewGenericCataloger(pathParsers map[string]ParserFn, globParsers map[string]ParserFn) GenericCataloger {
+	return GenericCataloger{
+		globParsers:   globParsers,
+		pathParsers:   pathParsers,
+		selectedFiles: make([]file.Reference, 0),
+		parsers:       make(map[file.Reference]ParserFn),
+	}
+}
+
+// register pairs a set of file references with a parser function for future cataloging (when the file contents are resolved)
+func (a *GenericCataloger) register(files []file.Reference, parser ParserFn) {
+	a.selectedFiles = append(a.selectedFiles, files...)
+	for _, f := range files {
+		a.parsers[f] = parser
+	}
+}
+
+// clear deletes all registered file-reference-to-parser-function pairings from former SelectFiles() and register() calls
+func (a *GenericCataloger) clear() {
+	a.selectedFiles = make([]file.Reference, 0)
+	a.parsers = make(map[file.Reference]ParserFn)
+}
+
+// SelectFiles takes a set of file trees and resolves and file references of interest for future cataloging
+func (a *GenericCataloger) SelectFiles(trees []tree.FileTreeReader) []file.Reference {
+	for _, t := range trees {
+		// select by exact path
+		for path, parser := range a.pathParsers {
+			f := t.File(file.Path(path))
+			if f != nil {
+				a.register([]file.Reference{*f}, parser)
+			}
+		}
+
+		// select by pattern
+		for globPattern, parser := range a.globParsers {
+			fileMatches, err := t.FilesByGlob(globPattern)
+			if err != nil {
+				log.Errorf("failed to find files by glob: %s", globPattern)
+			}
+			if fileMatches != nil {
+				a.register(fileMatches, parser)
+			}
+		}
+	}
+
+	return a.selectedFiles
+}
+
+// Catalog takes a set of file contents and uses any configured parser functions to resolve and return discovered packages
+func (a *GenericCataloger) Catalog(contents map[file.Reference]string, upstreamMatcher string) ([]pkg.Package, error) {
+	defer a.clear()
+
+	packages := make([]pkg.Package, 0)
+
+	for reference, parser := range a.parsers {
+		content, ok := contents[reference]
+		if !ok {
+			log.Errorf("cataloger '%s' missing file content: %+v", upstreamMatcher, reference)
+			continue
+		}
+
+		entries, err := parser(strings.NewReader(content))
+		if err != nil {
+			log.Errorf("cataloger '%s' failed to parse entries (reference=%+v): %w", upstreamMatcher, reference, err)
+			continue
+		}
+
+		for _, entry := range entries {
+			entry.FoundBy = upstreamMatcher
+			entry.Source = []file.Reference{reference}
+
+			packages = append(packages, entry)
+		}
+	}
+
+	return packages, nil
+}
@@ -0,0 +1,10 @@
+package common
+
+import (
+	"io"
+
+	"github.com/anchore/imgbom/imgbom/pkg"
+)
+
+// ParserFn standardizes a function signature for parser functions that accept file contents and return any discovered packages from that file
+type ParserFn func(io.Reader) ([]pkg.Package, error)
@@ -0,0 +1,81 @@
+package cataloger
+
+import (
+	"github.com/anchore/imgbom/imgbom/cataloger/bundler"
+	"github.com/anchore/imgbom/imgbom/cataloger/dpkg"
+	"github.com/anchore/imgbom/imgbom/cataloger/python"
+	"github.com/anchore/imgbom/imgbom/pkg"
+	"github.com/anchore/imgbom/imgbom/scope"
+	"github.com/anchore/imgbom/internal/log"
+	"github.com/anchore/stereoscope/pkg/file"
+	"github.com/hashicorp/go-multierror"
+)
+
+var controllerInstance controller
+
+func init() {
+	controllerInstance = newController()
+}
+
+func Catalog(s scope.Scope) (*pkg.Catalog, error) {
+	return controllerInstance.catalog(s)
+}
+
+type controller struct {
+	catalogers []Cataloger
+}
+
+func newController() controller {
+	ctrlr := controller{
+		catalogers: make([]Cataloger, 0),
+	}
+	ctrlr.add(dpkg.NewCataloger())
+	ctrlr.add(bundler.NewCataloger())
+	ctrlr.add(python.NewCataloger())
+	return ctrlr
+}
+
+func (c *controller) add(a Cataloger) {
+	log.Debugf("adding cataloger: %s", a.Name())
+	c.catalogers = append(c.catalogers, a)
+}
+
+func (c *controller) catalog(s scope.Scope) (*pkg.Catalog, error) {
+	catalog := pkg.NewCatalog()
+	fileSelection := make([]file.Reference, 0)
+
+	// ask catalogers for files to extract from the image tar
+	for _, a := range c.catalogers {
+		fileSelection = append(fileSelection, a.SelectFiles(s.Trees)...)
+		log.Debugf("cataloger '%s' selected '%d' files", a.Name(), len(fileSelection))
+	}
+
+	// fetch contents for requested selection by catalogers
+	contents, err := s.Image.MultipleFileContentsByRef(fileSelection...)
+	if err != nil {
+		return nil, err
+	}
+
+	// perform analysis, accumulating errors for each failed analysis
+	var errs error
+	for _, a := range c.catalogers {
+		// TODO: check for multiple rounds of analyses by Iterate error
+		packages, err := a.Catalog(contents)
+		if err != nil {
+			errs = multierror.Append(errs, err)
+			continue
+		}
+
+		log.Debugf("cataloger '%s' discovered '%d' packages", a.Name(), len(packages))
+
+		for _, p := range packages {
+			catalog.Add(p)
+		}
+	}
+
+	if errs != nil {
+		return nil, errs
+	}
+
+	return catalog, nil
+}
@@ -0,0 +1,34 @@
+package dpkg
+
+import (
+	"github.com/anchore/imgbom/imgbom/cataloger/common"
+	"github.com/anchore/imgbom/imgbom/pkg"
+	"github.com/anchore/stereoscope/pkg/file"
+	"github.com/anchore/stereoscope/pkg/tree"
+)
+
+type Cataloger struct {
+	cataloger common.GenericCataloger
+}
+
+func NewCataloger() *Cataloger {
+	pathParsers := map[string]common.ParserFn{
+		"/var/lib/dpkg/status": parseDpkgStatus,
+	}
+
+	return &Cataloger{
+		cataloger: common.NewGenericCataloger(pathParsers, nil),
+	}
+}
+
+func (a *Cataloger) Name() string {
+	return "dpkg-cataloger"
+}
+
+func (a *Cataloger) SelectFiles(trees []tree.FileTreeReader) []file.Reference {
+	return a.cataloger.SelectFiles(trees)
+}
+
+func (a *Cataloger) Catalog(contents map[file.Reference]string) ([]pkg.Package, error) {
+	return a.cataloger.Catalog(contents, a.Name())
+}
@@ -0,0 +1,99 @@
+package dpkg
+
+import (
+	"bufio"
+	"fmt"
+	"io"
+	"strings"
+
+	"github.com/anchore/imgbom/imgbom/pkg"
+	"github.com/mitchellh/mapstructure"
+)
+
+var errEndOfPackages = fmt.Errorf("no more packages to read")
+
+func parseDpkgStatus(reader io.Reader) ([]pkg.Package, error) {
+	buffedReader := bufio.NewReader(reader)
+	var packages = make([]pkg.Package, 0)
+
+	for {
+		entry, err := parseDpkgStatusEntry(buffedReader)
+		if err != nil {
+			if err == errEndOfPackages {
+				break
+			}
+			return nil, err
+		}
+		packages = append(packages, pkg.Package{
+			Name:     entry.Package,
+			Version:  entry.Version,
+			Type:     pkg.DebPkg,
+			Metadata: entry,
+		})
+	}
+
+	return packages, nil
+}
+
+func parseDpkgStatusEntry(reader *bufio.Reader) (entry pkg.DpkgMetadata, err error) {
+	dpkgFields := make(map[string]string)
+	var key string
+
+	for {
+		line, err := reader.ReadString('\n')
+		if err != nil {
+			if err == io.EOF {
+				return pkg.DpkgMetadata{}, errEndOfPackages
+			}
+			return pkg.DpkgMetadata{}, err
+		}
+
+		line = strings.TrimRight(line, "\n")
+
+		// empty line indicates end of entry
+		if len(line) == 0 {
+			// if the entry has not started, keep parsing lines
+			if len(dpkgFields) == 0 {
+				continue
+			}
+			break
+		}
+
+		switch {
+		case strings.HasPrefix(line, " "):
+			// a field-body continuation
+			if len(key) == 0 {
+				return pkg.DpkgMetadata{}, fmt.Errorf("no match for continuation: line: '%s'", line)
+			}
+
+			val, ok := dpkgFields[key]
+			if !ok {
+				return pkg.DpkgMetadata{}, fmt.Errorf("no previous key exists, expecting: %s", key)
+			}
+			// concatenate onto previous value
+			val = fmt.Sprintf("%s\n %s", val, strings.TrimSpace(line))
+			dpkgFields[key] = val
+		default:
+			// parse a new key
+			if i := strings.Index(line, ":"); i > 0 {
+				key = strings.TrimSpace(line[0:i])
+				val := strings.TrimSpace(line[i+1:])
+
+				if _, ok := dpkgFields[key]; ok {
+					return pkg.DpkgMetadata{}, fmt.Errorf("duplicate key discovered: %s", key)
+				}
+
+				dpkgFields[key] = val
+			} else {
+				return pkg.DpkgMetadata{}, fmt.Errorf("cannot parse field from line: '%s'", line)
+			}
+		}
+	}
+
+	err = mapstructure.Decode(dpkgFields, &entry)
+	if err != nil {
+		return pkg.DpkgMetadata{}, err
+	}
+
+	return entry, nil
+}
@@ -0,0 +1,108 @@
+package dpkg
+
+import (
+	"bufio"
+	"os"
+	"testing"
+
+	"github.com/anchore/imgbom/imgbom/pkg"
+	"github.com/go-test/deep"
+)
+
+func compareEntries(t *testing.T, left, right pkg.DpkgMetadata) {
+	t.Helper()
+	if diff := deep.Equal(left, right); diff != nil {
+		t.Error(diff)
+	}
+}
+
+func TestSinglePackage(t *testing.T) {
+	tests := []struct {
+		name     string
+		expected pkg.DpkgMetadata
+	}{
+		{
+			name: "Test Single Package",
+			expected: pkg.DpkgMetadata{
+				Package: "apt",
+				Source:  "apt-dev",
+				Version: "1.8.2",
+			},
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			file, err := os.Open("test-fixtures/single")
+			if err != nil {
+				t.Fatal("Unable to read test_fixtures/single: ", err)
+			}
+			defer func() {
+				err := file.Close()
+				if err != nil {
+					t.Fatal("closing file failed:", err)
+				}
+			}()
+
+			reader := bufio.NewReader(file)
+
+			entry, err := parseDpkgStatusEntry(reader)
+			if err != nil {
+				t.Fatal("Unable to read file contents: ", err)
+			}
+
+			compareEntries(t, entry, test.expected)
+		})
+	}
+}
+
+func TestMultiplePackages(t *testing.T) {
+	tests := []struct {
+		name     string
+		expected []pkg.DpkgMetadata
+	}{
+		{
+			name: "Test Multiple Package",
+			expected: []pkg.DpkgMetadata{
+				{
+					Package: "tzdata",
+					Version: "2020a-0+deb10u1",
+					Source:  "tzdata-dev",
+				},
+				{
+					Package: "util-linux",
+					Version: "2.33.1-0.1",
+				},
+			},
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			file, err := os.Open("test-fixtures/multiple")
+			if err != nil {
+				t.Fatal("Unable to read: ", err)
+			}
+			defer func() {
+				err := file.Close()
+				if err != nil {
+					t.Fatal("closing file failed:", err)
+				}
+			}()
+
+			pkgs, err := parseDpkgStatus(file)
+			if err != nil {
+				t.Fatal("Unable to read file contents: ", err)
+			}
+
+			if len(pkgs) != 2 {
+				t.Fatalf("unexpected number of entries: %d", len(pkgs))
+			}
+
+			for idx, entry := range pkgs {
+				compareEntries(t, entry.Metadata.(pkg.DpkgMetadata), test.expected[idx])
+			}
+
+		})
+	}
+}
@@ -0,0 +1,52 @@
+Package: tzdata
+Status: install ok installed
+Priority: required
+Source: tzdata-dev
+Section: localization
+Installed-Size: 3036
+Maintainer: GNU Libc Maintainers <debian-glibc@lists.debian.org>
+Architecture: all
+Multi-Arch: foreign
+Version: 2020a-0+deb10u1
+Replaces: libc0.1, libc0.3, libc6, libc6.1
+Provides: tzdata-buster
+Depends: debconf (>= 0.5) | debconf-2.0
+Description: time zone and daylight-saving time data
+ This package contains data required for the implementation of
+ standard local time for many representative locations around the
+ globe. It is updated periodically to reflect changes made by
+ political bodies to time zone boundaries, UTC offsets, and
+ daylight-saving rules.
+Homepage: https://www.iana.org/time-zones
+
+Package: util-linux
+Essential: yes
+Status: install ok installed
+Priority: required
+Section: utils
+Installed-Size: 4327
+Maintainer: LaMont Jones <lamont@debian.org>
+Architecture: amd64
+Multi-Arch: foreign
+Version: 2.33.1-0.1
+Replaces: bash-completion (<< 1:2.8), initscripts (<< 2.88dsf-59.2~), login (<< 1:4.5-1.1~), mount (<< 2.29.2-3~), s390-tools (<< 2.2.0-1~), setpriv (<< 2.32.1-0.2~), sysvinit-utils (<< 2.88dsf-59.1~)
+Depends: fdisk, login (>= 1:4.5-1.1~)
+Pre-Depends: libaudit1 (>= 1:2.2.1), libblkid1 (>= 2.31.1), libc6 (>= 2.25), libcap-ng0 (>= 0.7.9), libmount1 (>= 2.25), libpam0g (>= 0.99.7.1), libselinux1 (>= 2.6-3~), libsmartcols1 (>= 2.33), libsystemd0, libtinfo6 (>= 6), libudev1 (>= 183), libuuid1 (>= 2.16), zlib1g (>= 1:1.1.4)
+Suggests: dosfstools, kbd | console-tools, util-linux-locales
+Breaks: bash-completion (<< 1:2.8), grml-debootstrap (<< 0.68), mount (<< 2.29.2-3~), s390-tools (<< 2.2.0-1~), setpriv (<< 2.32.1-0.2~), sysvinit-utils (<< 2.88dsf-59.4~)
+Conffiles:
+ /etc/default/hwclock 3916544450533eca69131f894db0ca12
+ /etc/init.d/hwclock.sh 1ca5c0743fa797ffa364db95bb8d8d8e
+ /etc/pam.d/runuser b8b44b045259525e0fae9e38fdb2aeeb
+ /etc/pam.d/runuser-l 2106ea05877e8913f34b2c77fa02be45
+ /etc/pam.d/su ce6dcfda3b190a27a455bb38a45ff34a
+ /etc/pam.d/su-l 756fef5687fecc0d986e5951427b0c4f
+Description: miscellaneous system utilities
+ This package contains a number of important utilities, most of which
+ are oriented towards maintenance of your system. Some of the more
+ important utilities included in this package allow you to view kernel
+ messages, create new filesystems, view block device information,
+ interface with real time clock, etc.
+
+
+
@@ -0,0 +1,35 @@
+Package: apt
+Status: install ok installed
+Priority: required
+Section: admin
+Installed-Size: 4064
+Maintainer: APT Development Team <deity@lists.debian.org>
+Architecture: amd64
+Version: 1.8.2
+Source: apt-dev
+Replaces: apt-transport-https (<< 1.5~alpha4~), apt-utils (<< 1.3~exp2~)
+Provides: apt-transport-https (= 1.8.2)
+Depends: adduser, gpgv | gpgv2 | gpgv1, debian-archive-keyring, libapt-pkg5.0 (>= 1.7.0~alpha3~), libc6 (>= 2.15), libgcc1 (>= 1:3.0), libgnutls30 (>= 3.6.6), libseccomp2 (>= 1.0.1), libstdc++6 (>= 5.2)
+Recommends: ca-certificates
+Suggests: apt-doc, aptitude | synaptic | wajig, dpkg-dev (>= 1.17.2), gnupg | gnupg2 | gnupg1, powermgmt-base
+Breaks: apt-transport-https (<< 1.5~alpha4~), apt-utils (<< 1.3~exp2~), aptitude (<< 0.8.10)
+Conffiles:
+ /etc/apt/apt.conf.d/01autoremove 76120d358bc9037bb6358e737b3050b5
+ /etc/cron.daily/apt-compat 49e9b2cfa17849700d4db735d04244f3
+ /etc/kernel/postinst.d/apt-auto-removal 4ad976a68f045517cf4696cec7b8aa3a
+ /etc/logrotate.d/apt 179f2ed4f85cbaca12fa3d69c2a4a1c3
+Description: commandline package manager
+ This package provides commandline tools for searching and
+ managing as well as querying information about packages
+ as a low-level access to all features of the libapt-pkg library.
+ .
+ These include:
+  * apt-get for retrieval of packages and information about them
+    from authenticated sources and for installation, upgrade and
+    removal of packages together with their dependencies
+  * apt-cache for querying available information about installed
+    as well as installable packages
+  * apt-cdrom to use removable media as a source for packages
+  * apt-config as an interface to the configuration settings
+  * apt-key as an interface to manage authentication keys
+
@@ -0,0 +1,35 @@
+package python
+
+import (
+	"github.com/anchore/imgbom/imgbom/cataloger/common"
+	"github.com/anchore/imgbom/imgbom/pkg"
+	"github.com/anchore/stereoscope/pkg/file"
+	"github.com/anchore/stereoscope/pkg/tree"
+)
+
+type Cataloger struct {
+	cataloger common.GenericCataloger
+}
+
+func NewCataloger() *Cataloger {
+	globParsers := map[string]common.ParserFn{
+		"*egg-info/PKG-INFO":  parseEggMetadata,
+		"*dist-info/METADATA": parseWheelMetadata,
+	}
+
+	return &Cataloger{
+		cataloger: common.NewGenericCataloger(nil, globParsers),
+	}
+}
+
+func (a *Cataloger) Name() string {
+	return "python-cataloger"
+}
+
+func (a *Cataloger) SelectFiles(trees []tree.FileTreeReader) []file.Reference {
+	return a.cataloger.SelectFiles(trees)
+}
+
+func (a *Cataloger) Catalog(contents map[file.Reference]string) ([]pkg.Package, error) {
+	return a.cataloger.Catalog(contents, a.Name())
+}
@@ -0,0 +1,89 @@
+package python
+
+import (
+	"bufio"
+	"fmt"
+	"io"
+	"strings"
+
+	"github.com/anchore/imgbom/imgbom/pkg"
+)
+
+func parseWheelMetadata(reader io.Reader) ([]pkg.Package, error) {
+	packages, err := parseWheelOrEggMetadata(reader)
+	for idx := range packages {
+		packages[idx].Type = pkg.WheelPkg
+	}
+	return packages, err
+}
+
+func parseEggMetadata(reader io.Reader) ([]pkg.Package, error) {
+	packages, err := parseWheelOrEggMetadata(reader)
+	for idx := range packages {
+		packages[idx].Type = pkg.EggPkg
+	}
+	return packages, err
+}
+
+func parseWheelOrEggMetadata(reader io.Reader) ([]pkg.Package, error) {
+	fields := make(map[string]string)
+	var key string
+
+	scanner := bufio.NewScanner(reader)
+	for scanner.Scan() {
+		line := scanner.Text()
+
+		line = strings.TrimRight(line, "\n")
+
+		// empty line indicates end of entry
+		if len(line) == 0 {
+			// if the entry has not started, keep parsing lines
+			if len(fields) == 0 {
+				continue
+			}
+			break
+		}
+
+		switch {
+		case strings.HasPrefix(line, " "):
+			// a field-body continuation
+			if len(key) == 0 {
+				return nil, fmt.Errorf("no match for continuation: line: '%s'", line)
+			}
+
+			val, ok := fields[key]
+			if !ok {
+				return nil, fmt.Errorf("no previous key exists, expecting: %s", key)
+			}
+			// concatenate onto previous value
+			val = fmt.Sprintf("%s\n %s", val, strings.TrimSpace(line))
+			fields[key] = val
+		default:
+			// parse a new key (note, duplicate keys are overridden)
+			if i := strings.Index(line, ":"); i > 0 {
+				key = strings.TrimSpace(line[0:i])
+				val := strings.TrimSpace(line[i+1:])
+
+				fields[key] = val
+			} else {
+				return nil, fmt.Errorf("cannot parse field from line: '%s'", line)
+			}
+		}
+	}
+
+	if err := scanner.Err(); err != nil {
+		return nil, fmt.Errorf("failed to parse python wheel/egg: %w", err)
+	}
+
+	p := pkg.Package{
+		Name:     fields["Name"],
+		Version:  fields["Version"],
+		Language: pkg.Python,
+	}
+
+	if license, ok := fields["License"]; ok && license != "" {
+		p.Licenses = []string{license}
+	}
+
+	return []pkg.Package{p}, nil
+}
@@ -0,0 +1,91 @@
+package python
+
+import (
+	"os"
+	"testing"
+
+	"github.com/anchore/imgbom/imgbom/pkg"
+)
+
+func assertPkgsEqual(t *testing.T, actual []pkg.Package, expected map[string]pkg.Package) {
+	t.Helper()
+	if len(actual) != 1 {
+		for _, a := range actual {
+			t.Log("   ", a)
+		}
+		t.Fatalf("unexpected package count: %d!=%d", len(actual), 1)
+	}
+
+	for _, a := range actual {
+		expectedPkg, ok := expected[a.Name]
+		if !ok {
+			t.Errorf("unexpected package found: '%s'", a.Name)
+		}
+
+		if expectedPkg.Version != a.Version {
+			t.Errorf("unexpected package version: '%s'", a.Version)
+		}
+
+		if a.Language != expectedPkg.Language {
+			t.Errorf("bad language: '%+v'", a.Language)
+		}
+
+		if a.Type != expectedPkg.Type {
+			t.Errorf("bad package type: %+v", a.Type)
+		}
+
+		if len(a.Licenses) < 1 {
+			t.Errorf("bad package licenses count: '%+v'", a.Licenses)
+		} else if a.Licenses[0] != expectedPkg.Licenses[0] {
+			t.Errorf("bad package licenses: '%+v'", a.Licenses)
+		}
+
+	}
+}
+func TestParseEggMetadata(t *testing.T) {
+	expected := map[string]pkg.Package{
+		"requests": {
+			Name:     "requests",
+			Version:  "2.22.0",
+			Language: pkg.Python,
+			Type:     pkg.EggPkg,
+			Licenses: []string{"Apache 2.0"},
+		},
+	}
+	fixture, err := os.Open("test-fixtures/egg-info/PKG-INFO")
+	if err != nil {
+		t.Fatalf("failed to open fixture: %+v", err)
+	}
+
+	actual, err := parseEggMetadata(fixture)
+	if err != nil {
+		t.Fatalf("failed to parse egg-info: %+v", err)
+	}
+
+	assertPkgsEqual(t, actual, expected)
+
+}
+
+func TestParseWheelMetadata(t *testing.T) {
+	expected := map[string]pkg.Package{
+		"Pygments": {
+			Name:     "Pygments",
+			Version:  "2.6.1",
+			Language: pkg.Python,
+			Type:     pkg.WheelPkg,
+			Licenses: []string{"BSD License"},
+		},
+	}
+	fixture, err := os.Open("test-fixtures/dist-info/METADATA")
+	if err != nil {
+		t.Fatalf("failed to open fixture: %+v", err)
+	}
+
+	actual, err := parseWheelMetadata(fixture)
+	if err != nil {
+		t.Fatalf("failed to parse dist-info: %+v", err)
+	}
+
+	assertPkgsEqual(t, actual, expected)
+
+}
@@ -0,0 +1,47 @@
+Metadata-Version: 2.1
+Name: Pygments
+Version: 2.6.1
+Summary: Pygments is a syntax highlighting package written in Python.
+Home-page: https://pygments.org/
+Author: Georg Brandl
+Author-email: georg@python.org
+License: BSD License
+Keywords: syntax highlighting
+Platform: any
+Classifier: License :: OSI Approved :: BSD License
+Classifier: Intended Audience :: Developers
+Classifier: Intended Audience :: End Users/Desktop
+Classifier: Intended Audience :: System Administrators
+Classifier: Development Status :: 6 - Mature
+Classifier: Programming Language :: Python
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.5
+Classifier: Programming Language :: Python :: 3.6
+Classifier: Programming Language :: Python :: 3.7
+Classifier: Programming Language :: Python :: 3.8
+Classifier: Programming Language :: Python :: Implementation :: CPython
+Classifier: Programming Language :: Python :: Implementation :: PyPy
+Classifier: Operating System :: OS Independent
+Classifier: Topic :: Text Processing :: Filters
+Classifier: Topic :: Utilities
+Requires-Python: >=3.5
+
+
+Pygments
+~~~~~~~~
+
+Pygments is a syntax highlighting package written in Python.
+
+It is a generic syntax highlighter suitable for use in code hosting, forums,
+wikis or other applications that need to prettify source code.  Highlights
+are:
+
+* a wide range of over 500 languages and other text formats is supported
+* special attention is paid to details, increasing quality by a fair amount
+* support for new languages and formats are added easily
+* a number of output formats, presently HTML, LaTeX, RTF, SVG, all image     formats that PIL supports and ANSI sequences
+* it is usable as a command-line tool and as a library
+
+:copyright: Copyright 2006-2019 by the Pygments team, see AUTHORS.
+:license: BSD, see LICENSE for details.
+
@@ -0,0 +1,134 @@
+Metadata-Version: 2.1
+Name: requests
+Version: 2.22.0
+Summary: Python HTTP for Humans.
+Home-page: http://python-requests.org
+Author: Kenneth Reitz
+Author-email: me@kennethreitz.org
+License: Apache 2.0
+Description: Requests: HTTP for Humans™
+        ==========================
+        
+        [![image](https://img.shields.io/pypi/v/requests.svg)](https://pypi.org/project/requests/)
+        [![image](https://img.shields.io/pypi/l/requests.svg)](https://pypi.org/project/requests/)
+        [![image](https://img.shields.io/pypi/pyversions/requests.svg)](https://pypi.org/project/requests/)
+        [![codecov.io](https://codecov.io/github/requests/requests/coverage.svg?branch=master)](https://codecov.io/github/requests/requests)
+        [![image](https://img.shields.io/github/contributors/requests/requests.svg)](https://github.com/requests/requests/graphs/contributors)
+        [![image](https://img.shields.io/badge/Say%20Thanks-!-1EAEDB.svg)](https://saythanks.io/to/kennethreitz)
+        
+        Requests is the only *Non-GMO* HTTP library for Python, safe for human
+        consumption.
+        
+        ![image](https://farm5.staticflickr.com/4317/35198386374_1939af3de6_k_d.jpg)
+        
+        Behold, the power of Requests:
+        
+        ``` {.sourceCode .python}
+        >>> import requests
+        >>> r = requests.get('https://api.github.com/user', auth=('user', 'pass'))
+        >>> r.status_code
+        200
+        >>> r.headers['content-type']
+        'application/json; charset=utf8'
+        >>> r.encoding
+        'utf-8'
+        >>> r.text
+        u'{"type":"User"...'
+        >>> r.json()
+        {u'disk_usage': 368627, u'private_gists': 484, ...}
+        ```
+        
+        See [the similar code, sans Requests](https://gist.github.com/973705).
+        
+        [![image](https://raw.githubusercontent.com/requests/requests/master/docs/_static/requests-logo-small.png)](http://docs.python-requests.org/)
+        
+        Requests allows you to send *organic, grass-fed* HTTP/1.1 requests,
+        without the need for manual labor. There's no need to manually add query
+        strings to your URLs, or to form-encode your POST data. Keep-alive and
+        HTTP connection pooling are 100% automatic, thanks to
+        [urllib3](https://github.com/shazow/urllib3).
+        
+        Besides, all the cool kids are doing it. Requests is one of the most
+        downloaded Python packages of all time, pulling in over 11,000,000
+        downloads every month. You don't want to be left out!
+        
+        Feature Support
+        ---------------
+        
+        Requests is ready for today's web.
+        
+        -   International Domains and URLs
+        -   Keep-Alive & Connection Pooling
+        -   Sessions with Cookie Persistence
+        -   Browser-style SSL Verification
+        -   Basic/Digest Authentication
+        -   Elegant Key/Value Cookies
+        -   Automatic Decompression
+        -   Automatic Content Decoding
+        -   Unicode Response Bodies
+        -   Multipart File Uploads
+        -   HTTP(S) Proxy Support
+        -   Connection Timeouts
+        -   Streaming Downloads
+        -   `.netrc` Support
+        -   Chunked Requests
+        
+        Requests officially supports Python 2.7 & 3.4–3.7, and runs great on
+        PyPy.
+        
+        Installation
+        ------------
+        
+        To install Requests, simply use [pipenv](http://pipenv.org/) (or pip, of
+        course):
+        
+        ``` {.sourceCode .bash}
+        $ pipenv install requests
+        ✨🍰✨
+        ```
+        
+        Satisfaction guaranteed.
+        
+        Documentation
+        -------------
+        
+        Fantastic documentation is available at
+        <http://docs.python-requests.org/>, for a limited time only.
+        
+        How to Contribute
+        -----------------
+        
+        1.  Become more familiar with the project by reading our [Contributor's Guide](http://docs.python-requests.org/en/latest/dev/contributing/) and our [development philosophy](http://docs.python-requests.org/en/latest/dev/philosophy/).
+        2.  Check for open issues or open a fresh issue to start a discussion
+            around a feature idea or a bug. There is a [Contributor
+            Friendly](https://github.com/requests/requests/issues?direction=desc&labels=Contributor+Friendly&page=1&sort=updated&state=open)
+            tag for issues that should be ideal for people who are not very
+            familiar with the codebase yet.
+        3.  Fork [the repository](https://github.com/requests/requests) on
+            GitHub to start making your changes to the **master** branch (or
+            branch off of it).
+        4.  Write a test which shows that the bug was fixed or that the feature
+            works as expected.
+        5.  Send a pull request and bug the maintainer until it gets merged and
+            published. :) Make sure to add yourself to
+            [AUTHORS](https://github.com/requests/requests/blob/master/AUTHORS.rst).
+        
+        
+Platform: UNKNOWN
+Classifier: Development Status :: 5 - Production/Stable
+Classifier: Intended Audience :: Developers
+Classifier: Natural Language :: English
+Classifier: License :: OSI Approved :: Apache Software License
+Classifier: Programming Language :: Python
+Classifier: Programming Language :: Python :: 2
+Classifier: Programming Language :: Python :: 2.7
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.5
+Classifier: Programming Language :: Python :: 3.6
+Classifier: Programming Language :: Python :: 3.7
+Classifier: Programming Language :: Python :: Implementation :: CPython
+Classifier: Programming Language :: Python :: Implementation :: PyPy
+Requires-Python: >=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*
+Description-Content-Type: text/markdown
+Provides-Extra: security
+Provides-Extra: socks