Add poetry cataloger (#121)

* Minor cleanup

Signed-off-by: Dan Luhring <dan.luhring@anchore.com>

* Update pkg Type definition to string

Signed-off-by: Dan Luhring <dan.luhring@anchore.com>

* Implement poetry.lock parsing

Signed-off-by: Dan Luhring <dan.luhring@anchore.com>

* Address CI issues

Signed-off-by: Dan Luhring <dan.luhring@anchore.com>

* Integrate Alex's changes

Signed-off-by: Dan Luhring <dan.luhring@anchore.com>

Makefile

@@ -73,9 +73,9 @@ help:
 ci-bootstrap: bootstrap
 	sudo apt install -y bc
 
-.PHONY: boostrap
+.PHONY: bootstrap
 bootstrap: ## Download and install all go dependencies (+ prep tooling in the ./tmp dir)
-	$(call title,Boostrapping dependencies)
+	$(call title,Bootstrapping dependencies)
 	@pwd
 	# prep temp dirs
 	mkdir -p $(TEMPDIR)

cmd/root.go

@@ -19,7 +19,7 @@ import (
 var rootCmd = &cobra.Command{
 	Use:   fmt.Sprintf("%s [SOURCE]", internal.ApplicationName),
 	Short: "A tool for generating a Software Bill Of Materials (SBOM) from container images and filesystems",
-	Long: internal.Tprintf(`\
+	Long: internal.Tprintf(`
 Supports the following image sources:
     {{.appName}} yourrepo/yourimage:tag             defaults to using images from a docker daemon
     {{.appName}} docker://yourrepo/yourimage:tag    explicitly use the docker daemon

go.mod

@@ -19,6 +19,7 @@ require (
 	github.com/mitchellh/go-homedir v1.1.0
 	github.com/mitchellh/mapstructure v1.3.1
 	github.com/olekukonko/tablewriter v0.0.4
+	github.com/pelletier/go-toml v1.8.0
 	github.com/rogpeppe/go-internal v1.5.2
 	github.com/sergi/go-diff v1.1.0
 	github.com/sirupsen/logrus v1.6.0

json-schema/schema.json

@@ -347,7 +347,7 @@
                                                 "type": "null"
                                             },
                                             "type": {
-                                                "type": "integer"
+                                                "type": "string"
                                             },
                                             "version": {
                                                 "type": "string"
@@ -520,4 +520,4 @@
         "artifacts"
     ],
     "type": "object"
-}
+}

syft/cataloger/java/archive_parser.go

@@ -133,6 +133,7 @@ func (j *archiveParser) discoverMainPackage() (*pkg.Package, error) {
 		Name:     selectName(manifest, j.fileInfo),
 		Version:  selectVersion(manifest, j.fileInfo),
 		Language: pkg.Java,
+		Type:     pkg.JavaPkg,
 		Metadata: pkg.JavaMetadata{
 			Manifest: manifest,
 		},
@@ -165,6 +166,7 @@ func (j *archiveParser) discoverPkgsFromPomProperties(parentPkg *pkg.Package) ([
 					Name:     propsObj.ArtifactID,
 					Version:  propsObj.Version,
 					Language: pkg.Java,
+					Type:     pkg.JavaPkg,
 					Metadata: pkg.JavaMetadata{
 						PomProperties: propsObj,
 						Parent:        parentPkg,

syft/cataloger/java/archive_parser_test.go

@@ -227,7 +227,7 @@ func TestParseJar(t *testing.T) {
 					Name:     "joda-time",
 					Version:  "2.9.2",
 					Language: pkg.Java,
-					Type:     pkg.UnknownPkg,
+					Type:     pkg.JavaPkg,
 					Metadata: pkg.JavaMetadata{
 						PomProperties: &pkg.PomProperties{
 							Path:       "META-INF/maven/joda-time/joda-time/pom.properties",

syft/cataloger/python/cataloger.go

@@ -19,6 +19,7 @@ func NewCataloger() *Cataloger {
 		"**/*egg-info/PKG-INFO":  parseEggMetadata,
 		"**/*dist-info/METADATA": parseWheelMetadata,
 		"**/requirements.txt":    parseRequirementsTxt,
+		"**/poetry.lock":         parsePoetryLock,
 	}
 
 	return &Cataloger{

syft/cataloger/python/parse_poetry_lock.go

@@ -0,0 +1,24 @@
+package python
+
+import (
+	"fmt"
+	"io"
+
+	"github.com/anchore/syft/syft/pkg"
+	"github.com/pelletier/go-toml"
+)
+
+func parsePoetryLock(_ string, reader io.Reader) ([]pkg.Package, error) {
+	tree, err := toml.LoadReader(reader)
+	if err != nil {
+		return nil, fmt.Errorf("unable to load poetry.lock for parsing: %v", err)
+	}
+
+	metadata := PoetryMetadata{}
+	err = tree.Unmarshal(&metadata)
+	if err != nil {
+		return nil, fmt.Errorf("unable to parse poetry.lock: %v", err)
+	}
+
+	return metadata.Pkgs(), nil
+}

syft/cataloger/python/parse_poetry_lock_test.go

@@ -0,0 +1,56 @@
+package python
+
+import (
+	"github.com/anchore/syft/syft/pkg"
+	"github.com/go-test/deep"
+	"os"
+	"testing"
+)
+
+func TestParsePoetryLock(t *testing.T) {
+	expected := []pkg.Package{
+		{
+			Name:     "added-value",
+			Version:  "0.14.2",
+			Language: pkg.Python,
+			Type:     pkg.PoetryPkg,
+			Licenses: nil,
+		},
+		{
+			Name:     "alabaster",
+			Version:  "0.7.12",
+			Language: pkg.Python,
+			Type:     pkg.PoetryPkg,
+			Licenses: nil,
+		},
+		{
+			Name:     "appnope",
+			Version:  "0.1.0",
+			Language: pkg.Python,
+			Type:     pkg.PoetryPkg,
+			Licenses: nil,
+		},
+		{
+			Name:     "asciitree",
+			Version:  "0.3.3",
+			Language: pkg.Python,
+			Type:     pkg.PoetryPkg,
+			Licenses: nil,
+		},
+	}
+
+	fixture, err := os.Open("test-fixtures/poetry/poetry.lock")
+	if err != nil {
+		t.Fatalf("failed to open fixture: %+v", err)
+	}
+
+	actual, err := parsePoetryLock(fixture.Name(), fixture)
+	if err != nil {
+		t.Error(err)
+	}
+
+	differences := deep.Equal(expected, actual)
+	if differences != nil {
+		t.Errorf("returned package list differed from expectation: %+v", differences)
+	}
+}

syft/cataloger/python/parse_wheel_egg_test.go

@@ -45,6 +45,7 @@ func assertPkgsEqual(t *testing.T, actual []pkg.Package, expected map[string]pkg
 
 	}
 }
+
 func TestParseEggMetadata(t *testing.T) {
 	expected := map[string]pkg.Package{
 		"requests": {
@@ -66,7 +67,6 @@ func TestParseEggMetadata(t *testing.T) {
 	}
 
 	assertPkgsEqual(t, actual, expected)
-
 }
 
 func TestParseWheelMetadata(t *testing.T) {
@@ -90,5 +90,4 @@ func TestParseWheelMetadata(t *testing.T) {
 	}
 
 	assertPkgsEqual(t, actual, expected)
-
 }

syft/cataloger/python/poetry_metadata.go

@@ -0,0 +1,18 @@
+package python
+
+import "github.com/anchore/syft/syft/pkg"
+
+type PoetryMetadata struct {
+	Packages []PoetryMetadataPackage `toml:"package"`
+}
+
+// Pkgs returns all of the packages referenced within the poetry.lock metadata.
+func (m PoetryMetadata) Pkgs() []pkg.Package {
+	pkgs := make([]pkg.Package, 0)
+
+	for _, p := range m.Packages {
+		pkgs = append(pkgs, p.Pkg())
+	}
+
+	return pkgs
+}

syft/cataloger/python/poetry_metadata_package.go

@@ -0,0 +1,21 @@
+package python
+
+import "github.com/anchore/syft/syft/pkg"
+
+type PoetryMetadataPackage struct {
+	Name        string `toml:"name"`
+	Version     string `toml:"version"`
+	Category    string `toml:"category"`
+	Description string `toml:"description"`
+	Optional    bool   `toml:"optional"`
+}
+
+// Pkg returns the standard `pkg.Package` representation of the package referenced within the poetry.lock metadata.
+func (p PoetryMetadataPackage) Pkg() pkg.Package {
+	return pkg.Package{
+		Name:     p.Name,
+		Version:  p.Version,
+		Language: pkg.Python,
+		Type:     pkg.PoetryPkg,
+	}
+}

syft/cataloger/python/test-fixtures/poetry/poetry.lock

@@ -0,0 +1,64 @@
+[[package]]
+category = "dev"
+description = "Sphinx \"added-value\" extension"
+name = "added-value"
+optional = false
+python-versions = "*"
+version = "0.14.2"
+
+[package.dependencies]
+docutils = "*"
+natsort = "*"
+six = "*"
+sphinx = "*"
+
+[package.extras]
+deploy = ["bumpversion", "twine", "wheel"]
+docs = ["sphinx", "sphinx-rtd-theme"]
+test = ["pytest", "pytest-cov", "coveralls", "beautifulsoup4", "hypothesis"]
+
+[[package]]
+category = "dev"
+description = "A configurable sidebar-enabled Sphinx theme"
+name = "alabaster"
+optional = false
+python-versions = "*"
+version = "0.7.12"
+
+[[package]]
+category = "dev"
+description = "Disable App Nap on OS X 10.9"
+marker = "python_version >= \"3.4\" and sys_platform == \"darwin\" or sys_platform == \"darwin\""
+name = "appnope"
+optional = false
+python-versions = "*"
+version = "0.1.0"
+
+[[package]]
+category = "dev"
+description = "Draws ASCII trees."
+name = "asciitree"
+optional = false
+python-versions = "*"
+version = "0.3.3"
+
+[metadata]
+content-hash = "6e35f765c2f01c635c2fb0d54a9d7dd68742350f04449ee24efad03e3c9eb0bb"
+python-versions = "^3.6"
+
+[metadata.files]
+added-value = [
+    {file = "added-value-0.14.2.tar.gz", hash = "sha256:8c886aee74635cec15beb64c28a90ae664526f331c1a4941e4d6ab98af232028"},
+    {file = "added_value-0.14.2-py2.py3-none-any.whl", hash = "sha256:b25fcb86f9bfad9a40adf4d344322690e312741556c7b75681bc948380a251e6"},
+]
+alabaster = [
+    {file = "alabaster-0.7.12-py2.py3-none-any.whl", hash = "sha256:446438bdcca0e05bd45ea2de1668c1d9b032e1a9154c2c259092d77031ddd359"},
+    {file = "alabaster-0.7.12.tar.gz", hash = "sha256:a661d72d58e6ea8a57f7a86e37d86716863ee5e92788398526d58b26a4e4dc02"},
+]
+appnope = [
+    {file = "appnope-0.1.0-py2.py3-none-any.whl", hash = "sha256:5b26757dc6f79a3b7dc9fab95359328d5747fcb2409d331ea66d0272b90ab2a0"},
+    {file = "appnope-0.1.0.tar.gz", hash = "sha256:8b995ffe925347a2138d7ac0fe77155e4311a0ea6d6da4f5128fe4b3cbe5ed71"},
+]
+asciitree = [
+    {file = "asciitree-0.3.3.tar.gz", hash = "sha256:4aa4b9b649f85e3fcb343363d97564aa1fb62e249677f2e18a96765145cc0f6e"},
+]

syft/pkg/package.go

@@ -15,7 +15,7 @@ type Package struct {
 	id       ID               // this is set when a package is added to the catalog
 	Name     string           `json:"manifest"`
 	Version  string           `json:"version"`
-	FoundBy  string           `json:"found-by"`
+	FoundBy  string           `json:"found-by"` // FoundBy is the cataloger that discovered this package
 	Source   []file.Reference `json:"sources"`
 	Licenses []string         `json:"licenses"` // TODO: should we move this into metadata?
 	Language Language         `json:"language"` // TODO: should this support multiple languages as a slice?

syft/pkg/type.go

@@ -1,47 +1,31 @@
 package pkg
 
+type Type string
+
 const (
-	UnknownPkg Type = iota
+	UnknownPkg Type = "UnknownPackage"
-	ApkPkg
+	ApkPkg     Type = "apk"
-	BundlerPkg
+	BundlerPkg Type = "bundle"
-	DebPkg
+	DebPkg     Type = "deb"
-	EggPkg
+	EggPkg     Type = "egg"
-	//PacmanPkg
+	// PacmanPkg Type = "pacman"
-	RpmPkg
+	RpmPkg                Type = "rpm"
-	WheelPkg
+	WheelPkg              Type = "wheel"
-	NpmPkg
+	PoetryPkg             Type = "poetry"
-	YarnPkg
+	NpmPkg                Type = "npm"
-	PythonRequirementsPkg
+	YarnPkg               Type = "yarn"
-	JavaPkg
+	PythonRequirementsPkg Type = "python-requirements"
-	JenkinsPluginPkg
+	JavaPkg               Type = "java-archive"
-	GoModulePkg
+	JenkinsPluginPkg      Type = "jenkins-plugin"
+	GoModulePkg           Type = "go-module"
 )
 
-type Type uint
-
-var typeStr = []string{
-	"UnknownPackage",
-	"apk",
-	"bundle",
-	"deb",
-	"egg",
-	//"pacman",
-	"rpm",
-	"wheel",
-	"npm",
-	"yarn",
-	"python-requirements",
-	"java-archive",
-	"jenkins-plugin",
-	"go-module",
-}
-
 var AllPkgs = []Type{
 	ApkPkg,
 	BundlerPkg,
 	DebPkg,
 	EggPkg,
-	//PacmanPkg,
+	// PacmanPkg,
 	RpmPkg,
 	WheelPkg,
 	NpmPkg,
@@ -51,10 +35,3 @@ var AllPkgs = []Type{
 	JenkinsPluginPkg,
 	GoModulePkg,
 }
-
-func (t Type) String() string {
-	if int(t) >= len(typeStr) {
-		return typeStr[0]
-	}
-	return typeStr[t]
-}

syft/presenter/json/presenter.go

@@ -94,7 +94,7 @@ func (pres *Presenter) Present(output io.Writer) error {
 		art := artifact{
 			Name:     p.Name,
 			Version:  p.Version,
-			Type:     p.Type.String(),
+			Type:     string(p.Type),
 			Sources:  make([]source, len(p.Source)),
 			Metadata: p.Metadata,
 		}

syft/presenter/table/presenter.go

@@ -31,7 +31,7 @@ func (pres *Presenter) Present(output io.Writer) error {
 		row := []string{
 			p.Name,
 			p.Version,
-			p.Type.String(),
+			string(p.Type),
 		}
 		rows = append(rows, row)
 	}

syft/presenter/text/presenter.go

@@ -52,7 +52,7 @@ func (pres *Presenter) Present(output io.Writer) error {
 	for _, p := range pres.catalog.Sorted() {
 		fmt.Fprintln(w, fmt.Sprintf("[%s]", p.Name))
 		fmt.Fprintln(w, " Version:\t", p.Version)
-		fmt.Fprintln(w, " Type:\t", p.Type.String())
+		fmt.Fprintln(w, " Type:\t", string(p.Type))
 		fmt.Fprintln(w, " Found by:\t", p.FoundBy)
 		fmt.Fprintln(w)
 		w.Flush()

test/integration/pkg_cases.go

@@ -31,6 +31,7 @@ var cases = []struct {
 		pkgInfo: map[string]string{
 			"example-java-app-maven": "0.1.0",
 			"example-jenkins-plugin": "1.0-SNAPSHOT", // the jenkins HPI file has a nested JAR of the same name
+			"joda-time":              "2.9.2",
 		},
 	},
 	{

test/integration/pkg_coverage_test.go

@@ -34,7 +34,7 @@ func TestPkgCoverageImage(t *testing.T) {
 	observedPkgs := internal.NewStringSet()
 	definedPkgs := internal.NewStringSet()
 	for _, p := range pkg.AllPkgs {
-		definedPkgs.Add(p.String())
+		definedPkgs.Add(string(p))
 	}
 
 	for _, c := range cases {
@@ -44,7 +44,7 @@ func TestPkgCoverageImage(t *testing.T) {
 			for a := range catalog.Enumerate(c.pkgType) {
 
 				observedLanguages.Add(a.Language.String())
-				observedPkgs.Add(a.Type.String())
+				observedPkgs.Add(string(a.Type))
 
 				expectedVersion, ok := c.pkgInfo[a.Name]
 				if !ok {
@@ -77,8 +77,8 @@ func TestPkgCoverageImage(t *testing.T) {
 
 	observedLanguages.Remove(pkg.UnknownLanguage.String())
 	definedLanguages.Remove(pkg.UnknownLanguage.String())
-	observedPkgs.Remove(pkg.UnknownPkg.String())
+	observedPkgs.Remove(string(pkg.UnknownPkg))
-	definedPkgs.Remove(pkg.UnknownPkg.String())
+	definedPkgs.Remove(string(pkg.UnknownPkg))
 
 	// ensure that integration test cases stay in sync with the available catalogers
 	if len(observedLanguages) < len(definedLanguages) {
@@ -106,7 +106,7 @@ func TestPkgCoverageDirectory(t *testing.T) {
 	observedPkgs := internal.NewStringSet()
 	definedPkgs := internal.NewStringSet()
 	for _, p := range pkg.AllPkgs {
-		definedPkgs.Add(p.String())
+		definedPkgs.Add(string(p))
 	}
 
 	for _, c := range cases {
@@ -116,7 +116,7 @@ func TestPkgCoverageDirectory(t *testing.T) {
 			for a := range catalog.Enumerate(c.pkgType) {
 
 				observedLanguages.Add(a.Language.String())
-				observedPkgs.Add(a.Type.String())
+				observedPkgs.Add(string(a.Type))
 
 				expectedVersion, ok := c.pkgInfo[a.Name]
 				if !ok {
@@ -149,8 +149,8 @@ func TestPkgCoverageDirectory(t *testing.T) {
 
 	observedLanguages.Remove(pkg.UnknownLanguage.String())
 	definedLanguages.Remove(pkg.UnknownLanguage.String())
-	observedPkgs.Remove(pkg.UnknownPkg.String())
+	observedPkgs.Remove(string(pkg.UnknownPkg))
-	definedPkgs.Remove(pkg.UnknownPkg.String())
+	definedPkgs.Remove(string(pkg.UnknownPkg))
 
 	// ensure that integration test cases stay in sync with the available catalogers
 	if len(observedLanguages) < len(definedLanguages) {