From 70e673204c0cf335683b366295eb93257aefbaac Mon Sep 17 00:00:00 2001 From: Dan Luhring Date: Tue, 4 Aug 2020 18:22:43 -0400 Subject: [PATCH] Add poetry cataloger (#121) * Minor cleanup Signed-off-by: Dan Luhring * Update pkg Type definition to string Signed-off-by: Dan Luhring * Implement poetry.lock parsing Signed-off-by: Dan Luhring * Address CI issues Signed-off-by: Dan Luhring * Integrate Alex's changes Signed-off-by: Dan Luhring --- Makefile | 4 +- cmd/root.go | 2 +- go.mod | 1 + json-schema/schema.json | 4 +- syft/cataloger/java/archive_parser.go | 2 + syft/cataloger/java/archive_parser_test.go | 2 +- syft/cataloger/python/cataloger.go | 1 + syft/cataloger/python/parse_poetry_lock.go | 24 +++++++ .../python/parse_poetry_lock_test.go | 56 ++++++++++++++++ syft/cataloger/python/parse_wheel_egg_test.go | 3 +- syft/cataloger/python/poetry_metadata.go | 18 ++++++ .../python/poetry_metadata_package.go | 21 ++++++ .../python/test-fixtures/poetry/poetry.lock | 64 +++++++++++++++++++ syft/pkg/package.go | 2 +- syft/pkg/type.go | 59 ++++++----------- syft/presenter/json/presenter.go | 2 +- syft/presenter/table/presenter.go | 2 +- syft/presenter/text/presenter.go | 2 +- test/integration/pkg_cases.go | 1 + test/integration/pkg_coverage_test.go | 16 ++--- 20 files changed, 225 insertions(+), 61 deletions(-) create mode 100644 syft/cataloger/python/parse_poetry_lock.go create mode 100644 syft/cataloger/python/parse_poetry_lock_test.go create mode 100644 syft/cataloger/python/poetry_metadata.go create mode 100644 syft/cataloger/python/poetry_metadata_package.go create mode 100644 syft/cataloger/python/test-fixtures/poetry/poetry.lock diff --git a/Makefile b/Makefile index cbb754def..e77a8783a 100644 --- a/Makefile +++ b/Makefile @@ -73,9 +73,9 @@ help: ci-bootstrap: bootstrap sudo apt install -y bc -.PHONY: boostrap +.PHONY: bootstrap bootstrap: ## Download and install all go dependencies (+ prep tooling in the ./tmp dir) - $(call title,Boostrapping dependencies) + $(call title,Bootstrapping dependencies) @pwd # prep temp dirs mkdir -p $(TEMPDIR) diff --git a/cmd/root.go b/cmd/root.go index 8c92e1bd0..31b51b852 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -19,7 +19,7 @@ import ( var rootCmd = &cobra.Command{ Use: fmt.Sprintf("%s [SOURCE]", internal.ApplicationName), Short: "A tool for generating a Software Bill Of Materials (SBOM) from container images and filesystems", - Long: internal.Tprintf(`\ + Long: internal.Tprintf(` Supports the following image sources: {{.appName}} yourrepo/yourimage:tag defaults to using images from a docker daemon {{.appName}} docker://yourrepo/yourimage:tag explicitly use the docker daemon diff --git a/go.mod b/go.mod index 20a8baba4..c9357033a 100644 --- a/go.mod +++ b/go.mod @@ -19,6 +19,7 @@ require ( github.com/mitchellh/go-homedir v1.1.0 github.com/mitchellh/mapstructure v1.3.1 github.com/olekukonko/tablewriter v0.0.4 + github.com/pelletier/go-toml v1.8.0 github.com/rogpeppe/go-internal v1.5.2 github.com/sergi/go-diff v1.1.0 github.com/sirupsen/logrus v1.6.0 diff --git a/json-schema/schema.json b/json-schema/schema.json index 52899ed6f..10b1b447d 100644 --- a/json-schema/schema.json +++ b/json-schema/schema.json @@ -347,7 +347,7 @@ "type": "null" }, "type": { - "type": "integer" + "type": "string" }, "version": { "type": "string" @@ -520,4 +520,4 @@ "artifacts" ], "type": "object" -} \ No newline at end of file +} diff --git a/syft/cataloger/java/archive_parser.go b/syft/cataloger/java/archive_parser.go index 5481eb76e..ab8edc8cf 100644 --- a/syft/cataloger/java/archive_parser.go +++ b/syft/cataloger/java/archive_parser.go @@ -133,6 +133,7 @@ func (j *archiveParser) discoverMainPackage() (*pkg.Package, error) { Name: selectName(manifest, j.fileInfo), Version: selectVersion(manifest, j.fileInfo), Language: pkg.Java, + Type: pkg.JavaPkg, Metadata: pkg.JavaMetadata{ Manifest: manifest, }, @@ -165,6 +166,7 @@ func (j *archiveParser) discoverPkgsFromPomProperties(parentPkg *pkg.Package) ([ Name: propsObj.ArtifactID, Version: propsObj.Version, Language: pkg.Java, + Type: pkg.JavaPkg, Metadata: pkg.JavaMetadata{ PomProperties: propsObj, Parent: parentPkg, diff --git a/syft/cataloger/java/archive_parser_test.go b/syft/cataloger/java/archive_parser_test.go index 00185164a..df290195b 100644 --- a/syft/cataloger/java/archive_parser_test.go +++ b/syft/cataloger/java/archive_parser_test.go @@ -227,7 +227,7 @@ func TestParseJar(t *testing.T) { Name: "joda-time", Version: "2.9.2", Language: pkg.Java, - Type: pkg.UnknownPkg, + Type: pkg.JavaPkg, Metadata: pkg.JavaMetadata{ PomProperties: &pkg.PomProperties{ Path: "META-INF/maven/joda-time/joda-time/pom.properties", diff --git a/syft/cataloger/python/cataloger.go b/syft/cataloger/python/cataloger.go index e68bd0844..3cbcd5b50 100644 --- a/syft/cataloger/python/cataloger.go +++ b/syft/cataloger/python/cataloger.go @@ -19,6 +19,7 @@ func NewCataloger() *Cataloger { "**/*egg-info/PKG-INFO": parseEggMetadata, "**/*dist-info/METADATA": parseWheelMetadata, "**/requirements.txt": parseRequirementsTxt, + "**/poetry.lock": parsePoetryLock, } return &Cataloger{ diff --git a/syft/cataloger/python/parse_poetry_lock.go b/syft/cataloger/python/parse_poetry_lock.go new file mode 100644 index 000000000..ba0cc5bf0 --- /dev/null +++ b/syft/cataloger/python/parse_poetry_lock.go @@ -0,0 +1,24 @@ +package python + +import ( + "fmt" + "io" + + "github.com/anchore/syft/syft/pkg" + "github.com/pelletier/go-toml" +) + +func parsePoetryLock(_ string, reader io.Reader) ([]pkg.Package, error) { + tree, err := toml.LoadReader(reader) + if err != nil { + return nil, fmt.Errorf("unable to load poetry.lock for parsing: %v", err) + } + + metadata := PoetryMetadata{} + err = tree.Unmarshal(&metadata) + if err != nil { + return nil, fmt.Errorf("unable to parse poetry.lock: %v", err) + } + + return metadata.Pkgs(), nil +} diff --git a/syft/cataloger/python/parse_poetry_lock_test.go b/syft/cataloger/python/parse_poetry_lock_test.go new file mode 100644 index 000000000..e96c47836 --- /dev/null +++ b/syft/cataloger/python/parse_poetry_lock_test.go @@ -0,0 +1,56 @@ +package python + +import ( + "github.com/anchore/syft/syft/pkg" + "github.com/go-test/deep" + "os" + "testing" +) + +func TestParsePoetryLock(t *testing.T) { + expected := []pkg.Package{ + { + Name: "added-value", + Version: "0.14.2", + Language: pkg.Python, + Type: pkg.PoetryPkg, + Licenses: nil, + }, + { + Name: "alabaster", + Version: "0.7.12", + Language: pkg.Python, + Type: pkg.PoetryPkg, + Licenses: nil, + }, + { + Name: "appnope", + Version: "0.1.0", + Language: pkg.Python, + Type: pkg.PoetryPkg, + Licenses: nil, + }, + { + Name: "asciitree", + Version: "0.3.3", + Language: pkg.Python, + Type: pkg.PoetryPkg, + Licenses: nil, + }, + } + + fixture, err := os.Open("test-fixtures/poetry/poetry.lock") + if err != nil { + t.Fatalf("failed to open fixture: %+v", err) + } + + actual, err := parsePoetryLock(fixture.Name(), fixture) + if err != nil { + t.Error(err) + } + + differences := deep.Equal(expected, actual) + if differences != nil { + t.Errorf("returned package list differed from expectation: %+v", differences) + } +} diff --git a/syft/cataloger/python/parse_wheel_egg_test.go b/syft/cataloger/python/parse_wheel_egg_test.go index 86bcd8639..118573562 100644 --- a/syft/cataloger/python/parse_wheel_egg_test.go +++ b/syft/cataloger/python/parse_wheel_egg_test.go @@ -45,6 +45,7 @@ func assertPkgsEqual(t *testing.T, actual []pkg.Package, expected map[string]pkg } } + func TestParseEggMetadata(t *testing.T) { expected := map[string]pkg.Package{ "requests": { @@ -66,7 +67,6 @@ func TestParseEggMetadata(t *testing.T) { } assertPkgsEqual(t, actual, expected) - } func TestParseWheelMetadata(t *testing.T) { @@ -90,5 +90,4 @@ func TestParseWheelMetadata(t *testing.T) { } assertPkgsEqual(t, actual, expected) - } diff --git a/syft/cataloger/python/poetry_metadata.go b/syft/cataloger/python/poetry_metadata.go new file mode 100644 index 000000000..8b15dd7e0 --- /dev/null +++ b/syft/cataloger/python/poetry_metadata.go @@ -0,0 +1,18 @@ +package python + +import "github.com/anchore/syft/syft/pkg" + +type PoetryMetadata struct { + Packages []PoetryMetadataPackage `toml:"package"` +} + +// Pkgs returns all of the packages referenced within the poetry.lock metadata. +func (m PoetryMetadata) Pkgs() []pkg.Package { + pkgs := make([]pkg.Package, 0) + + for _, p := range m.Packages { + pkgs = append(pkgs, p.Pkg()) + } + + return pkgs +} diff --git a/syft/cataloger/python/poetry_metadata_package.go b/syft/cataloger/python/poetry_metadata_package.go new file mode 100644 index 000000000..20a42ae05 --- /dev/null +++ b/syft/cataloger/python/poetry_metadata_package.go @@ -0,0 +1,21 @@ +package python + +import "github.com/anchore/syft/syft/pkg" + +type PoetryMetadataPackage struct { + Name string `toml:"name"` + Version string `toml:"version"` + Category string `toml:"category"` + Description string `toml:"description"` + Optional bool `toml:"optional"` +} + +// Pkg returns the standard `pkg.Package` representation of the package referenced within the poetry.lock metadata. +func (p PoetryMetadataPackage) Pkg() pkg.Package { + return pkg.Package{ + Name: p.Name, + Version: p.Version, + Language: pkg.Python, + Type: pkg.PoetryPkg, + } +} diff --git a/syft/cataloger/python/test-fixtures/poetry/poetry.lock b/syft/cataloger/python/test-fixtures/poetry/poetry.lock new file mode 100644 index 000000000..489f1df6d --- /dev/null +++ b/syft/cataloger/python/test-fixtures/poetry/poetry.lock @@ -0,0 +1,64 @@ +[[package]] +category = "dev" +description = "Sphinx \"added-value\" extension" +name = "added-value" +optional = false +python-versions = "*" +version = "0.14.2" + +[package.dependencies] +docutils = "*" +natsort = "*" +six = "*" +sphinx = "*" + +[package.extras] +deploy = ["bumpversion", "twine", "wheel"] +docs = ["sphinx", "sphinx-rtd-theme"] +test = ["pytest", "pytest-cov", "coveralls", "beautifulsoup4", "hypothesis"] + +[[package]] +category = "dev" +description = "A configurable sidebar-enabled Sphinx theme" +name = "alabaster" +optional = false +python-versions = "*" +version = "0.7.12" + +[[package]] +category = "dev" +description = "Disable App Nap on OS X 10.9" +marker = "python_version >= \"3.4\" and sys_platform == \"darwin\" or sys_platform == \"darwin\"" +name = "appnope" +optional = false +python-versions = "*" +version = "0.1.0" + +[[package]] +category = "dev" +description = "Draws ASCII trees." +name = "asciitree" +optional = false +python-versions = "*" +version = "0.3.3" + +[metadata] +content-hash = "6e35f765c2f01c635c2fb0d54a9d7dd68742350f04449ee24efad03e3c9eb0bb" +python-versions = "^3.6" + +[metadata.files] +added-value = [ + {file = "added-value-0.14.2.tar.gz", hash = "sha256:8c886aee74635cec15beb64c28a90ae664526f331c1a4941e4d6ab98af232028"}, + {file = "added_value-0.14.2-py2.py3-none-any.whl", hash = "sha256:b25fcb86f9bfad9a40adf4d344322690e312741556c7b75681bc948380a251e6"}, +] +alabaster = [ + {file = "alabaster-0.7.12-py2.py3-none-any.whl", hash = "sha256:446438bdcca0e05bd45ea2de1668c1d9b032e1a9154c2c259092d77031ddd359"}, + {file = "alabaster-0.7.12.tar.gz", hash = "sha256:a661d72d58e6ea8a57f7a86e37d86716863ee5e92788398526d58b26a4e4dc02"}, +] +appnope = [ + {file = "appnope-0.1.0-py2.py3-none-any.whl", hash = "sha256:5b26757dc6f79a3b7dc9fab95359328d5747fcb2409d331ea66d0272b90ab2a0"}, + {file = "appnope-0.1.0.tar.gz", hash = "sha256:8b995ffe925347a2138d7ac0fe77155e4311a0ea6d6da4f5128fe4b3cbe5ed71"}, +] +asciitree = [ + {file = "asciitree-0.3.3.tar.gz", hash = "sha256:4aa4b9b649f85e3fcb343363d97564aa1fb62e249677f2e18a96765145cc0f6e"}, +] diff --git a/syft/pkg/package.go b/syft/pkg/package.go index 1964956ea..85b7bc432 100644 --- a/syft/pkg/package.go +++ b/syft/pkg/package.go @@ -15,7 +15,7 @@ type Package struct { id ID // this is set when a package is added to the catalog Name string `json:"manifest"` Version string `json:"version"` - FoundBy string `json:"found-by"` + FoundBy string `json:"found-by"` // FoundBy is the cataloger that discovered this package Source []file.Reference `json:"sources"` Licenses []string `json:"licenses"` // TODO: should we move this into metadata? Language Language `json:"language"` // TODO: should this support multiple languages as a slice? diff --git a/syft/pkg/type.go b/syft/pkg/type.go index bdf906e45..27ef5d271 100644 --- a/syft/pkg/type.go +++ b/syft/pkg/type.go @@ -1,47 +1,31 @@ package pkg +type Type string + const ( - UnknownPkg Type = iota - ApkPkg - BundlerPkg - DebPkg - EggPkg - //PacmanPkg - RpmPkg - WheelPkg - NpmPkg - YarnPkg - PythonRequirementsPkg - JavaPkg - JenkinsPluginPkg - GoModulePkg + UnknownPkg Type = "UnknownPackage" + ApkPkg Type = "apk" + BundlerPkg Type = "bundle" + DebPkg Type = "deb" + EggPkg Type = "egg" + // PacmanPkg Type = "pacman" + RpmPkg Type = "rpm" + WheelPkg Type = "wheel" + PoetryPkg Type = "poetry" + NpmPkg Type = "npm" + YarnPkg Type = "yarn" + PythonRequirementsPkg Type = "python-requirements" + JavaPkg Type = "java-archive" + JenkinsPluginPkg Type = "jenkins-plugin" + GoModulePkg Type = "go-module" ) -type Type uint - -var typeStr = []string{ - "UnknownPackage", - "apk", - "bundle", - "deb", - "egg", - //"pacman", - "rpm", - "wheel", - "npm", - "yarn", - "python-requirements", - "java-archive", - "jenkins-plugin", - "go-module", -} - var AllPkgs = []Type{ ApkPkg, BundlerPkg, DebPkg, EggPkg, - //PacmanPkg, + // PacmanPkg, RpmPkg, WheelPkg, NpmPkg, @@ -51,10 +35,3 @@ var AllPkgs = []Type{ JenkinsPluginPkg, GoModulePkg, } - -func (t Type) String() string { - if int(t) >= len(typeStr) { - return typeStr[0] - } - return typeStr[t] -} diff --git a/syft/presenter/json/presenter.go b/syft/presenter/json/presenter.go index 9f327670a..d703f95c2 100644 --- a/syft/presenter/json/presenter.go +++ b/syft/presenter/json/presenter.go @@ -94,7 +94,7 @@ func (pres *Presenter) Present(output io.Writer) error { art := artifact{ Name: p.Name, Version: p.Version, - Type: p.Type.String(), + Type: string(p.Type), Sources: make([]source, len(p.Source)), Metadata: p.Metadata, } diff --git a/syft/presenter/table/presenter.go b/syft/presenter/table/presenter.go index 0399da60d..be6be4112 100644 --- a/syft/presenter/table/presenter.go +++ b/syft/presenter/table/presenter.go @@ -31,7 +31,7 @@ func (pres *Presenter) Present(output io.Writer) error { row := []string{ p.Name, p.Version, - p.Type.String(), + string(p.Type), } rows = append(rows, row) } diff --git a/syft/presenter/text/presenter.go b/syft/presenter/text/presenter.go index 3fd75e600..2376f1aac 100644 --- a/syft/presenter/text/presenter.go +++ b/syft/presenter/text/presenter.go @@ -52,7 +52,7 @@ func (pres *Presenter) Present(output io.Writer) error { for _, p := range pres.catalog.Sorted() { fmt.Fprintln(w, fmt.Sprintf("[%s]", p.Name)) fmt.Fprintln(w, " Version:\t", p.Version) - fmt.Fprintln(w, " Type:\t", p.Type.String()) + fmt.Fprintln(w, " Type:\t", string(p.Type)) fmt.Fprintln(w, " Found by:\t", p.FoundBy) fmt.Fprintln(w) w.Flush() diff --git a/test/integration/pkg_cases.go b/test/integration/pkg_cases.go index 0ea5adc4c..e5a329606 100644 --- a/test/integration/pkg_cases.go +++ b/test/integration/pkg_cases.go @@ -31,6 +31,7 @@ var cases = []struct { pkgInfo: map[string]string{ "example-java-app-maven": "0.1.0", "example-jenkins-plugin": "1.0-SNAPSHOT", // the jenkins HPI file has a nested JAR of the same name + "joda-time": "2.9.2", }, }, { diff --git a/test/integration/pkg_coverage_test.go b/test/integration/pkg_coverage_test.go index 5b3a08197..da6f4bc7a 100644 --- a/test/integration/pkg_coverage_test.go +++ b/test/integration/pkg_coverage_test.go @@ -34,7 +34,7 @@ func TestPkgCoverageImage(t *testing.T) { observedPkgs := internal.NewStringSet() definedPkgs := internal.NewStringSet() for _, p := range pkg.AllPkgs { - definedPkgs.Add(p.String()) + definedPkgs.Add(string(p)) } for _, c := range cases { @@ -44,7 +44,7 @@ func TestPkgCoverageImage(t *testing.T) { for a := range catalog.Enumerate(c.pkgType) { observedLanguages.Add(a.Language.String()) - observedPkgs.Add(a.Type.String()) + observedPkgs.Add(string(a.Type)) expectedVersion, ok := c.pkgInfo[a.Name] if !ok { @@ -77,8 +77,8 @@ func TestPkgCoverageImage(t *testing.T) { observedLanguages.Remove(pkg.UnknownLanguage.String()) definedLanguages.Remove(pkg.UnknownLanguage.String()) - observedPkgs.Remove(pkg.UnknownPkg.String()) - definedPkgs.Remove(pkg.UnknownPkg.String()) + observedPkgs.Remove(string(pkg.UnknownPkg)) + definedPkgs.Remove(string(pkg.UnknownPkg)) // ensure that integration test cases stay in sync with the available catalogers if len(observedLanguages) < len(definedLanguages) { @@ -106,7 +106,7 @@ func TestPkgCoverageDirectory(t *testing.T) { observedPkgs := internal.NewStringSet() definedPkgs := internal.NewStringSet() for _, p := range pkg.AllPkgs { - definedPkgs.Add(p.String()) + definedPkgs.Add(string(p)) } for _, c := range cases { @@ -116,7 +116,7 @@ func TestPkgCoverageDirectory(t *testing.T) { for a := range catalog.Enumerate(c.pkgType) { observedLanguages.Add(a.Language.String()) - observedPkgs.Add(a.Type.String()) + observedPkgs.Add(string(a.Type)) expectedVersion, ok := c.pkgInfo[a.Name] if !ok { @@ -149,8 +149,8 @@ func TestPkgCoverageDirectory(t *testing.T) { observedLanguages.Remove(pkg.UnknownLanguage.String()) definedLanguages.Remove(pkg.UnknownLanguage.String()) - observedPkgs.Remove(pkg.UnknownPkg.String()) - definedPkgs.Remove(pkg.UnknownPkg.String()) + observedPkgs.Remove(string(pkg.UnknownPkg)) + definedPkgs.Remove(string(pkg.UnknownPkg)) // ensure that integration test cases stay in sync with the available catalogers if len(observedLanguages) < len(definedLanguages) {