oss/syft
2
0
Fork 0
mirror of https://github.com/anchore/syft.git synced 2025-11-17 16:33:21 +01:00
Code Issues Packages Projects Releases Wiki Activity

Add poetry cataloger (#121)

Browse Source 
* Minor cleanup

Signed-off-by: Dan Luhring <dan.luhring@anchore.com>

* Update pkg Type definition to string

Signed-off-by: Dan Luhring <dan.luhring@anchore.com>

* Implement poetry.lock parsing

Signed-off-by: Dan Luhring <dan.luhring@anchore.com>

* Address CI issues

Signed-off-by: Dan Luhring <dan.luhring@anchore.com>

* Integrate Alex's changes

Signed-off-by: Dan Luhring <dan.luhring@anchore.com>
This commit is contained in:
Dan Luhring 2020-08-04 18:22:43 -04:00 committed by GitHub
parent e2a874a277
commit 70e673204c
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
20 changed files with 225 additions and 61 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
Makefile
View File

@ -73,9 +73,9 @@ help:
ci-bootstrap: bootstrap
sudo apt install -y bc
.PHONY: boostrap
.PHONY: bootstrap
bootstrap: ## Download and install all go dependencies (+ prep tooling in the ./tmp dir)
$(call title,Boostrapping dependencies)
$(call title,Bootstrapping dependencies)
@pwd
# prep temp dirs
mkdir -p $(TEMPDIR)

2
cmd/root.go
View File

@ -19,7 +19,7 @@ import (
var rootCmd = &cobra.Command{
Use: fmt.Sprintf("%s [SOURCE]", internal.ApplicationName),
Short: "A tool for generating a Software Bill Of Materials (SBOM) from container images and filesystems",
Long: internal.Tprintf(`\
Long: internal.Tprintf(`
Supports the following image sources:
{{.appName}} yourrepo/yourimage:tag defaults to using images from a docker daemon
{{.appName}} docker://yourrepo/yourimage:tag explicitly use the docker daemon

1
go.mod
View File

@ -19,6 +19,7 @@ require (
github.com/mitchellh/go-homedir v1.1.0
github.com/mitchellh/mapstructure v1.3.1
github.com/olekukonko/tablewriter v0.0.4
github.com/pelletier/go-toml v1.8.0
github.com/rogpeppe/go-internal v1.5.2
github.com/sergi/go-diff v1.1.0
github.com/sirupsen/logrus v1.6.0

4
json-schema/schema.json
View File

@ -347,7 +347,7 @@
"type": "null"
},
"type": {
"type": "integer"
"type": "string"
},
"version": {
"type": "string"
@ -520,4 +520,4 @@
"artifacts"
],
"type": "object"
}
}

2
syft/cataloger/java/archive_parser.go
View File

@ -133,6 +133,7 @@ func (j *archiveParser) discoverMainPackage() (*pkg.Package, error) {
Name: selectName(manifest, j.fileInfo),
Version: selectVersion(manifest, j.fileInfo),
Language: pkg.Java,
Type: pkg.JavaPkg,
Metadata: pkg.JavaMetadata{
Manifest: manifest,
},
@ -165,6 +166,7 @@ func (j *archiveParser) discoverPkgsFromPomProperties(parentPkg *pkg.Package) ([
Name: propsObj.ArtifactID,
Version: propsObj.Version,
Language: pkg.Java,
Type: pkg.JavaPkg,
Metadata: pkg.JavaMetadata{
PomProperties: propsObj,
Parent: parentPkg,

2
syft/cataloger/java/archive_parser_test.go
View File

@ -227,7 +227,7 @@ func TestParseJar(t *testing.T) {
Name: "joda-time",
Version: "2.9.2",
Language: pkg.Java,
Type: pkg.UnknownPkg,
Type: pkg.JavaPkg,
Metadata: pkg.JavaMetadata{
PomProperties: &pkg.PomProperties{
Path: "META-INF/maven/joda-time/joda-time/pom.properties",

1
syft/cataloger/python/cataloger.go
View File

@ -19,6 +19,7 @@ func NewCataloger() *Cataloger {
"**/*egg-info/PKG-INFO": parseEggMetadata,
"**/*dist-info/METADATA": parseWheelMetadata,
"**/requirements.txt": parseRequirementsTxt,
"**/poetry.lock": parsePoetryLock,
}
return &Cataloger{

24
syft/cataloger/python/parse_poetry_lock.go Normal file
View File

@ -0,0 +1,24 @@
package python
import (
"fmt"
"io"
"github.com/anchore/syft/syft/pkg"
"github.com/pelletier/go-toml"
)
func parsePoetryLock(_ string, reader io.Reader) ([]pkg.Package, error) {
tree, err := toml.LoadReader(reader)
if err != nil {
return nil, fmt.Errorf("unable to load poetry.lock for parsing: %v", err)
}
metadata := PoetryMetadata{}
err = tree.Unmarshal(&metadata)
if err != nil {
return nil, fmt.Errorf("unable to parse poetry.lock: %v", err)
}
return metadata.Pkgs(), nil
}

56
syft/cataloger/python/parse_poetry_lock_test.go Normal file
View File

@ -0,0 +1,56 @@
package python
import (
"github.com/anchore/syft/syft/pkg"
"github.com/go-test/deep"
"os"
"testing"
)
func TestParsePoetryLock(t *testing.T) {
expected := []pkg.Package{
{
Name: "added-value",
Version: "0.14.2",
Language: pkg.Python,
Type: pkg.PoetryPkg,
Licenses: nil,
},
{
Name: "alabaster",
Version: "0.7.12",
Language: pkg.Python,
Type: pkg.PoetryPkg,
Licenses: nil,
},
{
Name: "appnope",
Version: "0.1.0",
Language: pkg.Python,
Type: pkg.PoetryPkg,
Licenses: nil,
},
{
Name: "asciitree",
Version: "0.3.3",
Language: pkg.Python,
Type: pkg.PoetryPkg,
Licenses: nil,
},
}
fixture, err := os.Open("test-fixtures/poetry/poetry.lock")
if err != nil {
t.Fatalf("failed to open fixture: %+v", err)
}
actual, err := parsePoetryLock(fixture.Name(), fixture)
if err != nil {
t.Error(err)
}
differences := deep.Equal(expected, actual)
if differences != nil {
t.Errorf("returned package list differed from expectation: %+v", differences)
}
}

3
syft/cataloger/python/parse_wheel_egg_test.go
View File

@ -45,6 +45,7 @@ func assertPkgsEqual(t *testing.T, actual []pkg.Package, expected map[string]pkg
}
}
func TestParseEggMetadata(t *testing.T) {
expected := map[string]pkg.Package{
"requests": {
@ -66,7 +67,6 @@ func TestParseEggMetadata(t *testing.T) {
}
assertPkgsEqual(t, actual, expected)
}
func TestParseWheelMetadata(t *testing.T) {
@ -90,5 +90,4 @@ func TestParseWheelMetadata(t *testing.T) {
}
assertPkgsEqual(t, actual, expected)
}

18
syft/cataloger/python/poetry_metadata.go Normal file
View File

@ -0,0 +1,18 @@
package python
import "github.com/anchore/syft/syft/pkg"
type PoetryMetadata struct {
Packages []PoetryMetadataPackage `toml:"package"`
}
// Pkgs returns all of the packages referenced within the poetry.lock metadata.
func (m PoetryMetadata) Pkgs() []pkg.Package {
pkgs := make([]pkg.Package, 0)
for _, p := range m.Packages {
pkgs = append(pkgs, p.Pkg())
}
return pkgs
}

21
syft/cataloger/python/poetry_metadata_package.go Normal file
View File

@ -0,0 +1,21 @@
package python
import "github.com/anchore/syft/syft/pkg"
type PoetryMetadataPackage struct {
Name string `toml:"name"`
Version string `toml:"version"`
Category string `toml:"category"`
Description string `toml:"description"`
Optional bool `toml:"optional"`
}
// Pkg returns the standard `pkg.Package` representation of the package referenced within the poetry.lock metadata.
func (p PoetryMetadataPackage) Pkg() pkg.Package {
return pkg.Package{
Name: p.Name,
Version: p.Version,
Language: pkg.Python,
Type: pkg.PoetryPkg,
}
}

64
syft/cataloger/python/test-fixtures/poetry/poetry.lock generated Normal file
View File

@ -0,0 +1,64 @@
[[package]]
category = "dev"
description = "Sphinx \"added-value\" extension"
name = "added-value"
optional = false
python-versions = "*"
version = "0.14.2"
[package.dependencies]
docutils = "*"
natsort = "*"
six = "*"
sphinx = "*"
[package.extras]
deploy = ["bumpversion", "twine", "wheel"]
docs = ["sphinx", "sphinx-rtd-theme"]
test = ["pytest", "pytest-cov", "coveralls", "beautifulsoup4", "hypothesis"]
[[package]]
category = "dev"
description = "A configurable sidebar-enabled Sphinx theme"
name = "alabaster"
optional = false
python-versions = "*"
version = "0.7.12"
[[package]]
category = "dev"
description = "Disable App Nap on OS X 10.9"
marker = "python_version >= \"3.4\" and sys_platform == \"darwin\" or sys_platform == \"darwin\""
name = "appnope"
optional = false
python-versions = "*"
version = "0.1.0"
[[package]]
category = "dev"
description = "Draws ASCII trees."
name = "asciitree"
optional = false
python-versions = "*"
version = "0.3.3"
[metadata]
content-hash = "6e35f765c2f01c635c2fb0d54a9d7dd68742350f04449ee24efad03e3c9eb0bb"
python-versions = "^3.6"
[metadata.files]
added-value = [
{file = "added-value-0.14.2.tar.gz", hash = "sha256:8c886aee74635cec15beb64c28a90ae664526f331c1a4941e4d6ab98af232028"},
{file = "added_value-0.14.2-py2.py3-none-any.whl", hash = "sha256:b25fcb86f9bfad9a40adf4d344322690e312741556c7b75681bc948380a251e6"},
]
alabaster = [
{file = "alabaster-0.7.12-py2.py3-none-any.whl", hash = "sha256:446438bdcca0e05bd45ea2de1668c1d9b032e1a9154c2c259092d77031ddd359"},
{file = "alabaster-0.7.12.tar.gz", hash = "sha256:a661d72d58e6ea8a57f7a86e37d86716863ee5e92788398526d58b26a4e4dc02"},
]
appnope = [
{file = "appnope-0.1.0-py2.py3-none-any.whl", hash = "sha256:5b26757dc6f79a3b7dc9fab95359328d5747fcb2409d331ea66d0272b90ab2a0"},
{file = "appnope-0.1.0.tar.gz", hash = "sha256:8b995ffe925347a2138d7ac0fe77155e4311a0ea6d6da4f5128fe4b3cbe5ed71"},
]
asciitree = [
{file = "asciitree-0.3.3.tar.gz", hash = "sha256:4aa4b9b649f85e3fcb343363d97564aa1fb62e249677f2e18a96765145cc0f6e"},
]

2
syft/pkg/package.go
View File

@ -15,7 +15,7 @@ type Package struct {
id ID // this is set when a package is added to the catalog
Name string `json:"manifest"`
Version string `json:"version"`
FoundBy string `json:"found-by"`
FoundBy string `json:"found-by"` // FoundBy is the cataloger that discovered this package
Source []file.Reference `json:"sources"`
Licenses []string `json:"licenses"` // TODO: should we move this into metadata?
Language Language `json:"language"` // TODO: should this support multiple languages as a slice?

59
syft/pkg/type.go
View File

@ -1,47 +1,31 @@
package pkg
type Type string
const (
UnknownPkg Type = iota
ApkPkg
BundlerPkg
DebPkg
EggPkg
//PacmanPkg
RpmPkg
WheelPkg
NpmPkg
YarnPkg
PythonRequirementsPkg
JavaPkg
JenkinsPluginPkg
GoModulePkg
UnknownPkg Type = "UnknownPackage"
ApkPkg Type = "apk"
BundlerPkg Type = "bundle"
DebPkg Type = "deb"
EggPkg Type = "egg"
// PacmanPkg Type = "pacman"
RpmPkg Type = "rpm"
WheelPkg Type = "wheel"
PoetryPkg Type = "poetry"
NpmPkg Type = "npm"
YarnPkg Type = "yarn"
PythonRequirementsPkg Type = "python-requirements"
JavaPkg Type = "java-archive"
JenkinsPluginPkg Type = "jenkins-plugin"
GoModulePkg Type = "go-module"
)
type Type uint
var typeStr = []string{
"UnknownPackage",
"apk",
"bundle",
"deb",
"egg",
//"pacman",
"rpm",
"wheel",
"npm",
"yarn",
"python-requirements",
"java-archive",
"jenkins-plugin",
"go-module",
}
var AllPkgs = []Type{
ApkPkg,
BundlerPkg,
DebPkg,
EggPkg,
//PacmanPkg,
// PacmanPkg,
RpmPkg,
WheelPkg,
NpmPkg,
@ -51,10 +35,3 @@ var AllPkgs = []Type{
JenkinsPluginPkg,
GoModulePkg,
}
func (t Type) String() string {
if int(t) >= len(typeStr) {
return typeStr[0]
}
return typeStr[t]
}

2
syft/presenter/json/presenter.go
View File

@ -94,7 +94,7 @@ func (pres *Presenter) Present(output io.Writer) error {
art := artifact{
Name: p.Name,
Version: p.Version,
Type: p.Type.String(),
Type: string(p.Type),
Sources: make([]source, len(p.Source)),
Metadata: p.Metadata,
}

2
syft/presenter/table/presenter.go
View File

@ -31,7 +31,7 @@ func (pres *Presenter) Present(output io.Writer) error {
row := []string{
p.Name,
p.Version,
p.Type.String(),
string(p.Type),
}
rows = append(rows, row)
}

2
syft/presenter/text/presenter.go
View File

@ -52,7 +52,7 @@ func (pres *Presenter) Present(output io.Writer) error {
for _, p := range pres.catalog.Sorted() {
fmt.Fprintln(w, fmt.Sprintf("[%s]", p.Name))
fmt.Fprintln(w, " Version:\t", p.Version)
fmt.Fprintln(w, " Type:\t", p.Type.String())
fmt.Fprintln(w, " Type:\t", string(p.Type))
fmt.Fprintln(w, " Found by:\t", p.FoundBy)
fmt.Fprintln(w)
w.Flush()

1
test/integration/pkg_cases.go
View File

@ -31,6 +31,7 @@ var cases = []struct {
pkgInfo: map[string]string{
"example-java-app-maven": "0.1.0",
"example-jenkins-plugin": "1.0-SNAPSHOT", // the jenkins HPI file has a nested JAR of the same name
"joda-time": "2.9.2",
},
},
{

16
test/integration/pkg_coverage_test.go
View File

@ -34,7 +34,7 @@ func TestPkgCoverageImage(t *testing.T) {
observedPkgs := internal.NewStringSet()
definedPkgs := internal.NewStringSet()
for _, p := range pkg.AllPkgs {
definedPkgs.Add(p.String())
definedPkgs.Add(string(p))
}
for _, c := range cases {
@ -44,7 +44,7 @@ func TestPkgCoverageImage(t *testing.T) {
for a := range catalog.Enumerate(c.pkgType) {
observedLanguages.Add(a.Language.String())
observedPkgs.Add(a.Type.String())
observedPkgs.Add(string(a.Type))
expectedVersion, ok := c.pkgInfo[a.Name]
if !ok {
@ -77,8 +77,8 @@ func TestPkgCoverageImage(t *testing.T) {
observedLanguages.Remove(pkg.UnknownLanguage.String())
definedLanguages.Remove(pkg.UnknownLanguage.String())
observedPkgs.Remove(pkg.UnknownPkg.String())
definedPkgs.Remove(pkg.UnknownPkg.String())
observedPkgs.Remove(string(pkg.UnknownPkg))
definedPkgs.Remove(string(pkg.UnknownPkg))
// ensure that integration test cases stay in sync with the available catalogers
if len(observedLanguages) < len(definedLanguages) {
@ -106,7 +106,7 @@ func TestPkgCoverageDirectory(t *testing.T) {
observedPkgs := internal.NewStringSet()
definedPkgs := internal.NewStringSet()
for _, p := range pkg.AllPkgs {
definedPkgs.Add(p.String())
definedPkgs.Add(string(p))
}
for _, c := range cases {
@ -116,7 +116,7 @@ func TestPkgCoverageDirectory(t *testing.T) {
for a := range catalog.Enumerate(c.pkgType) {
observedLanguages.Add(a.Language.String())
observedPkgs.Add(a.Type.String())
observedPkgs.Add(string(a.Type))
expectedVersion, ok := c.pkgInfo[a.Name]
if !ok {
@ -149,8 +149,8 @@ func TestPkgCoverageDirectory(t *testing.T) {
observedLanguages.Remove(pkg.UnknownLanguage.String())
definedLanguages.Remove(pkg.UnknownLanguage.String())
observedPkgs.Remove(pkg.UnknownPkg.String())
definedPkgs.Remove(pkg.UnknownPkg.String())
observedPkgs.Remove(string(pkg.UnknownPkg))
definedPkgs.Remove(string(pkg.UnknownPkg))
// ensure that integration test cases stay in sync with the available catalogers
if len(observedLanguages) < len(definedLanguages) {