From 27c207bbcaee3cafb738ddd8e398e10030488d97 Mon Sep 17 00:00:00 2001 From: Alfredo Deza Date: Mon, 17 Aug 2020 10:03:46 -0400 Subject: [PATCH 1/3] pkg: add a new type to identify setup.py packages Signed-off-by: Alfredo Deza --- syft/pkg/type.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/syft/pkg/type.go b/syft/pkg/type.go index e57be7836..3c43d0613 100644 --- a/syft/pkg/type.go +++ b/syft/pkg/type.go @@ -16,6 +16,7 @@ const ( NpmPkg Type = "npm" YarnPkg Type = "yarn" PythonRequirementsPkg Type = "python-requirements" + PythonSetupPkg Type = "python-setup" JavaPkg Type = "java-archive" JenkinsPluginPkg Type = "jenkins-plugin" GoModulePkg Type = "go-module" @@ -32,6 +33,7 @@ var AllPkgs = []Type{ NpmPkg, YarnPkg, PythonRequirementsPkg, + PythonSetupPkg, JavaPkg, JenkinsPluginPkg, GoModulePkg, From 9c4024d6fabae670598465371e773be4d05da4f4 Mon Sep 17 00:00:00 2001 From: Alfredo Deza Date: Mon, 17 Aug 2020 10:05:18 -0400 Subject: [PATCH 2/3] cataloger: add setup.py support for Python Signed-off-by: Alfredo Deza --- syft/cataloger/python/cataloger.go | 1 + syft/cataloger/python/parse_setup.go | 50 ++++++++++++++++ syft/cataloger/python/parse_setup_test.go | 60 +++++++++++++++++++ .../python/test-fixtures/setup/setup.py | 46 ++++++++++++++ 4 files changed, 157 insertions(+) create mode 100644 syft/cataloger/python/parse_setup.go create mode 100644 syft/cataloger/python/parse_setup_test.go create mode 100644 syft/cataloger/python/test-fixtures/setup/setup.py diff --git a/syft/cataloger/python/cataloger.go b/syft/cataloger/python/cataloger.go index 133a90fe0..396d7f28a 100644 --- a/syft/cataloger/python/cataloger.go +++ b/syft/cataloger/python/cataloger.go @@ -22,6 +22,7 @@ func New() *Cataloger { "**/*dist-info/METADATA": parseWheelMetadata, "**/requirements.txt": parseRequirementsTxt, "**/poetry.lock": parsePoetryLock, + "**/setup.py": parseSetup, } return &Cataloger{ diff --git a/syft/cataloger/python/parse_setup.go b/syft/cataloger/python/parse_setup.go new file mode 100644 index 000000000..7851ccb81 --- /dev/null +++ b/syft/cataloger/python/parse_setup.go @@ -0,0 +1,50 @@ +package python + +import ( + "bufio" + "io" + "regexp" + "strings" + + "github.com/anchore/syft/syft/cataloger/common" + "github.com/anchore/syft/syft/pkg" +) + +// integrity check +var _ common.ParserFn = parseSetup + +// match examples: +// 'pathlib3==2.2.0;python_version<"3.6"' --> match(name=pathlib3 version=2.2.0) +// "mypy==v0.770", --> match(name=mypy version=v0.770) +// " mypy2 == v0.770", ' mypy3== v0.770', --> match(name=mypy2 version=v0.770), match(name=mypy3, version=v0.770) +var pinnedDependency = regexp.MustCompile(`['"]\W?(\w+\W?==\W?[\w\.]*)`) + +func parseSetup(_ string, reader io.Reader) ([]pkg.Package, error) { + packages := make([]pkg.Package, 0) + + scanner := bufio.NewScanner(reader) + + for scanner.Scan() { + line := scanner.Text() + line = strings.TrimRight(line, "\n") + + for _, match := range pinnedDependency.FindAllString(line, -1) { + parts := strings.Split(match, "==") + if len(parts) != 2 { + continue + } + name := strings.Trim(parts[0], "'\"") + name = strings.TrimSpace(name) + + version := strings.TrimSpace(parts[len(parts)-1]) + packages = append(packages, pkg.Package{ + Name: strings.Trim(name, "'\""), + Version: strings.Trim(version, "'\""), + Language: pkg.Python, + Type: pkg.PythonSetupPkg, + }) + } + } + + return packages, nil +} diff --git a/syft/cataloger/python/parse_setup_test.go b/syft/cataloger/python/parse_setup_test.go new file mode 100644 index 000000000..6ac911f8c --- /dev/null +++ b/syft/cataloger/python/parse_setup_test.go @@ -0,0 +1,60 @@ +package python + +import ( + "os" + "testing" + + "github.com/anchore/syft/syft/pkg" +) + +func TestParseSetup(t *testing.T) { + expected := map[string]pkg.Package{ + "pathlib3": { + Name: "pathlib3", + Version: "2.2.0", + Language: pkg.Python, + Type: pkg.PythonSetupPkg, + Licenses: []string{}, + }, + "mypy": { + Name: "mypy", + Version: "v0.770", + Language: pkg.Python, + Type: pkg.PythonSetupPkg, + Licenses: []string{}, + }, + "mypy1": { + Name: "mypy1", + Version: "v0.770", + Language: pkg.Python, + Type: pkg.PythonSetupPkg, + Licenses: []string{}, + }, + "mypy2": { + Name: "mypy2", + Version: "v0.770", + Language: pkg.Python, + Type: pkg.PythonSetupPkg, + Licenses: []string{}, + }, + "mypy3": { + Name: "mypy3", + Version: "v0.770", + Language: pkg.Python, + Type: pkg.PythonSetupPkg, + Licenses: []string{}, + }, + } + fixture, err := os.Open("test-fixtures/setup/setup.py") + if err != nil { + t.Fatalf("failed to open fixture: %+v", err) + } + + actual, err := parseSetup(fixture.Name(), fixture) + if err != nil { + t.Fatalf("failed to parse requirements: %+v", err) + } + + assertPkgsEqual(t, actual, expected) + +} diff --git a/syft/cataloger/python/test-fixtures/setup/setup.py b/syft/cataloger/python/test-fixtures/setup/setup.py new file mode 100644 index 000000000..266c8c0ff --- /dev/null +++ b/syft/cataloger/python/test-fixtures/setup/setup.py @@ -0,0 +1,46 @@ +from setuptools import setup + +# Sample setup.py from the pytest project with added comments specific +# to the cataloger + +INSTALL_REQUIRES = [ + "py>=1.5.0", + "packaging", + "attrs>=17.4.0", + "more-itertools>=4.0.0", + 'atomicwrites>=1.0;sys_platform=="win32"', # sys_platform is ignored + 'pathlib2>=2.2.0;python_version=="3.6"', # python_version is ignored + 'pathlib3==2.2.0;python_version<"3.6"', # this is caught + 'colorama;sys_platform=="win32"', + "pluggy>=0.12,<1.0", + 'importlib-metadata>=0.12;python_version<"3.8"', + "wcwidth", +] + + +def main(): + setup( + use_scm_version={"write_to": "src/_pytest/_version.py"}, + setup_requires=["setuptools-scm", "setuptools>=40.0"], + package_dir={"": "src"}, + extras_require={ + "testing": [ + "argcomplete", + "hypothesis>=3.56", + "mock", + "nose", + "requests", + "xmlschema", + ], + "checkqa-mypy": [ + "mypy==v0.770", # this is caught + " mypy1==v0.770", # this is caught + " mypy2 == v0.770", ' mypy3== v0.770', # this is caught + ], + }, + install_requires=INSTALL_REQUIRES, + ) + + +if __name__ == "__main__": + main() From 861806f1dd812b5baf0f6387f358afb6b5e01f27 Mon Sep 17 00:00:00 2001 From: Alfredo Deza Date: Mon, 17 Aug 2020 10:07:39 -0400 Subject: [PATCH 3/3] test: add setup.py integration cases Signed-off-by: Alfredo Deza --- test/integration/pkg_cases.go | 10 ++++- .../image-pkg-coverage/python/setup/setup.py | 42 +++++++++++++++++++ 2 files changed, 51 insertions(+), 1 deletion(-) create mode 100644 test/integration/test-fixtures/image-pkg-coverage/python/setup/setup.py diff --git a/test/integration/pkg_cases.go b/test/integration/pkg_cases.go index eefc3f43f..35ce32a63 100644 --- a/test/integration/pkg_cases.go +++ b/test/integration/pkg_cases.go @@ -77,13 +77,21 @@ var cases = []struct { }, }, { - name: "find python packages", + name: "find python requirements.txt packages", pkgType: pkg.PythonRequirementsPkg, pkgLanguage: pkg.Python, pkgInfo: map[string]string{ "flask": "4.0.0", }, }, + { + name: "find python setup.py packages", + pkgType: pkg.PythonSetupPkg, + pkgLanguage: pkg.Python, + pkgInfo: map[string]string{ + "mypy": "v0.770", + }, + }, { name: "find bundler packages", pkgType: pkg.BundlerPkg, diff --git a/test/integration/test-fixtures/image-pkg-coverage/python/setup/setup.py b/test/integration/test-fixtures/image-pkg-coverage/python/setup/setup.py new file mode 100644 index 000000000..6ebfd67fb --- /dev/null +++ b/test/integration/test-fixtures/image-pkg-coverage/python/setup/setup.py @@ -0,0 +1,42 @@ +from setuptools import setup + +# TODO: if py gets upgrade to >=1.6, +# remove _width_of_current_line in terminal.py +INSTALL_REQUIRES = [ + "py>=1.5.0", + "packaging", + "attrs>=17.4.0", # should match oldattrs tox env. + "more-itertools>=4.0.0", + 'atomicwrites>=1.0;sys_platform=="win32"', + 'pathlib2>=2.2.0;python_version<"3.6"', + 'colorama;sys_platform=="win32"', + "pluggy>=0.12,<1.0", + 'importlib-metadata>=0.12;python_version<"3.8"', + "wcwidth", +] + + +def main(): + setup( + use_scm_version={"write_to": "src/_pytest/_version.py"}, + setup_requires=["setuptools-scm", "setuptools>=40.0"], + package_dir={"": "src"}, + extras_require={ + "testing": [ + "argcomplete", + "hypothesis>=3.56", + "mock", + "nose", + "requests", + "xmlschema", + ], + "checkqa-mypy": [ + "mypy==v0.770", # keep this in sync with .pre-commit-config.yaml. + ], + }, + install_requires=INSTALL_REQUIRES, + ) + + +if __name__ == "__main__": + main()