diff --git a/imgbom/cataloger/python/cataloger.go b/imgbom/cataloger/python/cataloger.go index 6b0e6e5cd..81342ff5c 100644 --- a/imgbom/cataloger/python/cataloger.go +++ b/imgbom/cataloger/python/cataloger.go @@ -15,6 +15,7 @@ func NewCataloger() *Cataloger { globParsers := map[string]common.ParserFn{ "**/egg-info/PKG-INFO": parseEggMetadata, "**/dist-info/METADATA": parseWheelMetadata, + "**/requirements.txt": parseRequirementsTxt, } return &Cataloger{ diff --git a/imgbom/cataloger/python/parse_requirements.go b/imgbom/cataloger/python/parse_requirements.go new file mode 100644 index 000000000..decf25d88 --- /dev/null +++ b/imgbom/cataloger/python/parse_requirements.go @@ -0,0 +1,69 @@ +package python + +import ( + "bufio" + "fmt" + "io" + "strings" + + "github.com/anchore/imgbom/imgbom/pkg" +) + +func parseRequirementsTxt(_ string, reader io.Reader) ([]pkg.Package, error) { + packages := make([]pkg.Package, 0) + + scanner := bufio.NewScanner(reader) + for scanner.Scan() { + line := scanner.Text() + + line = strings.TrimRight(line, "\n") + + switch { + case strings.HasPrefix(line, "#"): + // commented out line, skip + continue + case strings.HasPrefix(line, "-e"): + // editable packages aren't parsed (yet) + continue + case len(strings.Split(line, "==")) < 2: + // a package without a version, or a range (unpinned) which + // does not tell us exactly what will be installed + // XXX only needed if we want to log this, otherwise the next case catches it + continue + case len(strings.Split(line, "==")) == 2: + // remove comments if present + uncommented := removeTrailingComment(line) + // parse a new requirement + parts := strings.Split(uncommented, "==") + name := strings.TrimSpace(parts[0]) + version := strings.TrimSpace(parts[1]) + packages = append(packages, pkg.Package{ + Name: name, + Version: version, + Language: pkg.Python, + Type: pkg.PythonRequirementsPkg, + }) + default: + continue + } + } + + if err := scanner.Err(); err != nil { + return nil, fmt.Errorf("failed to parse python requirements file: %w", err) + } + + return packages, nil +} + +func removeTrailingComment(line string) string { + parts := strings.Split(line, "#") + switch len(parts) { + case 1: + // there aren't any comments + return line + default: + // any number of "#" means we only want the first part, assuming this + // isn't prefixed with "#" (up to the caller) + return parts[0] + } +} diff --git a/imgbom/cataloger/python/parse_requirements_test.go b/imgbom/cataloger/python/parse_requirements_test.go new file mode 100644 index 000000000..2b980751d --- /dev/null +++ b/imgbom/cataloger/python/parse_requirements_test.go @@ -0,0 +1,39 @@ +package python + +import ( + "os" + "testing" + + "github.com/anchore/imgbom/imgbom/pkg" +) + +func TestParseRequirementsTxt(t *testing.T) { + expected := map[string]pkg.Package{ + "foo": { + Name: "foo", + Version: "1.0.0", + Language: pkg.Python, + Type: pkg.PythonRequirementsPkg, + Licenses: []string{}, + }, + "flask": { + Name: "flask", + Version: "4.0.0", + Language: pkg.Python, + Type: pkg.PythonRequirementsPkg, + Licenses: []string{}, + }, + } + fixture, err := os.Open("test-fixtures/requires/requirements.txt") + if err != nil { + t.Fatalf("failed to open fixture: %+v", err) + } + + actual, err := parseRequirementsTxt(fixture.Name(), fixture) + if err != nil { + t.Fatalf("failed to parse requirements: %+v", err) + } + + assertPkgsEqual(t, actual, expected) + +} diff --git a/imgbom/cataloger/python/parse_wheel_egg_test.go b/imgbom/cataloger/python/parse_wheel_egg_test.go index c58b27bf6..bd268c636 100644 --- a/imgbom/cataloger/python/parse_wheel_egg_test.go +++ b/imgbom/cataloger/python/parse_wheel_egg_test.go @@ -9,11 +9,11 @@ import ( func assertPkgsEqual(t *testing.T, actual []pkg.Package, expected map[string]pkg.Package) { t.Helper() - if len(actual) != 1 { + if len(actual) != len(expected) { for _, a := range actual { t.Log(" ", a) } - t.Fatalf("unexpected package count: %d!=%d", len(actual), 1) + t.Fatalf("unexpected package count: %d!=%d", len(actual), len(expected)) } for _, a := range actual { @@ -34,10 +34,13 @@ func assertPkgsEqual(t *testing.T, actual []pkg.Package, expected map[string]pkg t.Errorf("bad package type: %+v", a.Type) } - if len(a.Licenses) < 1 { + if len(a.Licenses) < len(expectedPkg.Licenses) { t.Errorf("bad package licenses count: '%+v'", a.Licenses) - } else if a.Licenses[0] != expectedPkg.Licenses[0] { - t.Errorf("bad package licenses: '%+v'", a.Licenses) + } + if len(a.Licenses) > 0 { + if a.Licenses[0] != expectedPkg.Licenses[0] { + t.Errorf("bad package licenses: '%+v'", a.Licenses) + } } } diff --git a/imgbom/cataloger/python/test-fixtures/requires/requirements.txt b/imgbom/cataloger/python/test-fixtures/requires/requirements.txt new file mode 100644 index 000000000..8b115d3af --- /dev/null +++ b/imgbom/cataloger/python/test-fixtures/requires/requirements.txt @@ -0,0 +1,7 @@ + flask == 4.0.0 +# a line that is ignored +sqlalchemy >= 1.0.0 + foo == 1.0.0 # a comment that needs to be ignored +-e https://github.com/pecan/pecan.git +-r other-requirements.txt +--requirements super-secretrequirements.txt \ No newline at end of file diff --git a/imgbom/pkg/type.go b/imgbom/pkg/type.go index 47be98ebd..c4b745b03 100644 --- a/imgbom/pkg/type.go +++ b/imgbom/pkg/type.go @@ -9,6 +9,7 @@ const ( //PacmanPkg RpmPkg WheelPkg + PythonRequirementsPkg JavaPkg JenkinsPluginPkg ) @@ -24,6 +25,7 @@ var typeStr = []string{ //"pacman", "rpm", "wheel", + "python-requirements", "java-archive", "jenkins-plugin", } @@ -36,6 +38,7 @@ var AllPkgs = []Type{ //PacmanPkg, RpmPkg, WheelPkg, + PythonRequirementsPkg, JavaPkg, JenkinsPluginPkg, } diff --git a/integration/fixture_pkg_coverage_test.go b/integration/fixture_pkg_coverage_test.go index 04a992b81..60ffc09f9 100644 --- a/integration/fixture_pkg_coverage_test.go +++ b/integration/fixture_pkg_coverage_test.go @@ -67,6 +67,14 @@ var cases = []struct { "requests": "2.22.0", }, }, + { + name: "find python packages", + pkgType: pkg.PythonRequirementsPkg, + pkgLanguage: pkg.Python, + pkgInfo: map[string]string{ + "flask": "4.0.0", + }, + }, { name: "find bundler packages", pkgType: pkg.BundlerPkg, diff --git a/integration/test-fixtures/image-pkg-coverage/python/requires/requirements.txt b/integration/test-fixtures/image-pkg-coverage/python/requires/requirements.txt new file mode 100644 index 000000000..cb93b3b55 --- /dev/null +++ b/integration/test-fixtures/image-pkg-coverage/python/requires/requirements.txt @@ -0,0 +1,3 @@ +flask==4.0.0 +# this is an ignored line +