Merge pull request #92 from anchore/issue-58

catalog Python packages from a requirements.txt file

imgbom/cataloger/python/cataloger.go

@@ -15,6 +15,7 @@ func NewCataloger() *Cataloger {
 	globParsers := map[string]common.ParserFn{
 		"**/egg-info/PKG-INFO":  parseEggMetadata,
 		"**/dist-info/METADATA": parseWheelMetadata,
+		"**/requirements.txt":   parseRequirementsTxt,
 	}
 
 	return &Cataloger{

imgbom/cataloger/python/parse_requirements.go

@@ -0,0 +1,69 @@
+package python
+
+import (
+	"bufio"
+	"fmt"
+	"io"
+	"strings"
+
+	"github.com/anchore/imgbom/imgbom/pkg"
+)
+
+func parseRequirementsTxt(_ string, reader io.Reader) ([]pkg.Package, error) {
+	packages := make([]pkg.Package, 0)
+
+	scanner := bufio.NewScanner(reader)
+	for scanner.Scan() {
+		line := scanner.Text()
+
+		line = strings.TrimRight(line, "\n")
+
+		switch {
+		case strings.HasPrefix(line, "#"):
+			// commented out line, skip
+			continue
+		case strings.HasPrefix(line, "-e"):
+			// editable packages aren't parsed (yet)
+			continue
+		case len(strings.Split(line, "==")) < 2:
+			// a package without a version, or a range (unpinned) which
+			// does not tell us exactly what will be installed
+			// XXX only needed if we want to log this, otherwise the next case catches it
+			continue
+		case len(strings.Split(line, "==")) == 2:
+			// remove comments if present
+			uncommented := removeTrailingComment(line)
+			// parse a new requirement
+			parts := strings.Split(uncommented, "==")
+			name := strings.TrimSpace(parts[0])
+			version := strings.TrimSpace(parts[1])
+			packages = append(packages, pkg.Package{
+				Name:     name,
+				Version:  version,
+				Language: pkg.Python,
+				Type:     pkg.PythonRequirementsPkg,
+			})
+		default:
+			continue
+		}
+	}
+
+	if err := scanner.Err(); err != nil {
+		return nil, fmt.Errorf("failed to parse python requirements file: %w", err)
+	}
+
+	return packages, nil
+}
+
+func removeTrailingComment(line string) string {
+	parts := strings.Split(line, "#")
+	switch len(parts) {
+	case 1:
+		// there aren't any comments
+		return line
+	default:
+		// any number of "#" means we only want the first part, assuming this
+		// isn't prefixed with "#" (up to the caller)
+		return parts[0]
+	}
+}

imgbom/cataloger/python/parse_requirements_test.go

@@ -0,0 +1,39 @@
+package python
+
+import (
+	"os"
+	"testing"
+
+	"github.com/anchore/imgbom/imgbom/pkg"
+)
+
+func TestParseRequirementsTxt(t *testing.T) {
+	expected := map[string]pkg.Package{
+		"foo": {
+			Name:     "foo",
+			Version:  "1.0.0",
+			Language: pkg.Python,
+			Type:     pkg.PythonRequirementsPkg,
+			Licenses: []string{},
+		},
+		"flask": {
+			Name:     "flask",
+			Version:  "4.0.0",
+			Language: pkg.Python,
+			Type:     pkg.PythonRequirementsPkg,
+			Licenses: []string{},
+		},
+	}
+	fixture, err := os.Open("test-fixtures/requires/requirements.txt")
+	if err != nil {
+		t.Fatalf("failed to open fixture: %+v", err)
+	}
+
+	actual, err := parseRequirementsTxt(fixture.Name(), fixture)
+	if err != nil {
+		t.Fatalf("failed to parse requirements: %+v", err)
+	}
+
+	assertPkgsEqual(t, actual, expected)
+
+}

imgbom/cataloger/python/parse_wheel_egg_test.go

@@ -9,11 +9,11 @@ import (
 
 func assertPkgsEqual(t *testing.T, actual []pkg.Package, expected map[string]pkg.Package) {
 	t.Helper()
-	if len(actual) != 1 {
+	if len(actual) != len(expected) {
 		for _, a := range actual {
 			t.Log("   ", a)
 		}
-		t.Fatalf("unexpected package count: %d!=%d", len(actual), 1)
+		t.Fatalf("unexpected package count: %d!=%d", len(actual), len(expected))
 	}
 
 	for _, a := range actual {
@@ -34,11 +34,14 @@ func assertPkgsEqual(t *testing.T, actual []pkg.Package, expected map[string]pkg
 			t.Errorf("bad package type: %+v", a.Type)
 		}
 
-		if len(a.Licenses) < 1 {
+		if len(a.Licenses) < len(expectedPkg.Licenses) {
 			t.Errorf("bad package licenses count: '%+v'", a.Licenses)
-		} else if a.Licenses[0] != expectedPkg.Licenses[0] {
+		}
+		if len(a.Licenses) > 0 {
+			if a.Licenses[0] != expectedPkg.Licenses[0] {
 				t.Errorf("bad package licenses: '%+v'", a.Licenses)
 			}
+		}
 
 	}
 }

imgbom/cataloger/python/test-fixtures/requires/requirements.txt

@@ -0,0 +1,7 @@
+ flask == 4.0.0
+# a line that is ignored
+sqlalchemy >= 1.0.0
+ foo == 1.0.0 # a comment that needs to be ignored
+-e https://github.com/pecan/pecan.git
+-r other-requirements.txt
+--requirements super-secretrequirements.txt

imgbom/pkg/type.go

@@ -9,6 +9,7 @@ const (
 	//PacmanPkg
 	RpmPkg
 	WheelPkg
+	PythonRequirementsPkg
 	JavaPkg
 	JenkinsPluginPkg
 )
@@ -24,6 +25,7 @@ var typeStr = []string{
 	//"pacman",
 	"rpm",
 	"wheel",
+	"python-requirements",
 	"java-archive",
 	"jenkins-plugin",
 }
@@ -36,6 +38,7 @@ var AllPkgs = []Type{
 	//PacmanPkg,
 	RpmPkg,
 	WheelPkg,
+	PythonRequirementsPkg,
 	JavaPkg,
 	JenkinsPluginPkg,
 }

integration/fixture_pkg_coverage_test.go

@@ -67,6 +67,14 @@ var cases = []struct {
 			"requests": "2.22.0",
 		},
 	},
+	{
+		name:        "find python packages",
+		pkgType:     pkg.PythonRequirementsPkg,
+		pkgLanguage: pkg.Python,
+		pkgInfo: map[string]string{
+			"flask": "4.0.0",
+		},
+	},
 	{
 		name:        "find bundler packages",
 		pkgType:     pkg.BundlerPkg,

integration/test-fixtures/image-pkg-coverage/python/requires/requirements.txt

@@ -0,0 +1,3 @@
+flask==4.0.0
+# this is an ignored line
+