oss/syft
2
0
Fork 0
mirror of https://github.com/anchore/syft.git synced 2025-11-17 08:23:15 +01:00
Code Issues Packages Projects Releases Wiki Activity

Fix panic in requirements.txt parsing (#834)

Browse Source 
* Stable sort for pipfile.lock parsing

Signed-off-by: Dan Luhring <dan+github@luhrings.com>

* Adjust python parsing tests to use go-cmp

Signed-off-by: Dan Luhring <dan+github@luhrings.com>

* Add failing cases for requirements.txt parsing

Signed-off-by: Dan Luhring <dan+github@luhrings.com>

* Fix failing cases for requirements.txt parsing

Signed-off-by: Dan Luhring <dan+github@luhrings.com>

* Refactor parseRequirementsTxt

Signed-off-by: Dan Luhring <dan+github@luhrings.com>

* Fix static-analysis failure

Signed-off-by: Dan Luhring <dan+github@luhrings.com>

* Fix comment

Signed-off-by: Dan Luhring <dan+github@luhrings.com>
This commit is contained in:
Dan Luhring 2022-02-17 10:00:16 -05:00 committed by GitHub
parent 55c7f3d1e7
commit 641c44f449
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
6 changed files with 108 additions and 83 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
syft/pkg/cataloger/python/parse_pipfile_lock.go
View File

@ -4,6 +4,7 @@ import (
"encoding/json"
"fmt"
"io"
"sort"
"strings"
"github.com/anchore/syft/syft/artifact"
@ -60,5 +61,10 @@ func parsePipfileLock(_ string, reader io.Reader) ([]*pkg.Package, []artifact.Re
}
}
// Without sorting the packages slice, the order of packages will be unstable, due to ranging over a map.
sort.Slice(packages, func(i, j int) bool {
return packages[i].String() < packages[j].String()
})
return packages, nil, nil
}

18
syft/pkg/cataloger/python/parse_pipfile_lock_test.go
View File

@ -4,36 +4,39 @@ import (
"os"
"testing"
"github.com/google/go-cmp/cmp"
"github.com/anchore/syft/syft/pkg"
)
func TestParsePipFileLock(t *testing.T) {
expected := map[string]pkg.Package{
"aio-pika": {
expected := []*pkg.Package{
{
Name: "aio-pika",
Version: "6.8.0",
Language: pkg.Python,
Type: pkg.PythonPkg,
},
"aiodns": {
{
Name: "aiodns",
Version: "2.0.0",
Language: pkg.Python,
Type: pkg.PythonPkg,
},
"aiohttp": {
{
Name: "aiohttp",
Version: "3.7.4.post0",
Language: pkg.Python,
Type: pkg.PythonPkg,
},
"aiohttp-jinja2": {
{
Name: "aiohttp-jinja2",
Version: "1.4.2",
Language: pkg.Python,
Type: pkg.PythonPkg,
},
}
fixture, err := os.Open("test-fixtures/pipfile-lock/Pipfile.lock")
if err != nil {
t.Fatalf("failed to open fixture: %+v", err)
@ -45,6 +48,7 @@ func TestParsePipFileLock(t *testing.T) {
t.Fatalf("failed to parse requirements: %+v", err)
}
assertPackagesEqual(t, actual, expected)
if diff := cmp.Diff(expected, actual, cmp.AllowUnexported(pkg.Package{})); diff != "" {
t.Errorf("unexpected result from parsing (-expected +actual)\n%s", diff)
}
}

88
syft/pkg/cataloger/python/parse_requirements.go
View File

@ -22,37 +22,34 @@ func parseRequirementsTxt(_ string, reader io.Reader) ([]*pkg.Package, []artifac
scanner := bufio.NewScanner(reader)
for scanner.Scan() {
line := scanner.Text()
line = trimRequirementsTxtLine(line)
line = strings.TrimRight(line, "\n")
switch {
case strings.HasPrefix(line, "#"):
// commented out line, skip
continue
case strings.HasPrefix(line, "-e"):
// editable packages aren't parsed (yet)
continue
case len(strings.Split(line, "==")) < 2:
// a package without a version, or a range (unpinned) which
// does not tell us exactly what will be installed
// XXX only needed if we want to log this, otherwise the next case catches it
continue
case len(strings.Split(line, "==")) == 2:
// remove comments if present
uncommented := removeTrailingComment(line)
// parse a new requirement
parts := strings.Split(uncommented, "==")
name := strings.TrimSpace(parts[0])
version := strings.TrimSpace(parts[1])
packages = append(packages, &pkg.Package{
Name: name,
Version: version,
Language: pkg.Python,
Type: pkg.PythonPkg,
})
default:
if line == "" {
// nothing to parse on this line
continue
}
if strings.HasPrefix(line, "-e") {
// editable packages aren't parsed (yet)
continue
}
if !strings.Contains(line, "==") {
// a package without a version, or a range (unpinned) which does not tell us
// exactly what will be installed.
continue
}
// parse a new requirement
parts := strings.Split(line, "==")
name := strings.TrimSpace(parts[0])
version := strings.TrimSpace(parts[1])
packages = append(packages, &pkg.Package{
Name: name,
Version: version,
Language: pkg.Python,
Type: pkg.PythonPkg,
})
}
if err := scanner.Err(); err != nil {
@ -62,16 +59,37 @@ func parseRequirementsTxt(_ string, reader io.Reader) ([]*pkg.Package, []artifac
return packages, nil, nil
}
// trimRequirementsTxtLine removes content from the given requirements.txt line
// that should not be considered for parsing.
func trimRequirementsTxtLine(line string) string {
line = strings.TrimSpace(line)
line = removeTrailingComment(line)
line = removeEnvironmentMarkers(line)
return line
}
// removeTrailingComment takes a requirements.txt line and strips off comment strings.
func removeTrailingComment(line string) string {
parts := strings.Split(line, "#")
switch len(parts) {
case 1:
parts := strings.SplitN(line, "#", 2)
if len(parts) < 2 {
// there aren't any comments
return line
default:
// any number of "#" means we only want the first part, assuming this
// isn't prefixed with "#" (up to the caller)
return parts[0]
}
return parts[0]
}
// removeEnvironmentMarkers removes any instances of environment markers (delimited by ';') from the line.
// For more information, see https://www.python.org/dev/peps/pep-0508/#environment-markers.
func removeEnvironmentMarkers(line string) string {
parts := strings.SplitN(line, ";", 2)
if len(parts) < 2 {
// there aren't any environment markers
return line
}
return parts[0]
}

51
syft/pkg/cataloger/python/parse_requirements_test.go
View File

@ -4,47 +4,33 @@ import (
"os"
"testing"
"github.com/stretchr/testify/assert"
"github.com/go-test/deep"
"github.com/google/go-cmp/cmp"
"github.com/anchore/syft/syft/pkg"
)
func assertPackagesEqual(t *testing.T, actual []*pkg.Package, expected map[string]pkg.Package) {
t.Helper()
if len(actual) != len(expected) {
for _, a := range actual {
t.Log(" ", a)
}
t.Fatalf("unexpected package count: %d!=%d", len(actual), len(expected))
}
for _, a := range actual {
expectedPkg, ok := expected[a.Name]
assert.True(t, ok)
for _, d := range deep.Equal(a, &expectedPkg) {
t.Errorf("diff: %+v", d)
}
}
}
func TestParseRequirementsTxt(t *testing.T) {
expected := map[string]pkg.Package{
"foo": {
Name: "foo",
Version: "1.0.0",
Language: pkg.Python,
Type: pkg.PythonPkg,
},
"flask": {
expected := []*pkg.Package{
{
Name: "flask",
Version: "4.0.0",
Language: pkg.Python,
Type: pkg.PythonPkg,
},
{
Name: "foo",
Version: "1.0.0",
Language: pkg.Python,
Type: pkg.PythonPkg,
},
{
Name: "SomeProject",
Version: "5.4",
Language: pkg.Python,
Type: pkg.PythonPkg,
},
}
fixture, err := os.Open("test-fixtures/requires/requirements.txt")
if err != nil {
t.Fatalf("failed to open fixture: %+v", err)
@ -56,6 +42,7 @@ func TestParseRequirementsTxt(t *testing.T) {
t.Fatalf("failed to parse requirements: %+v", err)
}
assertPackagesEqual(t, actual, expected)
if diff := cmp.Diff(expected, actual, cmp.AllowUnexported(pkg.Package{})); diff != "" {
t.Errorf("unexpected result from parsing (-expected +actual)\n%s", diff)
}
}

21
syft/pkg/cataloger/python/parse_setup_test.go
View File

@ -4,52 +4,57 @@ import (
"os"
"testing"
"github.com/google/go-cmp/cmp"
"github.com/anchore/syft/syft/pkg"
)
func TestParseSetup(t *testing.T) {
expected := map[string]pkg.Package{
"pathlib3": {
expected := []*pkg.Package{
{
Name: "pathlib3",
Version: "2.2.0",
Language: pkg.Python,
Type: pkg.PythonPkg,
},
"mypy": {
{
Name: "mypy",
Version: "v0.770",
Language: pkg.Python,
Type: pkg.PythonPkg,
},
"mypy1": {
{
Name: "mypy1",
Version: "v0.770",
Language: pkg.Python,
Type: pkg.PythonPkg,
},
"mypy2": {
{
Name: "mypy2",
Version: "v0.770",
Language: pkg.Python,
Type: pkg.PythonPkg,
},
"mypy3": {
{
Name: "mypy3",
Version: "v0.770",
Language: pkg.Python,
Type: pkg.PythonPkg,
},
}
fixture, err := os.Open("test-fixtures/setup/setup.py")
if err != nil {
t.Fatalf("failed to open fixture: %+v", err)
}
// TODO: no relationships are under test yet
actual, _, err := parseSetup(fixture.Name(), fixture)
if err != nil {
t.Fatalf("failed to parse requirements: %+v", err)
}
assertPackagesEqual(t, actual, expected)
if diff := cmp.Diff(expected, actual, cmp.AllowUnexported(pkg.Package{})); diff != "" {
t.Errorf("unexpected result from parsing (-expected +actual)\n%s", diff)
}
}

5
syft/pkg/cataloger/python/test-fixtures/requires/requirements.txt
View File

@ -5,3 +5,8 @@ sqlalchemy >= 1.0.0
-e https://github.com/pecan/pecan.git
-r other-requirements.txt
--requirements super-secretrequirements.txt
SomeProject ==5.4 ; python_version < '3.8'
coverage != 3.5 # Version Exclusion. Anything except version 3.5
numpyNew; sys_platform == 'win32'
numpy >= 3.4.1; sys_platform == 'win32'
Mopidy-Dirble ~= 1.1 # Compatible release. Same as >= 1.1, == 1.*