oss/syft
2
0
Fork 0
mirror of https://github.com/anchore/syft.git synced 2026-11-01 23:16:30 +01:00
Code Issues Packages Projects Releases Wiki Activity

Create python requirements metadata (#1759)

Browse Source 
- Create new metadata struct and type for python requirements.
- Update parsing of python requirements to use python requirements metadata.
- Remove extras and url from line. Add them to metadata instead.
- Add unit test to test that extras are removed from package name.
- Update test to look at requirements metadata.
- Will need updated in future to support more than just == for the version constraint.
- Update JSON schema data

Closes anchore/grype#1246
Closes anchore/grype#1251

Signed-off-by: Shane Dell <shanedell100@gmail.com>
This commit is contained in:
Shane Dell
2023-04-27 09:04:30 -04:00
committed by GitHub
parent 451cb9d5ca
commit a07bfe7dfa
13 changed files with 2102 additions and 81 deletions
Download Patch File Download Diff File Expand all files Collapse all files
syft/pkg/cataloger/python/package.go
+17
View File
@@ -40,6 +40,23 @@ func newPackageForIndexWithMetadata(name, version string, metadata pkg.PythonPip
return p
}
func newPackageForRequirementsWithMetadata(name, version string, metadata pkg.PythonRequirementsMetadata, locations ...source.Location) pkg.Package {
p := pkg.Package{
Name: name,
Version: version,
Locations: source.NewLocationSet(locations...),
PURL: packageURL(name, version, nil),
Language: pkg.Python,
Type: pkg.PythonPkg,
MetadataType: pkg.PythonRequirementsMetadataType,
Metadata: metadata,
}
p.SetID()
return p
}
func newPackageForPackage(m pkg.PythonPackageMetadata, sources ...source.Location) pkg.Package {
var licenses []string
if m.License != "" {
syft/pkg/cataloger/python/parse_requirements.go
+100 -1
View File
@@ -3,6 +3,7 @@ package python
import (
"bufio"
"fmt"
"regexp"
"strings"
"unicode"
@@ -15,6 +16,11 @@ import (
var _ generic.Parser = parseRequirementsTxt
var (
extrasRegex = regexp.MustCompile(`\[.*\]`)
urlRegex = regexp.MustCompile("@.*git.*")
)
// parseRequirementsTxt takes a Python requirements.txt file, returning all Python packages that are locked to a
// specific version.
func parseRequirementsTxt(_ source.FileResolver, _ *generic.Environment, reader source.LocationReadCloser) ([]pkg.Package, []artifact.Relationship, error) {
@@ -23,6 +29,7 @@ func parseRequirementsTxt(_ source.FileResolver, _ *generic.Environment, reader
scanner := bufio.NewScanner(reader)
for scanner.Scan() {
line := scanner.Text()
rawLineNoComments := removeTrailingComment(line)
line = trimRequirementsTxtLine(line)
if line == "" {
@@ -57,15 +64,25 @@ func parseRequirementsTxt(_ source.FileResolver, _ *generic.Environment, reader
return !unicode.IsLetter(r) && !unicode.IsNumber(r)
})
// TODO: Update to support more than only ==
versionConstraint := fmt.Sprintf("== %s", version)
if name == "" || version == "" {
log.WithFields("path", reader.RealPath).Debugf("found empty package in requirements.txt line: %q", line)
continue
}
packages = append(
packages,
newPackageForIndex(
newPackageForRequirementsWithMetadata(
name,
version,
pkg.PythonRequirementsMetadata{
Name: name,
Extras: parseExtras(rawLineNoComments),
VersionConstraint: versionConstraint,
URL: parseURL(rawLineNoComments),
Markers: parseMarkers(rawLineNoComments),
},
reader.Location.WithAnnotation(pkg.EvidenceAnnotationKey, pkg.PrimaryEvidenceAnnotation),
),
)
@@ -93,6 +110,7 @@ func trimRequirementsTxtLine(line string) string {
line = strings.TrimSpace(line)
line = removeTrailingComment(line)
line = removeEnvironmentMarkers(line)
line = checkForRegex(line) // remove extras and url from line if found
return line
}
@@ -121,3 +139,84 @@ func removeEnvironmentMarkers(line string) string {
return parts[0]
}
func parseExtras(packageName string) []string {
if extrasRegex.MatchString(packageName) {
// Remove square brackets
extras := strings.TrimFunc(extrasRegex.FindString(packageName), func(r rune) bool {
return !unicode.IsLetter(r) && !unicode.IsNumber(r)
})
// Remove any additional whitespace
extras = strings.ReplaceAll(extras, " ", "")
return strings.Split(extras, ",")
}
return []string{}
}
func parseMarkers(line string) map[string]string {
markers := map[string]string{}
parts := strings.SplitN(line, ";", 2)
if len(parts) == 2 {
splittableMarkers := parts[1]
for _, combineString := range []string{" or ", " and "} {
splittableMarkers = strings.TrimSpace(
strings.ReplaceAll(splittableMarkers, combineString, ","),
)
}
splittableMarkers = strings.TrimSpace(splittableMarkers)
for _, mark := range strings.Split(splittableMarkers, ",") {
markparts := strings.Split(mark, " ")
markers[markparts[0]] = strings.Join(markparts[1:], " ")
}
}
return markers
}
func parseURL(line string) string {
parts := strings.Split(line, "@")
if len(parts) > 1 {
desiredIndex := -1
for index, part := range parts {
part := strings.TrimFunc(part, func(r rune) bool {
return !unicode.IsLetter(r) && !unicode.IsNumber(r)
})
if strings.HasPrefix(part, "git") {
desiredIndex = index
break
}
}
if desiredIndex != -1 {
return strings.TrimSpace(strings.Join(parts[desiredIndex:], "@"))
}
}
return ""
}
// function to check a string for all possilbe regex expressions, replacing it if found
func checkForRegex(stringToCheck string) string {
stringToReturn := stringToCheck
for _, r := range []*regexp.Regexp{
urlRegex,
extrasRegex,
} {
if r.MatchString(stringToCheck) {
stringToReturn = r.ReplaceAllString(stringToCheck, "")
}
}
return stringToReturn
}
syft/pkg/cataloger/python/parse_requirements_test.go
+121 -30
View File
@@ -14,44 +14,135 @@ func TestParseRequirementsTxt(t *testing.T) {
locations := source.NewLocationSet(source.NewLocation(fixture))
expectedPkgs := []pkg.Package{
{
Name: "flask",
Version: "4.0.0",
PURL: "pkg:pypi/flask@4.0.0",
Locations: locations,
Language: pkg.Python,
Type: pkg.PythonPkg,
Name: "flask",
Version: "4.0.0",
PURL: "pkg:pypi/flask@4.0.0",
Locations: locations,
Language: pkg.Python,
Type: pkg.PythonPkg,
MetadataType: pkg.PythonRequirementsMetadataType,
Metadata: pkg.PythonRequirementsMetadata{
Name: "flask",
Extras: []string{},
VersionConstraint: "== 4.0.0",
URL: "",
Markers: map[string]string{},
},
},
{
Name: "foo",
Version: "1.0.0",
PURL: "pkg:pypi/foo@1.0.0",
Locations: locations,
Language: pkg.Python,
Type: pkg.PythonPkg,
Name: "foo",
Version: "1.0.0",
PURL: "pkg:pypi/foo@1.0.0",
Locations: locations,
Language: pkg.Python,
Type: pkg.PythonPkg,
MetadataType: pkg.PythonRequirementsMetadataType,
Metadata: pkg.PythonRequirementsMetadata{
Name: "foo",
Extras: []string{},
VersionConstraint: "== 1.0.0",
URL: "",
Markers: map[string]string{},
},
},
{
Name: "SomeProject",
Version: "5.4",
PURL: "pkg:pypi/SomeProject@5.4",
Locations: locations,
Language: pkg.Python,
Type: pkg.PythonPkg,
Name: "SomeProject",
Version: "5.4",
PURL: "pkg:pypi/SomeProject@5.4",
Locations: locations,
Language: pkg.Python,
Type: pkg.PythonPkg,
MetadataType: pkg.PythonRequirementsMetadataType,
Metadata: pkg.PythonRequirementsMetadata{
Name: "SomeProject",
Extras: []string{},
VersionConstraint: "== 5.4",
URL: "",
Markers: map[string]string{"python_version": "< '3.8'"},
},
},
{
Name: "argh",
Version: "0.26.2",
PURL: "pkg:pypi/argh@0.26.2",
Locations: locations,
Language: pkg.Python,
Type: pkg.PythonPkg,
Name: "argh",
Version: "0.26.2",
PURL: "pkg:pypi/argh@0.26.2",
Locations: locations,
Language: pkg.Python,
Type: pkg.PythonPkg,
MetadataType: pkg.PythonRequirementsMetadataType,
Metadata: pkg.PythonRequirementsMetadata{
Name: "argh",
Extras: []string{},
VersionConstraint: "== 0.26.2",
URL: "",
Markers: map[string]string{},
},
},
{
Name: "argh",
Version: "0.26.3",
PURL: "pkg:pypi/argh@0.26.3",
Locations: locations,
Language: pkg.Python,
Type: pkg.PythonPkg,
Name: "argh",
Version: "0.26.3",
PURL: "pkg:pypi/argh@0.26.3",
Locations: locations,
Language: pkg.Python,
Type: pkg.PythonPkg,
MetadataType: pkg.PythonRequirementsMetadataType,
Metadata: pkg.PythonRequirementsMetadata{
Name: "argh",
Extras: []string{},
VersionConstraint: "== 0.26.3",
URL: "",
Markers: map[string]string{},
},
},
{
Name: "celery",
Version: "4.4.7",
PURL: "pkg:pypi/celery@4.4.7",
Locations: locations,
Language: pkg.Python,
Type: pkg.PythonPkg,
MetadataType: pkg.PythonRequirementsMetadataType,
Metadata: pkg.PythonRequirementsMetadata{
Name: "celery",
Extras: []string{"redis", "pytest"},
VersionConstraint: "== 4.4.7",
URL: "",
Markers: map[string]string{},
},
},
{
Name: "requests",
Version: "2.8",
PURL: "pkg:pypi/requests@2.8",
Locations: locations,
Language: pkg.Python,
Type: pkg.PythonPkg,
MetadataType: pkg.PythonRequirementsMetadataType,
Metadata: pkg.PythonRequirementsMetadata{
Name: "requests",
Extras: []string{"security"},
VersionConstraint: "== 2.8",
URL: "",
Markers: map[string]string{
"python_version": `< "2.7"`,
"sys_platform": `== "linux"`,
},
},
},
{
Name: "GithubSampleProject",
Version: "3.7.1",
PURL: "pkg:pypi/GithubSampleProject@3.7.1",
Locations: locations,
Language: pkg.Python,
Type: pkg.PythonPkg,
MetadataType: pkg.PythonRequirementsMetadataType,
Metadata: pkg.PythonRequirementsMetadata{
Name: "GithubSampleProject",
Extras: []string{},
VersionConstraint: "== 3.7.1",
URL: "git+https://github.com/owner/repo@releases/tag/v3.7.1",
Markers: map[string]string{},
},
},
}
syft/pkg/cataloger/python/test-fixtures/requires/requirements.txt
+4 -1
View File
@@ -16,4 +16,7 @@ argh==0.26.2 \
argh==0.26.3 --hash=sha256:a9b3aaa1904eeb78e32394cd46c6f37ac0fb4af6dc488daa58971bdc7d7fcaf3 --hash=sha256:e9535b8c84dc9571a48999094fda7f33e63c3f1b74f3e5f3ac0105a58405bb65
# CommentedOut == 1.2.3
# maybe invalid, but found out in the wild
==2.3.4
==2.3.4
celery[redis, pytest] == 4.4.7 # should remove [redis, pytest]
requests[security] == 2.8.* ; python_version < "2.7" and sys_platform == "linux"
GithubSampleProject == 3.7.1 @ git+https://github.com/owner/repo@releases/tag/v3.7.1