oss/syft
2
0
Fork 0
mirror of https://github.com/anchore/syft.git synced 2026-10-31 06:26:30 +01:00
Code Issues Packages Projects Releases Wiki Activity
Files
2293641e3bd628a01bb37639318d62c0ebe89b39
syft/syft/pkg/cataloger/python/parse_requirements_test.go
T

616 lines
16 KiB
Go
Raw Blame History

package python
import (
"context"
"testing"
"github.com/stretchr/testify/assert"
"github.com/anchore/syft/syft/artifact"
"github.com/anchore/syft/syft/file"
"github.com/anchore/syft/syft/pkg"
"github.com/anchore/syft/syft/pkg/cataloger/internal/pkgtest"
)
func TestParseRequirementsTxt(t *testing.T) {
fixture := "testdata/requires/requirements.txt"
locations := file.NewLocationSet(file.NewLocation(fixture))
pinnedPkgs := []pkg.Package{
{
Name: "flask",
Version: "4.0.0",
PURL: "pkg:pypi/flask@4.0.0",
Locations: locations,
Language: pkg.Python,
Type: pkg.PythonPkg,
Metadata: pkg.PythonRequirementsEntry{
Name: "flask",
VersionConstraint: "== 4.0.0",
},
},
{
Name: "urllib3",
Version: "1.26.20",
PURL: "pkg:pypi/urllib3@1.26.20",
Locations: locations,
Language: pkg.Python,
Type: pkg.PythonPkg,
Metadata: pkg.PythonRequirementsEntry{
Name: "urllib3",
VersionConstraint: "===1.26.20",
},
},
{
Name: "foo",
Version: "1.0.0",
PURL: "pkg:pypi/foo@1.0.0",
Locations: locations,
Language: pkg.Python,
Type: pkg.PythonPkg,
Metadata: pkg.PythonRequirementsEntry{
Name: "foo",
VersionConstraint: "== 1.0.0",
},
},
{
Name: "someproject",
Version: "5.4",
PURL: "pkg:pypi/someproject@5.4",
Locations: locations,
Language: pkg.Python,
Type: pkg.PythonPkg,
Metadata: pkg.PythonRequirementsEntry{
Name: "SomeProject",
VersionConstraint: "==5.4",
Markers: "python_version < '3.8'",
},
},
{
Name: "dots-allowed",
Version: "1.0.0",
PURL: "pkg:pypi/dots-allowed@1.0.0",
Locations: locations,
Language: pkg.Python,
Type: pkg.PythonPkg,
Metadata: pkg.PythonRequirementsEntry{
Name: "dots-._allowed",
VersionConstraint: "== 1.0.0",
},
},
{
Name: "argh",
Version: "0.26.2",
PURL: "pkg:pypi/argh@0.26.2",
Locations: locations,
Language: pkg.Python,
Type: pkg.PythonPkg,
Metadata: pkg.PythonRequirementsEntry{
Name: "argh",
VersionConstraint: "==0.26.2",
},
},
{
Name: "argh",
Version: "0.26.3",
PURL: "pkg:pypi/argh@0.26.3",
Locations: locations,
Language: pkg.Python,
Type: pkg.PythonPkg,
Metadata: pkg.PythonRequirementsEntry{
Name: "argh",
VersionConstraint: "==0.26.3",
},
},
{
Name: "celery",
Version: "4.4.7",
PURL: "pkg:pypi/celery@4.4.7",
Locations: locations,
Language: pkg.Python,
Type: pkg.PythonPkg,
Metadata: pkg.PythonRequirementsEntry{
Name: "celery",
Extras: []string{"redis", "pytest"},
VersionConstraint: "== 4.4.7",
},
},
{
Name: "githubsampleproject",
Version: "3.7.1",
PURL: "pkg:pypi/githubsampleproject@3.7.1",
Locations: locations,
Language: pkg.Python,
Type: pkg.PythonPkg,
Metadata: pkg.PythonRequirementsEntry{
Name: "GithubSampleProject",
VersionConstraint: "== 3.7.1",
URL: "git+https://github.com/owner/repo@releases/tag/v3.7.1",
},
},
{
Name: "friendly-bard",
Version: "1.0.0",
PURL: "pkg:pypi/friendly-bard@1.0.0",
Locations: locations,
Language: pkg.Python,
Type: pkg.PythonPkg,
Metadata: pkg.PythonRequirementsEntry{
Name: "FrIeNdLy-_-bArD",
VersionConstraint: "== 1.0.0",
},
},
{
Name: "local-version",
Version: "1.2.3+gcr.2",
PURL: "pkg:pypi/local-version@1.2.3%2Bgcr.2",
Locations: locations,
Language: pkg.Python,
Type: pkg.PythonPkg,
Metadata: pkg.PythonRequirementsEntry{
Name: "local-version",
VersionConstraint: "== 1.2.3+gcr.2",
},
},
}
var testCases = []struct {
name string
fixture string
cfg CatalogerConfig
expectedPkgs []pkg.Package
expectedRelationships []artifact.Relationship
}{
{
name: "pinned dependencies only",
fixture: fixture,
cfg: CatalogerConfig{
GuessUnpinnedRequirements: false,
},
expectedPkgs: pinnedPkgs,
},
{
name: "guess unpinned requirements (lowest version)",
fixture: fixture,
cfg: CatalogerConfig{
GuessUnpinnedRequirements: true,
},
expectedPkgs: append([]pkg.Package{
{
Name: "mopidy-dirble",
Version: "1.1",
PURL: "pkg:pypi/mopidy-dirble@1.1",
Locations: locations,
Language: pkg.Python,
Type: pkg.PythonPkg,
Metadata: pkg.PythonRequirementsEntry{
Name: "Mopidy-Dirble",
VersionConstraint: "~= 1.1",
},
},
{
Name: "sqlalchemy",
Version: "2.0.0",
PURL: "pkg:pypi/sqlalchemy@2.0.0",
Locations: locations,
Language: pkg.Python,
Type: pkg.PythonPkg,
Metadata: pkg.PythonRequirementsEntry{
Name: "sqlalchemy",
VersionConstraint: ">= 1.0.0, <= 2.0.0, != 3.0.0, <= 3.0.0",
},
},
{
Name: "bar",
Version: "2.0.0",
PURL: "pkg:pypi/bar@2.0.0",
Locations: locations,
Language: pkg.Python,
Type: pkg.PythonPkg,
Metadata: pkg.PythonRequirementsEntry{
Name: "bar",
VersionConstraint: ">= 1.0.0, <= 2.0.0, != 3.0.0, <= 3.0.0",
},
},
{
Name: "numpy",
Version: "3.4.1",
PURL: "pkg:pypi/numpy@3.4.1",
Locations: locations,
Language: pkg.Python,
Type: pkg.PythonPkg,
Metadata: pkg.PythonRequirementsEntry{
Name: "numpy",
VersionConstraint: ">= 3.4.1",
Markers: `sys_platform == 'win32'`,
},
},
{
Name: "requests",
Version: "2.8.0",
PURL: "pkg:pypi/requests@2.8.0",
Locations: locations,
Language: pkg.Python,
Type: pkg.PythonPkg,
Metadata: pkg.PythonRequirementsEntry{
Name: "requests",
Extras: []string{"security"},
VersionConstraint: "== 2.8.*",
Markers: `python_version < "2.7" and sys_platform == "linux"`,
},
},
}, pinnedPkgs...),
},
}
for _, tc := range testCases {
t.Run(tc.name, func(t *testing.T) {
parser := newRequirementsParser(tc.cfg)
pkgtest.TestFileParser(t, tc.fixture, parser.parseRequirementsTxt, tc.expectedPkgs, tc.expectedRelationships)
})
}
}
func TestParseRequirementsTxtWithLicenseEnrichment(t *testing.T) {
ctx := context.TODO()
fixture := "testdata/pypi-remote/requirements.txt"
locations := file.NewLocationSet(file.NewLocation(fixture))
mux, url, teardown := setupPypiRegistry()
defer teardown()
tests := []struct {
name string
fixture string
config CatalogerConfig
requestHandlers []handlerPath
expectedPackages []pkg.Package
}{
{
name: "search remote licenses returns the expected licenses when search is set to true",
config: CatalogerConfig{SearchRemoteLicenses: true},
requestHandlers: []handlerPath{
{
path: "/certifi/2025.10.5/json",
handler: generateMockPypiRegistryHandler("testdata/pypi-remote/registry_response.json"),
},
},
expectedPackages: []pkg.Package{
{
Name: "certifi",
Version: "2025.10.5",
Locations: locations,
PURL: "pkg:pypi/certifi@2025.10.5",
Licenses: pkg.NewLicenseSet(pkg.NewLicenseWithContext(ctx, "MPL-2.0")),
Language: pkg.Python,
Type: pkg.PythonPkg,
Metadata: pkg.PythonRequirementsEntry{
Name: "certifi",
VersionConstraint: "== 2025.10.5",
},
},
},
},
}
for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
// set up the mock server
for _, handler := range tc.requestHandlers {
mux.HandleFunc(handler.path, handler.handler)
}
tc.config.PypiBaseURL = url
requirementsParser := newRequirementsParser(tc.config)
pkgtest.TestFileParser(t, fixture, requirementsParser.parseRequirementsTxt, tc.expectedPackages, nil)
})
}
}
func Test_newRequirement(t *testing.T) {
tests := []struct {
name string
raw string
want *unprocessedRequirement
}{
{
name: "simple",
raw: "requests==2.8",
want: &unprocessedRequirement{
Name: "requests",
VersionConstraint: "==2.8",
},
},
{
name: "arbitrary equality",
raw: "urllib3===1.26.20",
want: &unprocessedRequirement{
Name: "urllib3",
VersionConstraint: "===1.26.20",
},
},
{
name: "comment + constraint",
raw: "Mopidy-Dirble ~= 1.1 # Compatible release. Same as >= 1.1, == 1.*",
want: &unprocessedRequirement{
Name: "Mopidy-Dirble",
VersionConstraint: "~= 1.1",
},
},
{
name: "hashes",
raw: "argh==0.26.3 --hash=sha256:a9b3aaa1904eeb78e32394cd46c6f37ac0fb4af6dc488daa58971bdc7d7fcaf3 --hash=sha256:e9535b8c84dc9571a48999094fda7f33e63c3f1b74f3e5f3ac0105a58405bb65",
want: &unprocessedRequirement{
Name: "argh",
VersionConstraint: "==0.26.3",
Hashes: "--hash=sha256:a9b3aaa1904eeb78e32394cd46c6f37ac0fb4af6dc488daa58971bdc7d7fcaf3 --hash=sha256:e9535b8c84dc9571a48999094fda7f33e63c3f1b74f3e5f3ac0105a58405bb65",
},
},
{
name: "extras",
raw: "celery[redis, pytest] == 4.4.7 # should remove [redis, pytest]",
want: &unprocessedRequirement{
Name: "celery[redis, pytest]",
VersionConstraint: "== 4.4.7",
},
},
{
name: "url",
raw: "GithubSampleProject == 3.7.1 @ git+https://github.com/owner/repo@releases/tag/v3.7.1",
want: &unprocessedRequirement{
Name: "GithubSampleProject",
VersionConstraint: "== 3.7.1",
URL: "git+https://github.com/owner/repo@releases/tag/v3.7.1",
},
},
{
name: "markers",
raw: "numpy >= 3.4.1 ; sys_platform == 'win32'",
want: &unprocessedRequirement{
Name: "numpy",
VersionConstraint: ">= 3.4.1",
Markers: "sys_platform == 'win32'",
},
},
{
name: "local version identifier",
raw: "local-version == 1.2.3+gcr.2",
want: &unprocessedRequirement{
Name: "local-version",
VersionConstraint: "== 1.2.3+gcr.2",
},
},
{
name: "local version identifier with markers",
raw: "local-version == 1.2.3+ubuntu1 ; sys_platform == 'linux'",
want: &unprocessedRequirement{
Name: "local-version",
VersionConstraint: "== 1.2.3+ubuntu1",
Markers: "sys_platform == 'linux'",
},
},
{
name: "epoch",
raw: "pkg == 1!2.0.0",
want: &unprocessedRequirement{
Name: "pkg",
VersionConstraint: "== 1!2.0.0",
},
},
{
name: "pre-release",
raw: "pkg == 1.0rc1",
want: &unprocessedRequirement{
Name: "pkg",
VersionConstraint: "== 1.0rc1",
},
},
{
name: "pre-release with dash separators",
raw: "pkg == 1.0-alpha-1",
want: &unprocessedRequirement{
Name: "pkg",
VersionConstraint: "== 1.0-alpha-1",
},
},
{
name: "pre-release with underscore separators",
raw: "pkg == 1.0_beta_2",
want: &unprocessedRequirement{
Name: "pkg",
VersionConstraint: "== 1.0_beta_2",
},
},
{
name: "post-release",
raw: "pkg == 1.0.post1",
want: &unprocessedRequirement{
Name: "pkg",
VersionConstraint: "== 1.0.post1",
},
},
{
name: "implicit post-release",
raw: "pkg == 1.0-1",
want: &unprocessedRequirement{
Name: "pkg",
VersionConstraint: "== 1.0-1",
},
},
{
name: "dev-release",
raw: "pkg == 1.0.dev1",
want: &unprocessedRequirement{
Name: "pkg",
VersionConstraint: "== 1.0.dev1",
},
},
{
name: "local version with dash separator",
raw: "pkg == 1.0+ubuntu-1",
want: &unprocessedRequirement{
Name: "pkg",
VersionConstraint: "== 1.0+ubuntu-1",
},
},
{
name: "local version with underscore separator",
raw: "pkg == 1.0+ubuntu_1",
want: &unprocessedRequirement{
Name: "pkg",
VersionConstraint: "== 1.0+ubuntu_1",
},
},
{
name: "all segments combined",
raw: "pkg == 1!1.0a1.post2.dev3+local.1",
want: &unprocessedRequirement{
Name: "pkg",
VersionConstraint: "== 1!1.0a1.post2.dev3+local.1",
},
},
{
name: "release wildcard",
raw: "pkg == 2.*",
want: &unprocessedRequirement{
Name: "pkg",
VersionConstraint: "== 2.*",
},
},
{
name: "epoch with markers and hashes",
raw: "pkg == 1!2.0 ; python_version < '3.8' --hash=sha256:abc123",
want: &unprocessedRequirement{
Name: "pkg",
VersionConstraint: "== 1!2.0",
Markers: "python_version < '3.8' --hash=sha256:abc123",
},
},
{
name: "compound constraint with exclusion is unaffected",
raw: "pkg >= 1.0.0, != 1.1.0, < 2.0.0",
want: &unprocessedRequirement{
Name: "pkg",
VersionConstraint: ">= 1.0.0, != 1.1.0, < 2.0.0",
},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
assert.Equal(t, tt.want, newRequirement(tt.raw))
})
}
}
// checkout https://www.darius.page/pipdev/ for help here! (github.com/nok/pipdev)
func Test_parseVersion(t *testing.T) {
tests := []struct {
name string
version string
guess bool
want string
}{
{
name: "exact",
version: "1.0.0",
want: "", // we can only parse constraints, not assume that a single version is a pin
},
{
name: "exact constraint",
version: " == 1.0.0 ",
want: "1.0.0",
},
{
name: "arbitrary equality constraint",
version: " === 1.26.20 ",
want: "1.26.20",
},
{
name: "local version identifier",
version: " == 1.2.3+gcr.2 ",
want: "1.2.3+gcr.2",
},
{
name: "arbitrary equality with local version identifier",
version: " === 1.2.3+ubuntu1 ",
want: "1.2.3+ubuntu1",
},
{
name: "pre-release",
version: "== 1.0rc1",
want: "1.0rc1",
},
{
name: "pre-release with dash separators",
version: "== 1.0-alpha-1",
want: "1.0-alpha-1",
},
{
name: "post-release",
version: "== 1.0.post1",
want: "1.0.post1",
},
{
name: "dev-release",
version: "== 1.0.dev1",
want: "1.0.dev1",
},
{
name: "local version with dash separator",
version: "== 1.0+ubuntu-1",
want: "1.0+ubuntu-1",
},
{
name: "all segments combined",
version: "== 1.0a1.post2.dev3+local.1",
want: "1.0a1.post2.dev3+local.1",
},
{
name: "resolve lowest, simple constraint",
version: " >= 1.0.0 ",
guess: true,
want: "1.0.0",
},
{
name: "resolve lowest, compound constraint",
version: " < 2.0.0, >= 1.0.0, != 1.1.0 ",
guess: true,
want: "1.0.0",
},
{
name: "resolve lowest, handle asterisk",
version: "==2.8.*",
guess: true,
want: "2.8.0",
},
{
name: "resolve lowest, handle exceptions",
version: " !=4.0.2,!=4.1.0,!=4.2.0,>=4.0.1,!=4.3.0,!=5.0.0,!=5.1.0,<6.0.0",
guess: true,
want: "4.0.1",
},
{
name: "resolve lowest, compatible version constraint",
version: "~=0.6.10", // equates to >=0.6.10, ==0.6.*
guess: true,
want: "0.6.10",
},
{
name: "resolve lowest, with character in version",
version: "~=1.2b,<=1.3a,!=1.1,!=1.2",
guess: true,
want: "1.3a0", // note: 1.3a == 1.3a0
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
assert.Equal(t, tt.want, parseVersion(tt.version, tt.guess))
})
}
}
func Test_corruptRequirementsTxt(t *testing.T) {
rp := newRequirementsParser(DefaultCatalogerConfig())
pkgtest.NewCatalogTester().
FromFile(t, "testdata/glob-paths/src/requirements.txt").
WithError().
TestParser(t, rp.parseRequirementsTxt)
}
Reference in New Issue View Git Blame Copy Permalink