oss/syft
2
0
Fork 0
mirror of https://github.com/anchore/syft.git synced 2026-11-05 00:46:38 +01:00
Code Issues Packages Projects Releases Wiki Activity

migrate syft/cataloger to syft/pkg/cataloger

Browse Source 
Signed-off-by: Alex Goodman <alex.goodman@anchore.com>
This commit is contained in:
Alex Goodman
2021-03-22 10:46:51 -04:00
parent cb5e7d0e08
commit 4666ca8469
157 changed files with 394 additions and 563 deletions
Download Patch File Download Diff File Expand all files Collapse all files
syft/pkg/cataloger/python/index_cataloger.go
+19
View File
@@ -0,0 +1,19 @@
/*
Package python provides a concrete Cataloger implementation for Python ecosystem files (egg, wheel, requirements.txt).
*/
package python
import (
"github.com/anchore/syft/syft/pkg/cataloger/common"
)
// NewPythonIndexCataloger returns a new cataloger for python packages referenced from poetry lock files, requirements.txt files, and setup.py files.
func NewPythonIndexCataloger() *common.GenericCataloger {
globParsers := map[string]common.ParserFn{
"**/*requirements*.txt": parseRequirementsTxt,
"**/poetry.lock": parsePoetryLock,
"**/setup.py": parseSetup,
}
return common.NewGenericCataloger(nil, globParsers, "python-index-cataloger")
}
syft/pkg/cataloger/python/package_cataloger.go
+178
View File
@@ -0,0 +1,178 @@
package python
import (
"bufio"
"fmt"
"path/filepath"
"github.com/anchore/syft/syft/pkg"
"github.com/anchore/syft/syft/source"
)
const (
eggMetadataGlob = "**/*egg-info/PKG-INFO"
eggFileMetadataGlob = "**/*.egg-info"
wheelMetadataGlob = "**/*dist-info/METADATA"
)
type PackageCataloger struct{}
// NewPythonPackageCataloger returns a new cataloger for python packages within egg or wheel installation directories.
func NewPythonPackageCataloger() *PackageCataloger {
return &PackageCataloger{}
}
// Name returns a string that uniquely describes a cataloger
func (c *PackageCataloger) Name() string {
return "python-package-cataloger"
}
// Catalog is given an object to resolve file references and content, this function returns any discovered Packages after analyzing python egg and wheel installations.
func (c *PackageCataloger) Catalog(resolver source.FileResolver) ([]pkg.Package, error) {
// nolint:prealloc
var fileMatches []source.Location
for _, glob := range []string{eggMetadataGlob, wheelMetadataGlob, eggFileMetadataGlob} {
matches, err := resolver.FilesByGlob(glob)
if err != nil {
return nil, fmt.Errorf("failed to find files by glob: %s", glob)
}
fileMatches = append(fileMatches, matches...)
}
var pkgs []pkg.Package
for _, location := range fileMatches {
p, err := c.catalogEggOrWheel(resolver, location)
if err != nil {
return nil, fmt.Errorf("unable to catalog python package=%+v: %w", location.RealPath, err)
}
if p != nil {
pkgs = append(pkgs, *p)
}
}
return pkgs, nil
}
// catalogEggOrWheel takes the primary metadata file reference and returns the python package it represents.
func (c *PackageCataloger) catalogEggOrWheel(resolver source.FileResolver, metadataLocation source.Location) (*pkg.Package, error) {
metadata, sources, err := c.assembleEggOrWheelMetadata(resolver, metadataLocation)
if err != nil {
return nil, err
}
// This can happen for Python 2.7 where it is reported from an egg-info, but Python is
// the actual runtime, it isn't a "package". The special-casing here allows to skip it
if metadata.Name == "Python" {
return nil, nil
}
var licenses []string
if metadata.License != "" {
licenses = []string{metadata.License}
}
return &pkg.Package{
Name: metadata.Name,
Version: metadata.Version,
FoundBy: c.Name(),
Locations: sources,
Licenses: licenses,
Language: pkg.Python,
Type: pkg.PythonPkg,
MetadataType: pkg.PythonPackageMetadataType,
Metadata: *metadata,
}, nil
}
// fetchRecordFiles finds a corresponding RECORD file for the given python package metadata file and returns the set of file records contained.
func (c *PackageCataloger) fetchRecordFiles(resolver source.FileResolver, metadataLocation source.Location) (files []pkg.PythonFileRecord, sources []source.Location, err error) {
// we've been given a file reference to a specific wheel METADATA file. note: this may be for a directory
// or for an image... for an image the METADATA file may be present within multiple layers, so it is important
// to reconcile the RECORD path to the same layer (or the next adjacent lower layer).
// lets find the RECORD file relative to the directory where the METADATA file resides (in path AND layer structure)
recordPath := filepath.Join(filepath.Dir(metadataLocation.RealPath), "RECORD")
recordRef := resolver.RelativeFileByPath(metadataLocation, recordPath)
if recordRef != nil {
sources = append(sources, *recordRef)
recordContents, err := resolver.FileContentsByLocation(*recordRef)
if err != nil {
return nil, nil, err
}
// parse the record contents
records, err := parseWheelOrEggRecord(recordContents)
if err != nil {
return nil, nil, err
}
files = append(files, records...)
}
return files, sources, nil
}
// fetchTopLevelPackages finds a corresponding top_level.txt file for the given python package metadata file and returns the set of package names contained.
func (c *PackageCataloger) fetchTopLevelPackages(resolver source.FileResolver, metadataLocation source.Location) (pkgs []string, sources []source.Location, err error) {
// a top_level.txt file specifies the python top-level packages (provided by this python package) installed into site-packages
parentDir := filepath.Dir(metadataLocation.RealPath)
topLevelPath := filepath.Join(parentDir, "top_level.txt")
topLevelLocation := resolver.RelativeFileByPath(metadataLocation, topLevelPath)
if topLevelLocation == nil {
return nil, nil, nil
}
sources = append(sources, *topLevelLocation)
topLevelContents, err := resolver.FileContentsByLocation(*topLevelLocation)
if err != nil {
return nil, nil, err
}
scanner := bufio.NewScanner(topLevelContents)
for scanner.Scan() {
pkgs = append(pkgs, scanner.Text())
}
if err := scanner.Err(); err != nil {
return nil, nil, fmt.Errorf("could not read python package top_level.txt: %w", err)
}
return pkgs, sources, nil
}
// assembleEggOrWheelMetadata discovers and accumulates python package metadata from multiple file sources and returns a single metadata object as well as a list of files where the metadata was derived from.
func (c *PackageCataloger) assembleEggOrWheelMetadata(resolver source.FileResolver, metadataLocation source.Location) (*pkg.PythonPackageMetadata, []source.Location, error) {
var sources = []source.Location{metadataLocation}
metadataContents, err := resolver.FileContentsByLocation(metadataLocation)
if err != nil {
return nil, nil, err
}
metadata, err := parseWheelOrEggMetadata(metadataLocation.RealPath, metadataContents)
if err != nil {
return nil, nil, err
}
// attach any python files found for the given wheel/egg installation
r, s, err := c.fetchRecordFiles(resolver, metadataLocation)
if err != nil {
return nil, nil, err
}
sources = append(sources, s...)
metadata.Files = r
// attach any top-level package names found for the given wheel/egg installation
p, s, err := c.fetchTopLevelPackages(resolver, metadataLocation)
if err != nil {
return nil, nil, err
}
sources = append(sources, s...)
metadata.TopLevelPackages = p
return &metadata, sources, nil
}
syft/pkg/cataloger/python/package_cataloger_test.go
+184
View File
@@ -0,0 +1,184 @@
package python
import (
"testing"
"github.com/anchore/syft/syft/pkg"
"github.com/anchore/syft/syft/source"
"github.com/go-test/deep"
)
func TestPythonPackageWheelCataloger(t *testing.T) {
tests := []struct {
name string
fixtures []string
expectedPackage pkg.Package
}{
{
name: "egg-info directory",
fixtures: []string{
"test-fixtures/egg-info/PKG-INFO",
"test-fixtures/egg-info/RECORD",
"test-fixtures/egg-info/top_level.txt",
},
expectedPackage: pkg.Package{
Name: "requests",
Version: "2.22.0",
Type: pkg.PythonPkg,
Language: pkg.Python,
Licenses: []string{"Apache 2.0"},
FoundBy: "python-package-cataloger",
MetadataType: pkg.PythonPackageMetadataType,
Metadata: pkg.PythonPackageMetadata{
Name: "requests",
Version: "2.22.0",
License: "Apache 2.0",
Platform: "UNKNOWN",
Author: "Kenneth Reitz",
AuthorEmail: "me@kennethreitz.org",
SitePackagesRootPath: "test-fixtures",
Files: []pkg.PythonFileRecord{
{Path: "requests-2.22.0.dist-info/INSTALLER", Digest: &pkg.PythonFileDigest{"sha256", "zuuue4knoyJ-UwPPXg8fezS7VCrXJQrAP7zeNuwvFQg"}, Size: "4"},
{Path: "requests/__init__.py", Digest: &pkg.PythonFileDigest{"sha256", "PnKCgjcTq44LaAMzB-7--B2FdewRrE8F_vjZeaG9NhA"}, Size: "3921"},
{Path: "requests/__pycache__/__version__.cpython-38.pyc"},
{Path: "requests/__pycache__/utils.cpython-38.pyc"},
{Path: "requests/__version__.py", Digest: &pkg.PythonFileDigest{"sha256", "Bm-GFstQaFezsFlnmEMrJDe8JNROz9n2XXYtODdvjjc"}, Size: "436"},
{Path: "requests/utils.py", Digest: &pkg.PythonFileDigest{"sha256", "LtPJ1db6mJff2TJSJWKi7rBpzjPS3mSOrjC9zRhoD3A"}, Size: "30049"},
},
TopLevelPackages: []string{"requests"},
},
},
},
{
name: "dist-info directory",
fixtures: []string{
"test-fixtures/dist-info/METADATA",
"test-fixtures/dist-info/RECORD",
"test-fixtures/dist-info/top_level.txt",
},
expectedPackage: pkg.Package{
Name: "Pygments",
Version: "2.6.1",
Type: pkg.PythonPkg,
Language: pkg.Python,
Licenses: []string{"BSD License"},
FoundBy: "python-package-cataloger",
MetadataType: pkg.PythonPackageMetadataType,
Metadata: pkg.PythonPackageMetadata{
Name: "Pygments",
Version: "2.6.1",
License: "BSD License",
Platform: "any",
Author: "Georg Brandl",
AuthorEmail: "georg@python.org",
SitePackagesRootPath: "test-fixtures",
Files: []pkg.PythonFileRecord{
{Path: "../../../bin/pygmentize", Digest: &pkg.PythonFileDigest{"sha256", "dDhv_U2jiCpmFQwIRHpFRLAHUO4R1jIJPEvT_QYTFp8"}, Size: "220"},
{Path: "Pygments-2.6.1.dist-info/AUTHORS", Digest: &pkg.PythonFileDigest{"sha256", "PVpa2_Oku6BGuiUvutvuPnWGpzxqFy2I8-NIrqCvqUY"}, Size: "8449"},
{Path: "Pygments-2.6.1.dist-info/RECORD"},
{Path: "pygments/__pycache__/__init__.cpython-38.pyc"},
{Path: "pygments/util.py", Digest: &pkg.PythonFileDigest{"sha256", "586xXHiJGGZxqk5PMBu3vBhE68DLuAe5MBARWrSPGxA"}, Size: "10778"},
},
TopLevelPackages: []string{"pygments", "something_else"},
},
},
},
{
// in cases where the metadata file is available and the record is not we should still record there is a package
// additionally empty top_level.txt files should not result in an error
name: "partial dist-info directory",
fixtures: []string{"test-fixtures/partial.dist-info/METADATA"},
expectedPackage: pkg.Package{
Name: "Pygments",
Version: "2.6.1",
Type: pkg.PythonPkg,
Language: pkg.Python,
Licenses: []string{"BSD License"},
FoundBy: "python-package-cataloger",
MetadataType: pkg.PythonPackageMetadataType,
Metadata: pkg.PythonPackageMetadata{
Name: "Pygments",
Version: "2.6.1",
License: "BSD License",
Platform: "any",
Author: "Georg Brandl",
AuthorEmail: "georg@python.org",
SitePackagesRootPath: "test-fixtures",
},
},
},
{
name: "egg-info regular file",
fixtures: []string{"test-fixtures/test.egg-info"},
expectedPackage: pkg.Package{
Name: "requests",
Version: "2.22.0",
Type: pkg.PythonPkg,
Language: pkg.Python,
Licenses: []string{"Apache 2.0"},
FoundBy: "python-package-cataloger",
MetadataType: pkg.PythonPackageMetadataType,
Metadata: pkg.PythonPackageMetadata{
Name: "requests",
Version: "2.22.0",
License: "Apache 2.0",
Platform: "UNKNOWN",
Author: "Kenneth Reitz",
AuthorEmail: "me@kennethreitz.org",
SitePackagesRootPath: "test-fixtures",
},
},
},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
resolver := source.NewMockResolverForPaths(test.fixtures...)
locations, err := resolver.FilesByPath(test.fixtures...)
if err != nil {
t.Fatal(err)
}
test.expectedPackage.Locations = locations
actual, err := NewPythonPackageCataloger().Catalog(resolver)
if err != nil {
t.Fatalf("failed to catalog python package: %+v", err)
}
if len(actual) != 1 {
t.Fatalf("unexpected number of packages: %d", len(actual))
}
for _, d := range deep.Equal(actual[0], test.expectedPackage) {
t.Errorf("diff: %+v", d)
}
})
}
}
func TestIgnorePackage(t *testing.T) {
tests := []struct {
MetadataFixture string
}{
{
MetadataFixture: "test-fixtures/Python-2.7.egg-info",
},
}
for _, test := range tests {
t.Run(test.MetadataFixture, func(t *testing.T) {
resolver := source.NewMockResolverForPaths(test.MetadataFixture)
actual, err := NewPythonPackageCataloger().Catalog(resolver)
if err != nil {
t.Fatalf("failed to catalog python package: %+v", err)
}
if len(actual) != 0 {
t.Fatalf("Expected 0 packages but found: %d", len(actual))
}
})
}
}
syft/pkg/cataloger/python/parse_poetry_lock.go
+29
View File
@@ -0,0 +1,29 @@
package python
import (
"fmt"
"io"
"github.com/anchore/syft/syft/pkg"
"github.com/anchore/syft/syft/pkg/cataloger/common"
"github.com/pelletier/go-toml"
)
// integrity check
var _ common.ParserFn = parsePoetryLock
// parsePoetryLock is a parser function for poetry.lock contents, returning all python packages discovered.
func parsePoetryLock(_ string, reader io.Reader) ([]pkg.Package, error) {
tree, err := toml.LoadReader(reader)
if err != nil {
return nil, fmt.Errorf("unable to load poetry.lock for parsing: %v", err)
}
metadata := PoetryMetadata{}
err = tree.Unmarshal(&metadata)
if err != nil {
return nil, fmt.Errorf("unable to parse poetry.lock: %v", err)
}
return metadata.Pkgs(), nil
}
syft/pkg/cataloger/python/parse_poetry_lock_test.go
+57
View File
@@ -0,0 +1,57 @@
package python
import (
"os"
"testing"
"github.com/anchore/syft/syft/pkg"
"github.com/go-test/deep"
)
func TestParsePoetryLock(t *testing.T) {
expected := []pkg.Package{
{
Name: "added-value",
Version: "0.14.2",
Language: pkg.Python,
Type: pkg.PythonPkg,
Licenses: nil,
},
{
Name: "alabaster",
Version: "0.7.12",
Language: pkg.Python,
Type: pkg.PythonPkg,
Licenses: nil,
},
{
Name: "appnope",
Version: "0.1.0",
Language: pkg.Python,
Type: pkg.PythonPkg,
Licenses: nil,
},
{
Name: "asciitree",
Version: "0.3.3",
Language: pkg.Python,
Type: pkg.PythonPkg,
Licenses: nil,
},
}
fixture, err := os.Open("test-fixtures/poetry/poetry.lock")
if err != nil {
t.Fatalf("failed to open fixture: %+v", err)
}
actual, err := parsePoetryLock(fixture.Name(), fixture)
if err != nil {
t.Error(err)
}
differences := deep.Equal(expected, actual)
if differences != nil {
t.Errorf("returned package list differed from expectation: %+v", differences)
}
}
syft/pkg/cataloger/python/parse_requirements.go
+76
View File
@@ -0,0 +1,76 @@
package python
import (
"bufio"
"fmt"
"io"
"strings"
"github.com/anchore/syft/syft/pkg"
"github.com/anchore/syft/syft/pkg/cataloger/common"
)
// integrity check
var _ common.ParserFn = parseRequirementsTxt
// parseRequirementsTxt takes a Python requirements.txt file, returning all Python packages that are locked to a
// specific version.
func parseRequirementsTxt(_ string, reader io.Reader) ([]pkg.Package, error) {
packages := make([]pkg.Package, 0)
scanner := bufio.NewScanner(reader)
for scanner.Scan() {
line := scanner.Text()
line = strings.TrimRight(line, "\n")
switch {
case strings.HasPrefix(line, "#"):
// commented out line, skip
continue
case strings.HasPrefix(line, "-e"):
// editable packages aren't parsed (yet)
continue
case len(strings.Split(line, "==")) < 2:
// a package without a version, or a range (unpinned) which
// does not tell us exactly what will be installed
// XXX only needed if we want to log this, otherwise the next case catches it
continue
case len(strings.Split(line, "==")) == 2:
// remove comments if present
uncommented := removeTrailingComment(line)
// parse a new requirement
parts := strings.Split(uncommented, "==")
name := strings.TrimSpace(parts[0])
version := strings.TrimSpace(parts[1])
packages = append(packages, pkg.Package{
Name: name,
Version: version,
Language: pkg.Python,
Type: pkg.PythonPkg,
})
default:
continue
}
}
if err := scanner.Err(); err != nil {
return nil, fmt.Errorf("failed to parse python requirements file: %w", err)
}
return packages, nil
}
// removeTrailingComment takes a requirements.txt line and strips off comment strings.
func removeTrailingComment(line string) string {
parts := strings.Split(line, "#")
switch len(parts) {
case 1:
// there aren't any comments
return line
default:
// any number of "#" means we only want the first part, assuming this
// isn't prefixed with "#" (up to the caller)
return parts[0]
}
}
syft/pkg/cataloger/python/parse_requirements_test.go
+60
View File
@@ -0,0 +1,60 @@
package python
import (
"os"
"testing"
"github.com/go-test/deep"
"github.com/anchore/syft/syft/pkg"
)
func assertPackagesEqual(t *testing.T, actual []pkg.Package, expected map[string]pkg.Package) {
t.Helper()
if len(actual) != len(expected) {
for _, a := range actual {
t.Log(" ", a)
}
t.Fatalf("unexpected package count: %d!=%d", len(actual), len(expected))
}
for _, a := range actual {
expectedPkg, ok := expected[a.Name]
if !ok {
t.Errorf("unexpected package found: '%s'", a.Name)
}
for _, d := range deep.Equal(a, expectedPkg) {
t.Errorf("diff: %+v", d)
}
}
}
func TestParseRequirementsTxt(t *testing.T) {
expected := map[string]pkg.Package{
"foo": {
Name: "foo",
Version: "1.0.0",
Language: pkg.Python,
Type: pkg.PythonPkg,
},
"flask": {
Name: "flask",
Version: "4.0.0",
Language: pkg.Python,
Type: pkg.PythonPkg,
},
}
fixture, err := os.Open("test-fixtures/requires/requirements.txt")
if err != nil {
t.Fatalf("failed to open fixture: %+v", err)
}
actual, err := parseRequirementsTxt(fixture.Name(), fixture)
if err != nil {
t.Fatalf("failed to parse requirements: %+v", err)
}
assertPackagesEqual(t, actual, expected)
}
syft/pkg/cataloger/python/parse_setup.go
+50
View File
@@ -0,0 +1,50 @@
package python
import (
"bufio"
"io"
"regexp"
"strings"
"github.com/anchore/syft/syft/pkg"
"github.com/anchore/syft/syft/pkg/cataloger/common"
)
// integrity check
var _ common.ParserFn = parseSetup
// match examples:
// 'pathlib3==2.2.0;python_version<"3.6"' --> match(name=pathlib3 version=2.2.0)
// "mypy==v0.770", --> match(name=mypy version=v0.770)
// " mypy2 == v0.770", ' mypy3== v0.770', --> match(name=mypy2 version=v0.770), match(name=mypy3, version=v0.770)
var pinnedDependency = regexp.MustCompile(`['"]\W?(\w+\W?==\W?[\w\.]*)`)
func parseSetup(_ string, reader io.Reader) ([]pkg.Package, error) {
packages := make([]pkg.Package, 0)
scanner := bufio.NewScanner(reader)
for scanner.Scan() {
line := scanner.Text()
line = strings.TrimRight(line, "\n")
for _, match := range pinnedDependency.FindAllString(line, -1) {
parts := strings.Split(match, "==")
if len(parts) != 2 {
continue
}
name := strings.Trim(parts[0], "'\"")
name = strings.TrimSpace(name)
version := strings.TrimSpace(parts[len(parts)-1])
packages = append(packages, pkg.Package{
Name: strings.Trim(name, "'\""),
Version: strings.Trim(version, "'\""),
Language: pkg.Python,
Type: pkg.PythonPkg,
})
}
}
return packages, nil
}
syft/pkg/cataloger/python/parse_setup_test.go
+55
View File
@@ -0,0 +1,55 @@
package python
import (
"os"
"testing"
"github.com/anchore/syft/syft/pkg"
)
func TestParseSetup(t *testing.T) {
expected := map[string]pkg.Package{
"pathlib3": {
Name: "pathlib3",
Version: "2.2.0",
Language: pkg.Python,
Type: pkg.PythonPkg,
},
"mypy": {
Name: "mypy",
Version: "v0.770",
Language: pkg.Python,
Type: pkg.PythonPkg,
},
"mypy1": {
Name: "mypy1",
Version: "v0.770",
Language: pkg.Python,
Type: pkg.PythonPkg,
},
"mypy2": {
Name: "mypy2",
Version: "v0.770",
Language: pkg.Python,
Type: pkg.PythonPkg,
},
"mypy3": {
Name: "mypy3",
Version: "v0.770",
Language: pkg.Python,
Type: pkg.PythonPkg,
},
}
fixture, err := os.Open("test-fixtures/setup/setup.py")
if err != nil {
t.Fatalf("failed to open fixture: %+v", err)
}
actual, err := parseSetup(fixture.Name(), fixture)
if err != nil {
t.Fatalf("failed to parse requirements: %+v", err)
}
assertPackagesEqual(t, actual, expected)
}
syft/pkg/cataloger/python/parse_wheel_egg_metadata.go
+112
View File
@@ -0,0 +1,112 @@
package python
import (
"bufio"
"fmt"
"io"
"path/filepath"
"strings"
"github.com/anchore/syft/internal/file"
"github.com/anchore/syft/internal/log"
"github.com/mitchellh/mapstructure"
"github.com/anchore/syft/syft/pkg"
)
// parseWheelOrEggMetadata takes a Python Egg or Wheel (which share the same format and values for our purposes),
// returning all Python packages listed.
func parseWheelOrEggMetadata(path string, reader io.Reader) (pkg.PythonPackageMetadata, error) {
fields := make(map[string]string)
var key string
scanner := bufio.NewScanner(reader)
for scanner.Scan() {
line := scanner.Text()
line = strings.TrimRight(line, "\n")
// An empty line means we are done parsing (either because there's no more data,
// or because a description follows as specified in
// https://packaging.python.org/specifications/core-metadata/#description;
// and at this time, we're not interested in the description).
if len(line) == 0 {
if len(fields) > 0 {
break
}
// however, if the field parsing has not started yet, keep scanning lines
continue
}
switch {
case strings.HasPrefix(line, " "):
// a field-body continuation
updatedValue, err := handleFieldBodyContinuation(key, line, fields)
if err != nil {
return pkg.PythonPackageMetadata{}, err
}
fields[key] = updatedValue
default:
// parse a new key (note, duplicate keys are overridden)
if i := strings.Index(line, ":"); i > 0 {
// mapstruct cannot map keys with dashes, and we are expected to persist the "Author-email" field
key = strings.ReplaceAll(strings.TrimSpace(line[0:i]), "-", "")
val := strings.TrimSpace(line[i+1:])
fields[key] = val
} else {
log.Warnf("cannot parse field from path: %q from line: %q", path, line)
}
}
}
if err := scanner.Err(); err != nil {
return pkg.PythonPackageMetadata{}, fmt.Errorf("failed to parse python wheel/egg: %w", err)
}
var metadata pkg.PythonPackageMetadata
if err := mapstructure.Decode(fields, &metadata); err != nil {
return pkg.PythonPackageMetadata{}, fmt.Errorf("unable to parse APK metadata: %w", err)
}
// add additional metadata not stored in the egg/wheel metadata file
metadata.SitePackagesRootPath = determineSitePackagesRootPath(path)
return metadata, nil
}
// isEggRegularFile determines if the specified path is the regular file variant
// of egg metadata (as opposed to a directory that contains more metadata
// files).
func isEggRegularFile(path string) bool {
return file.GlobMatch(eggFileMetadataGlob, path)
}
// determineSitePackagesRootPath returns the path of the site packages root,
// given the egg metadata file or directory specified in the path.
func determineSitePackagesRootPath(path string) string {
if isEggRegularFile(path) {
return filepath.Clean(filepath.Dir(path))
}
return filepath.Clean(filepath.Dir(filepath.Dir(path)))
}
// handleFieldBodyContinuation returns the updated value for the specified field after processing the specified line.
// If the continuation cannot be processed, it returns an error.
func handleFieldBodyContinuation(key, line string, fields map[string]string) (string, error) {
if len(key) == 0 {
return "", fmt.Errorf("no match for continuation: line: '%s'", line)
}
val, ok := fields[key]
if !ok {
return "", fmt.Errorf("no previous key exists, expecting: %s", key)
}
// concatenate onto previous value
return fmt.Sprintf("%s\n %s", val, strings.TrimSpace(line)), nil
}
syft/pkg/cataloger/python/parse_wheel_egg_metadata_test.go
+153
View File
@@ -0,0 +1,153 @@
package python
import (
"os"
"testing"
"github.com/anchore/syft/syft/pkg"
"github.com/go-test/deep"
)
func TestParseWheelEggMetadata(t *testing.T) {
tests := []struct {
Fixture string
ExpectedMetadata pkg.PythonPackageMetadata
}{
{
Fixture: "test-fixtures/egg-info/PKG-INFO",
ExpectedMetadata: pkg.PythonPackageMetadata{
Name: "requests",
Version: "2.22.0",
License: "Apache 2.0",
Platform: "UNKNOWN",
Author: "Kenneth Reitz",
AuthorEmail: "me@kennethreitz.org",
SitePackagesRootPath: "test-fixtures",
},
},
{
Fixture: "test-fixtures/dist-info/METADATA",
ExpectedMetadata: pkg.PythonPackageMetadata{
Name: "Pygments",
Version: "2.6.1",
License: "BSD License",
Platform: "any",
Author: "Georg Brandl",
AuthorEmail: "georg@python.org",
SitePackagesRootPath: "test-fixtures",
},
},
}
for _, test := range tests {
t.Run(test.Fixture, func(t *testing.T) {
fixture, err := os.Open(test.Fixture)
if err != nil {
t.Fatalf("failed to open fixture: %+v", err)
}
actual, err := parseWheelOrEggMetadata(test.Fixture, fixture)
if err != nil {
t.Fatalf("failed to parse: %+v", err)
}
for _, d := range deep.Equal(actual, test.ExpectedMetadata) {
t.Errorf("diff: %+v", d)
}
})
}
}
func TestIsRegularEggFile(t *testing.T) {
cases := []struct {
path string
expected bool
}{
{
"/usr/lib64/python2.6/site-packages/M2Crypto-0.20.2-py2.6.egg-info",
true,
},
{
"/usr/lib64/python2.6/site-packages/M2Crypto-0.20.2-py2.6.egg-info/PKG-INFO",
false,
},
{
"/usr/lib64/python2.6/site-packages/M2Crypto-0.20.2-py2.6.dist-info/METADATA",
false,
},
}
for _, c := range cases {
t.Run(c.path, func(t *testing.T) {
actual := isEggRegularFile(c.path)
if actual != c.expected {
t.Errorf("expected %t but got %t", c.expected, actual)
}
})
}
}
func TestDetermineSitePackagesRootPath(t *testing.T) {
cases := []struct {
inputPath string
expected string
}{
{
inputPath: "/usr/lib64/python2.6/site-packages/ethtool-0.6-py2.6.egg-info",
expected: "/usr/lib64/python2.6/site-packages",
},
{
inputPath: "/usr/lib/python2.7/dist-packages/configobj-5.0.6.egg-info/top_level.txt",
expected: "/usr/lib/python2.7/dist-packages",
},
{
inputPath: "/usr/lib/python2.7/dist-packages/six-1.10.0.egg-info/PKG-INFO",
expected: "/usr/lib/python2.7/dist-packages",
},
}
for _, c := range cases {
t.Run(c.inputPath, func(t *testing.T) {
actual := determineSitePackagesRootPath(c.inputPath)
if actual != c.expected {
t.Errorf("expected %s but got %s", c.expected, actual)
}
})
}
}
func TestParseWheelEggMetadataInvalid(t *testing.T) {
tests := []struct {
Fixture string
ExpectedMetadata pkg.PythonPackageMetadata
}{
{
Fixture: "test-fixtures/egg-info/PKG-INFO-INVALID",
ExpectedMetadata: pkg.PythonPackageMetadata{
Name: "mxnet",
Version: "1.8.0",
SitePackagesRootPath: "test-fixtures",
},
},
}
for _, test := range tests {
t.Run(test.Fixture, func(t *testing.T) {
fixture, err := os.Open(test.Fixture)
if err != nil {
t.Fatalf("failed to open fixture: %+v", err)
}
actual, err := parseWheelOrEggMetadata(test.Fixture, fixture)
if err != nil {
t.Fatalf("failed to parse: %+v", err)
}
for _, d := range deep.Equal(actual, test.ExpectedMetadata) {
t.Errorf("diff: %+v", d)
}
})
}
}
syft/pkg/cataloger/python/parse_wheel_egg_record.go
+60
View File
@@ -0,0 +1,60 @@
package python
import (
"encoding/csv"
"fmt"
"io"
"strings"
"github.com/anchore/syft/syft/pkg"
)
// parseWheelOrEggMetadata takes a Python Egg or Wheel (which share the same format and values for our purposes),
// returning all Python packages listed.
func parseWheelOrEggRecord(reader io.Reader) ([]pkg.PythonFileRecord, error) {
var records []pkg.PythonFileRecord
r := csv.NewReader(reader)
for {
recordList, err := r.Read()
if err == io.EOF {
break
}
if err != nil {
return nil, fmt.Errorf("unable to read python record file: %w", err)
}
if len(recordList) != 3 {
return nil, fmt.Errorf("python record an unexpected length=%d: %q", len(recordList), recordList)
}
var record pkg.PythonFileRecord
for idx, item := range recordList {
switch idx {
case 0:
record.Path = item
case 1:
if item == "" {
continue
}
fields := strings.Split(item, "=")
if len(fields) != 2 {
return nil, fmt.Errorf("unexpected python record digest: %q", item)
}
record.Digest = &pkg.PythonFileDigest{
Algorithm: fields[0],
Value: fields[1],
}
case 2:
record.Size = item
}
}
records = append(records, record)
}
return records, nil
}
syft/pkg/cataloger/python/parse_wheel_egg_record_test.go
+57
View File
@@ -0,0 +1,57 @@
package python
import (
"os"
"testing"
"github.com/anchore/syft/syft/pkg"
"github.com/go-test/deep"
)
func TestParseWheelEggRecord(t *testing.T) {
tests := []struct {
Fixture string
ExpectedMetadata []pkg.PythonFileRecord
}{
{
Fixture: "test-fixtures/egg-info/RECORD",
ExpectedMetadata: []pkg.PythonFileRecord{
{Path: "requests-2.22.0.dist-info/INSTALLER", Digest: &pkg.PythonFileDigest{"sha256", "zuuue4knoyJ-UwPPXg8fezS7VCrXJQrAP7zeNuwvFQg"}, Size: "4"},
{Path: "requests/__init__.py", Digest: &pkg.PythonFileDigest{"sha256", "PnKCgjcTq44LaAMzB-7--B2FdewRrE8F_vjZeaG9NhA"}, Size: "3921"},
{Path: "requests/__pycache__/__version__.cpython-38.pyc"},
{Path: "requests/__pycache__/utils.cpython-38.pyc"},
{Path: "requests/__version__.py", Digest: &pkg.PythonFileDigest{"sha256", "Bm-GFstQaFezsFlnmEMrJDe8JNROz9n2XXYtODdvjjc"}, Size: "436"},
{Path: "requests/utils.py", Digest: &pkg.PythonFileDigest{"sha256", "LtPJ1db6mJff2TJSJWKi7rBpzjPS3mSOrjC9zRhoD3A"}, Size: "30049"},
},
},
{
Fixture: "test-fixtures/dist-info/RECORD",
ExpectedMetadata: []pkg.PythonFileRecord{
{Path: "../../../bin/pygmentize", Digest: &pkg.PythonFileDigest{"sha256", "dDhv_U2jiCpmFQwIRHpFRLAHUO4R1jIJPEvT_QYTFp8"}, Size: "220"},
{Path: "Pygments-2.6.1.dist-info/AUTHORS", Digest: &pkg.PythonFileDigest{"sha256", "PVpa2_Oku6BGuiUvutvuPnWGpzxqFy2I8-NIrqCvqUY"}, Size: "8449"},
{Path: "Pygments-2.6.1.dist-info/RECORD"},
{Path: "pygments/__pycache__/__init__.cpython-38.pyc"},
{Path: "pygments/util.py", Digest: &pkg.PythonFileDigest{"sha256", "586xXHiJGGZxqk5PMBu3vBhE68DLuAe5MBARWrSPGxA"}, Size: "10778"},
},
},
}
for _, test := range tests {
t.Run(test.Fixture, func(t *testing.T) {
fixture, err := os.Open(test.Fixture)
if err != nil {
t.Fatalf("failed to open fixture: %+v", err)
}
actual, err := parseWheelOrEggRecord(fixture)
if err != nil {
t.Fatalf("failed to parse: %+v", err)
}
for _, d := range deep.Equal(actual, test.ExpectedMetadata) {
t.Errorf("diff: %+v", d)
}
})
}
}
syft/pkg/cataloger/python/poetry_metadata.go
+18
View File
@@ -0,0 +1,18 @@
package python
import "github.com/anchore/syft/syft/pkg"
type PoetryMetadata struct {
Packages []PoetryMetadataPackage `toml:"package"`
}
// Pkgs returns all of the packages referenced within the poetry.lock metadata.
func (m PoetryMetadata) Pkgs() []pkg.Package {
pkgs := make([]pkg.Package, 0)
for _, p := range m.Packages {
pkgs = append(pkgs, p.Pkg())
}
return pkgs
}
syft/pkg/cataloger/python/poetry_metadata_package.go
+21
View File
@@ -0,0 +1,21 @@
package python
import "github.com/anchore/syft/syft/pkg"
type PoetryMetadataPackage struct {
Name string `toml:"name"`
Version string `toml:"version"`
Category string `toml:"category"`
Description string `toml:"description"`
Optional bool `toml:"optional"`
}
// Pkg returns the standard `pkg.Package` representation of the package referenced within the poetry.lock metadata.
func (p PoetryMetadataPackage) Pkg() pkg.Package {
return pkg.Package{
Name: p.Name,
Version: p.Version,
Language: pkg.Python,
Type: pkg.PythonPkg,
}
}
syft/pkg/cataloger/python/test-fixtures/Python-2.7.egg-info
+31
View File
@@ -0,0 +1,31 @@
Metadata-Version: 1.1
Name: Python
Version: 2.7.13
Summary: A high-level object-oriented programming language
Home-page: http://www.python.org/2.7
Author: Guido van Rossum and the Python community
Author-email: python-dev@python.org
License: PSF license
Description: Python is an interpreted, interactive, object-oriented programming
language. It is often compared to Tcl, Perl, Scheme or Java.
Python combines remarkable power with very clear syntax. It has
modules, classes, exceptions, very high level dynamic data types, and
dynamic typing. There are interfaces to many system calls and
libraries, as well as to various windowing systems (X11, Motif, Tk,
Mac, MFC). New built-in modules are easily written in C or C++. Python
is also usable as an extension language for applications that need a
programmable interface.
The Python implementation is portable: it runs on many brands of UNIX,
on Windows, DOS, OS/2, Mac, Amiga... If your favorite system isn't
listed here, it may still be supported, if there's a C compiler for
it. Ask around on comp.lang.python -- or just try compiling Python
yourself.
Platform: Many
Classifier: Development Status :: 6 - Mature
Classifier: License :: OSI Approved :: Python Software Foundation License
Classifier: Natural Language :: English
Classifier: Programming Language :: C
Classifier: Programming Language :: Python
Classifier: Topic :: Software Development
syft/pkg/cataloger/python/test-fixtures/dist-info/METADATA
+47
View File
@@ -0,0 +1,47 @@
Metadata-Version: 2.1
Name: Pygments
Version: 2.6.1
Summary: Pygments is a syntax highlighting package written in Python.
Home-page: https://pygments.org/
Author: Georg Brandl
Author-email: georg@python.org
License: BSD License
Keywords: syntax highlighting
Platform: any
Classifier: License :: OSI Approved :: BSD License
Classifier: Intended Audience :: Developers
Classifier: Intended Audience :: End Users/Desktop
Classifier: Intended Audience :: System Administrators
Classifier: Development Status :: 6 - Mature
Classifier: Programming Language :: Python
Classifier: Programming Language :: Python :: 3
Classifier: Programming Language :: Python :: 3.5
Classifier: Programming Language :: Python :: 3.6
Classifier: Programming Language :: Python :: 3.7
Classifier: Programming Language :: Python :: 3.8
Classifier: Programming Language :: Python :: Implementation :: CPython
Classifier: Programming Language :: Python :: Implementation :: PyPy
Classifier: Operating System :: OS Independent
Classifier: Topic :: Text Processing :: Filters
Classifier: Topic :: Utilities
Requires-Python: >=3.5
Pygments
~~~~~~~~
Pygments is a syntax highlighting package written in Python.
It is a generic syntax highlighter suitable for use in code hosting, forums,
wikis or other applications that need to prettify source code. Highlights
are:
* a wide range of over 500 languages and other text formats is supported
* special attention is paid to details, increasing quality by a fair amount
* support for new languages and formats are added easily
* a number of output formats, presently HTML, LaTeX, RTF, SVG, all image formats that PIL supports and ANSI sequences
* it is usable as a command-line tool and as a library
:copyright: Copyright 2006-2019 by the Pygments team, see AUTHORS.
:license: BSD, see LICENSE for details.
syft/pkg/cataloger/python/test-fixtures/dist-info/RECORD
+5
View File
@@ -0,0 +1,5 @@
../../../bin/pygmentize,sha256=dDhv_U2jiCpmFQwIRHpFRLAHUO4R1jIJPEvT_QYTFp8,220
Pygments-2.6.1.dist-info/AUTHORS,sha256=PVpa2_Oku6BGuiUvutvuPnWGpzxqFy2I8-NIrqCvqUY,8449
Pygments-2.6.1.dist-info/RECORD,,
pygments/__pycache__/__init__.cpython-38.pyc,,
pygments/util.py,sha256=586xXHiJGGZxqk5PMBu3vBhE68DLuAe5MBARWrSPGxA,10778
syft/pkg/cataloger/python/test-fixtures/dist-info/top_level.txt
+2
View File
@@ -0,0 +1,2 @@
pygments
something_else
syft/pkg/cataloger/python/test-fixtures/egg-info/PKG-INFO
+134
View File
@@ -0,0 +1,134 @@
Metadata-Version: 2.1
Name: requests
Version: 2.22.0
Summary: Python HTTP for Humans.
Home-page: http://python-requests.org
Author: Kenneth Reitz
Author-email: me@kennethreitz.org
License: Apache 2.0
Description: Requests: HTTP for Humans™
==========================
[![image](https://img.shields.io/pypi/v/requests.svg)](https://pypi.org/project/requests/)
[![image](https://img.shields.io/pypi/l/requests.svg)](https://pypi.org/project/requests/)
[![image](https://img.shields.io/pypi/pyversions/requests.svg)](https://pypi.org/project/requests/)
[![codecov.io](https://codecov.io/github/requests/requests/coverage.svg?branch=master)](https://codecov.io/github/requests/requests)
[![image](https://img.shields.io/github/contributors/requests/requests.svg)](https://github.com/requests/requests/graphs/contributors)
[![image](https://img.shields.io/badge/Say%20Thanks-!-1EAEDB.svg)](https://saythanks.io/to/kennethreitz)
Requests is the only *Non-GMO* HTTP library for Python, safe for human
consumption.
![image](https://farm5.staticflickr.com/4317/35198386374_1939af3de6_k_d.jpg)
Behold, the power of Requests:
``` {.sourceCode .python}
>>> import requests
>>> r = requests.get('https://api.github.com/user', auth=('user', 'pass'))
>>> r.status_code
200
>>> r.headers['content-type']
'application/json; charset=utf8'
>>> r.encoding
'utf-8'
>>> r.text
u'{"type":"User"...'
>>> r.json()
{u'disk_usage': 368627, u'private_gists': 484, ...}
```
See [the similar code, sans Requests](https://gist.github.com/973705).
[![image](https://raw.githubusercontent.com/requests/requests/master/docs/_static/requests-logo-small.png)](http://docs.python-requests.org/)
Requests allows you to send *organic, grass-fed* HTTP/1.1 requests,
without the need for manual labor. There's no need to manually add query
strings to your URLs, or to form-encode your POST data. Keep-alive and
HTTP connection pooling are 100% automatic, thanks to
[urllib3](https://github.com/shazow/urllib3).
Besides, all the cool kids are doing it. Requests is one of the most
downloaded Python packages of all time, pulling in over 11,000,000
downloads every month. You don't want to be left out!
Feature Support
---------------
Requests is ready for today's web.
- International Domains and URLs
- Keep-Alive & Connection Pooling
- Sessions with Cookie Persistence
- Browser-style SSL Verification
- Basic/Digest Authentication
- Elegant Key/Value Cookies
- Automatic Decompression
- Automatic Content Decoding
- Unicode Response Bodies
- Multipart File Uploads
- HTTP(S) Proxy Support
- Connection Timeouts
- Streaming Downloads
- `.netrc` Support
- Chunked Requests
Requests officially supports Python 2.7 & 3.4–3.7, and runs great on
PyPy.
Installation
------------
To install Requests, simply use [pipenv](http://pipenv.org/) (or pip, of
course):
``` {.sourceCode .bash}
$ pipenv install requests
✨🍰✨
```
Satisfaction guaranteed.
Documentation
-------------
Fantastic documentation is available at
<http://docs.python-requests.org/>, for a limited time only.
How to Contribute
-----------------
1. Become more familiar with the project by reading our [Contributor's Guide](http://docs.python-requests.org/en/latest/dev/contributing/) and our [development philosophy](http://docs.python-requests.org/en/latest/dev/philosophy/).
2. Check for open issues or open a fresh issue to start a discussion
around a feature idea or a bug. There is a [Contributor
Friendly](https://github.com/requests/requests/issues?direction=desc&labels=Contributor+Friendly&page=1&sort=updated&state=open)
tag for issues that should be ideal for people who are not very
familiar with the codebase yet.
3. Fork [the repository](https://github.com/requests/requests) on
GitHub to start making your changes to the **master** branch (or
branch off of it).
4. Write a test which shows that the bug was fixed or that the feature
works as expected.
5. Send a pull request and bug the maintainer until it gets merged and
published. :) Make sure to add yourself to
[AUTHORS](https://github.com/requests/requests/blob/master/AUTHORS.rst).
Platform: UNKNOWN
Classifier: Development Status :: 5 - Production/Stable
Classifier: Intended Audience :: Developers
Classifier: Natural Language :: English
Classifier: License :: OSI Approved :: Apache Software License
Classifier: Programming Language :: Python
Classifier: Programming Language :: Python :: 2
Classifier: Programming Language :: Python :: 2.7
Classifier: Programming Language :: Python :: 3
Classifier: Programming Language :: Python :: 3.5
Classifier: Programming Language :: Python :: 3.6
Classifier: Programming Language :: Python :: 3.7
Classifier: Programming Language :: Python :: Implementation :: CPython
Classifier: Programming Language :: Python :: Implementation :: PyPy
Requires-Python: >=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*
Description-Content-Type: text/markdown
Provides-Extra: security
Provides-Extra: socks
syft/pkg/cataloger/python/test-fixtures/egg-info/PKG-INFO-INVALID
+69
View File
@@ -0,0 +1,69 @@
Metadata-Version: 1.1
Name: mxnet
Version: 1.8.0
Summary: <!--- Licensed to the Apache Software Foundation (ASF) under one -->
<!--- or more contributor license agreements. See the NOTICE file -->
<!--- distributed with this work for additional information -->
<!--- regarding copyright ownership. The ASF licenses this file -->
<!--- to you under the Apache License, Version 2.0 (the -->
<!--- "License"); you may not use this file except in compliance -->
<!--- with the License. You may obtain a copy of the License at -->
<!--- http://www.apache.org/licenses/LICENSE-2.0 -->
<!--- Unless required by applicable law or agreed to in writing, -->
<!--- software distributed under the License is distributed on an -->
<!--- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -->
<!--- KIND, either express or implied. See the License for the -->
<!--- specific language governing permissions and limitations -->
<!--- under the License. -->
MXNet Python Package
====================
This directory and nested files contain MXNet Python package and language binding.
## Installation
To install MXNet Python package, visit MXNet [Install Instruction](https://mxnet.apache.org/get_started)
## Running the unit tests
For running unit tests, you will need the [nose PyPi package](https://pypi.python.org/pypi/nose). To install:
```bash
pip install --upgrade nose
```
Once ```nose``` is installed, run the following from MXNet root directory (please make sure the installation path of ```nosetests``` is included in your ```$PATH``` environment variable):
```
nosetests tests/python/unittest
nosetests tests/python/train
```
Home-page: https://github.com/apache/incubator-mxnet
Author: UNKNOWN
Author-email: UNKNOWN
License: UNKNOWN
Description: UNKNOWN
Platform: UNKNOWN
Classifier: Development Status :: 5 - Production/Stable
Classifier: Intended Audience :: Developers
Classifier: Intended Audience :: Education
Classifier: Intended Audience :: Science/Research
Classifier: License :: OSI Approved :: Apache Software License
Classifier: Programming Language :: C++
Classifier: Programming Language :: Cython
Classifier: Programming Language :: Other
Classifier: Programming Language :: Perl
Classifier: Programming Language :: Python
Classifier: Programming Language :: Python :: 2.7
Classifier: Programming Language :: Python :: 3.4
Classifier: Programming Language :: Python :: 3.5
Classifier: Programming Language :: Python :: 3.6
Classifier: Programming Language :: Python :: Implementation :: CPython
Classifier: Topic :: Scientific/Engineering
Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
Classifier: Topic :: Scientific/Engineering :: Mathematics
Classifier: Topic :: Software Development
Classifier: Topic :: Software Development :: Libraries
Classifier: Topic :: Software Development :: Libraries :: Python Modules
syft/pkg/cataloger/python/test-fixtures/egg-info/RECORD
+6
View File
@@ -0,0 +1,6 @@
requests-2.22.0.dist-info/INSTALLER,sha256=zuuue4knoyJ-UwPPXg8fezS7VCrXJQrAP7zeNuwvFQg,4
requests/__init__.py,sha256=PnKCgjcTq44LaAMzB-7--B2FdewRrE8F_vjZeaG9NhA,3921
requests/__pycache__/__version__.cpython-38.pyc,,
requests/__pycache__/utils.cpython-38.pyc,,
requests/__version__.py,sha256=Bm-GFstQaFezsFlnmEMrJDe8JNROz9n2XXYtODdvjjc,436
requests/utils.py,sha256=LtPJ1db6mJff2TJSJWKi7rBpzjPS3mSOrjC9zRhoD3A,30049
syft/pkg/cataloger/python/test-fixtures/egg-info/top_level.txt
+1
View File
@@ -0,0 +1 @@
requests
syft/pkg/cataloger/python/test-fixtures/partial.dist-info/METADATA
+47
View File
@@ -0,0 +1,47 @@
Metadata-Version: 2.1
Name: Pygments
Version: 2.6.1
Summary: Pygments is a syntax highlighting package written in Python.
Home-page: https://pygments.org/
Author: Georg Brandl
Author-email: georg@python.org
License: BSD License
Keywords: syntax highlighting
Platform: any
Classifier: License :: OSI Approved :: BSD License
Classifier: Intended Audience :: Developers
Classifier: Intended Audience :: End Users/Desktop
Classifier: Intended Audience :: System Administrators
Classifier: Development Status :: 6 - Mature
Classifier: Programming Language :: Python
Classifier: Programming Language :: Python :: 3
Classifier: Programming Language :: Python :: 3.5
Classifier: Programming Language :: Python :: 3.6
Classifier: Programming Language :: Python :: 3.7
Classifier: Programming Language :: Python :: 3.8
Classifier: Programming Language :: Python :: Implementation :: CPython
Classifier: Programming Language :: Python :: Implementation :: PyPy
Classifier: Operating System :: OS Independent
Classifier: Topic :: Text Processing :: Filters
Classifier: Topic :: Utilities
Requires-Python: >=3.5
Pygments
~~~~~~~~
Pygments is a syntax highlighting package written in Python.
It is a generic syntax highlighter suitable for use in code hosting, forums,
wikis or other applications that need to prettify source code. Highlights
are:
* a wide range of over 500 languages and other text formats is supported
* special attention is paid to details, increasing quality by a fair amount
* support for new languages and formats are added easily
* a number of output formats, presently HTML, LaTeX, RTF, SVG, all image formats that PIL supports and ANSI sequences
* it is usable as a command-line tool and as a library
:copyright: Copyright 2006-2019 by the Pygments team, see AUTHORS.
:license: BSD, see LICENSE for details.
syft/pkg/cataloger/python/test-fixtures/poetry/poetry.lock
Generated
+64
View File
@@ -0,0 +1,64 @@
[[package]]
category = "dev"
description = "Sphinx \"added-value\" extension"
name = "added-value"
optional = false
python-versions = "*"
version = "0.14.2"
[package.dependencies]
docutils = "*"
natsort = "*"
six = "*"
sphinx = "*"
[package.extras]
deploy = ["bumpversion", "twine", "wheel"]
docs = ["sphinx", "sphinx-rtd-theme"]
test = ["pytest", "pytest-cov", "coveralls", "beautifulsoup4", "hypothesis"]
[[package]]
category = "dev"
description = "A configurable sidebar-enabled Sphinx theme"
name = "alabaster"
optional = false
python-versions = "*"
version = "0.7.12"
[[package]]
category = "dev"
description = "Disable App Nap on OS X 10.9"
marker = "python_version >= \"3.4\" and sys_platform == \"darwin\" or sys_platform == \"darwin\""
name = "appnope"
optional = false
python-versions = "*"
version = "0.1.0"
[[package]]
category = "dev"
description = "Draws ASCII trees."
name = "asciitree"
optional = false
python-versions = "*"
version = "0.3.3"
[metadata]
content-hash = "6e35f765c2f01c635c2fb0d54a9d7dd68742350f04449ee24efad03e3c9eb0bb"
python-versions = "^3.6"
[metadata.files]
added-value = [
{file = "added-value-0.14.2.tar.gz", hash = "sha256:8c886aee74635cec15beb64c28a90ae664526f331c1a4941e4d6ab98af232028"},
{file = "added_value-0.14.2-py2.py3-none-any.whl", hash = "sha256:b25fcb86f9bfad9a40adf4d344322690e312741556c7b75681bc948380a251e6"},
]
alabaster = [
{file = "alabaster-0.7.12-py2.py3-none-any.whl", hash = "sha256:446438bdcca0e05bd45ea2de1668c1d9b032e1a9154c2c259092d77031ddd359"},
{file = "alabaster-0.7.12.tar.gz", hash = "sha256:a661d72d58e6ea8a57f7a86e37d86716863ee5e92788398526d58b26a4e4dc02"},
]
appnope = [
{file = "appnope-0.1.0-py2.py3-none-any.whl", hash = "sha256:5b26757dc6f79a3b7dc9fab95359328d5747fcb2409d331ea66d0272b90ab2a0"},
{file = "appnope-0.1.0.tar.gz", hash = "sha256:8b995ffe925347a2138d7ac0fe77155e4311a0ea6d6da4f5128fe4b3cbe5ed71"},
]
asciitree = [
{file = "asciitree-0.3.3.tar.gz", hash = "sha256:4aa4b9b649f85e3fcb343363d97564aa1fb62e249677f2e18a96765145cc0f6e"},
]
syft/pkg/cataloger/python/test-fixtures/requires/requirements.txt
+7
View File
@@ -0,0 +1,7 @@
flask == 4.0.0
# a line that is ignored
sqlalchemy >= 1.0.0
foo == 1.0.0 # a comment that needs to be ignored
-e https://github.com/pecan/pecan.git
-r other-requirements.txt
--requirements super-secretrequirements.txt
syft/pkg/cataloger/python/test-fixtures/setup/setup.py
+46
View File
@@ -0,0 +1,46 @@
from setuptools import setup
# Sample setup.py from the pytest project with added comments specific
# to the cataloger
INSTALL_REQUIRES = [
"py>=1.5.0",
"packaging",
"attrs>=17.4.0",
"more-itertools>=4.0.0",
'atomicwrites>=1.0;sys_platform=="win32"', # sys_platform is ignored
'pathlib2>=2.2.0;python_version=="3.6"', # python_version is ignored
'pathlib3==2.2.0;python_version<"3.6"', # this is caught
'colorama;sys_platform=="win32"',
"pluggy>=0.12,<1.0",
'importlib-metadata>=0.12;python_version<"3.8"',
"wcwidth",
]
def main():
setup(
use_scm_version={"write_to": "src/_pytest/_version.py"},
setup_requires=["setuptools-scm", "setuptools>=40.0"],
package_dir={"": "src"},
extras_require={
"testing": [
"argcomplete",
"hypothesis>=3.56",
"mock",
"nose",
"requests",
"xmlschema",
],
"checkqa-mypy": [
"mypy==v0.770", # this is caught
" mypy1==v0.770", # this is caught
" mypy2 == v0.770", ' mypy3== v0.770', # this is caught
],
},
install_requires=INSTALL_REQUIRES,
)
if __name__ == "__main__":
main()
syft/pkg/cataloger/python/test-fixtures/test.egg-info
+134
View File
@@ -0,0 +1,134 @@
Metadata-Version: 2.1
Name: requests
Version: 2.22.0
Summary: Python HTTP for Humans.
Home-page: http://python-requests.org
Author: Kenneth Reitz
Author-email: me@kennethreitz.org
License: Apache 2.0
Description: Requests: HTTP for Humans™
==========================
[![image](https://img.shields.io/pypi/v/requests.svg)](https://pypi.org/project/requests/)
[![image](https://img.shields.io/pypi/l/requests.svg)](https://pypi.org/project/requests/)
[![image](https://img.shields.io/pypi/pyversions/requests.svg)](https://pypi.org/project/requests/)
[![codecov.io](https://codecov.io/github/requests/requests/coverage.svg?branch=master)](https://codecov.io/github/requests/requests)
[![image](https://img.shields.io/github/contributors/requests/requests.svg)](https://github.com/requests/requests/graphs/contributors)
[![image](https://img.shields.io/badge/Say%20Thanks-!-1EAEDB.svg)](https://saythanks.io/to/kennethreitz)
Requests is the only *Non-GMO* HTTP library for Python, safe for human
consumption.
![image](https://farm5.staticflickr.com/4317/35198386374_1939af3de6_k_d.jpg)
Behold, the power of Requests:
``` {.sourceCode .python}
>>> import requests
>>> r = requests.get('https://api.github.com/user', auth=('user', 'pass'))
>>> r.status_code
200
>>> r.headers['content-type']
'application/json; charset=utf8'
>>> r.encoding
'utf-8'
>>> r.text
u'{"type":"User"...'
>>> r.json()
{u'disk_usage': 368627, u'private_gists': 484, ...}
```
See [the similar code, sans Requests](https://gist.github.com/973705).
[![image](https://raw.githubusercontent.com/requests/requests/master/docs/_static/requests-logo-small.png)](http://docs.python-requests.org/)
Requests allows you to send *organic, grass-fed* HTTP/1.1 requests,
without the need for manual labor. There's no need to manually add query
strings to your URLs, or to form-encode your POST data. Keep-alive and
HTTP connection pooling are 100% automatic, thanks to
[urllib3](https://github.com/shazow/urllib3).
Besides, all the cool kids are doing it. Requests is one of the most
downloaded Python packages of all time, pulling in over 11,000,000
downloads every month. You don't want to be left out!
Feature Support
---------------
Requests is ready for today's web.
- International Domains and URLs
- Keep-Alive & Connection Pooling
- Sessions with Cookie Persistence
- Browser-style SSL Verification
- Basic/Digest Authentication
- Elegant Key/Value Cookies
- Automatic Decompression
- Automatic Content Decoding
- Unicode Response Bodies
- Multipart File Uploads
- HTTP(S) Proxy Support
- Connection Timeouts
- Streaming Downloads
- `.netrc` Support
- Chunked Requests
Requests officially supports Python 2.7 & 3.4–3.7, and runs great on
PyPy.
Installation
------------
To install Requests, simply use [pipenv](http://pipenv.org/) (or pip, of
course):
``` {.sourceCode .bash}
$ pipenv install requests
✨🍰✨
```
Satisfaction guaranteed.
Documentation
-------------
Fantastic documentation is available at
<http://docs.python-requests.org/>, for a limited time only.
How to Contribute
-----------------
1. Become more familiar with the project by reading our [Contributor's Guide](http://docs.python-requests.org/en/latest/dev/contributing/) and our [development philosophy](http://docs.python-requests.org/en/latest/dev/philosophy/).
2. Check for open issues or open a fresh issue to start a discussion
around a feature idea or a bug. There is a [Contributor
Friendly](https://github.com/requests/requests/issues?direction=desc&labels=Contributor+Friendly&page=1&sort=updated&state=open)
tag for issues that should be ideal for people who are not very
familiar with the codebase yet.
3. Fork [the repository](https://github.com/requests/requests) on
GitHub to start making your changes to the **master** branch (or
branch off of it).
4. Write a test which shows that the bug was fixed or that the feature
works as expected.
5. Send a pull request and bug the maintainer until it gets merged and
published. :) Make sure to add yourself to
[AUTHORS](https://github.com/requests/requests/blob/master/AUTHORS.rst).
Platform: UNKNOWN
Classifier: Development Status :: 5 - Production/Stable
Classifier: Intended Audience :: Developers
Classifier: Natural Language :: English
Classifier: License :: OSI Approved :: Apache Software License
Classifier: Programming Language :: Python
Classifier: Programming Language :: Python :: 2
Classifier: Programming Language :: Python :: 2.7
Classifier: Programming Language :: Python :: 3
Classifier: Programming Language :: Python :: 3.5
Classifier: Programming Language :: Python :: 3.6
Classifier: Programming Language :: Python :: 3.7
Classifier: Programming Language :: Python :: Implementation :: CPython
Classifier: Programming Language :: Python :: Implementation :: PyPy
Requires-Python: >=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*
Description-Content-Type: text/markdown
Provides-Extra: security
Provides-Extra: socks