mirror of
https://github.com/anchore/syft.git
synced 2025-11-17 08:23:15 +01:00
add top_level.txt processing to python package cataloger
Signed-off-by: Alex Goodman <alex.goodman@anchore.com>
This commit is contained in:
parent
1414d1fbc3
commit
2e5ff4a995
@ -1,6 +1,7 @@
|
|||||||
package python
|
package python
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"bufio"
|
||||||
"fmt"
|
"fmt"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
"strings"
|
"strings"
|
||||||
@ -53,17 +54,17 @@ func (c *PackageCataloger) Catalog(resolver scope.Resolver) ([]pkg.Package, erro
|
|||||||
return pkgs, nil
|
return pkgs, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (c *PackageCataloger) catalogEggOrWheel(resolver scope.Resolver, metadataRef file.Reference) (*pkg.Package, error) {
|
func (c *PackageCataloger) assembleEggOrWheelMetadata(resolver scope.Resolver, metadataRef file.Reference) (*pkg.PythonPackageMetadata, []file.Reference, error) {
|
||||||
var sources = []file.Reference{metadataRef}
|
var sources = []file.Reference{metadataRef}
|
||||||
|
|
||||||
metadataContents, err := resolver.FileContentsByRef(metadataRef)
|
metadataContents, err := resolver.FileContentsByRef(metadataRef)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, nil, err
|
||||||
}
|
}
|
||||||
|
|
||||||
metadata, err := parseWheelOrEggMetadata(strings.NewReader(metadataContents))
|
metadata, err := parseWheelOrEggMetadata(metadataRef.Path, strings.NewReader(metadataContents))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, nil, err
|
||||||
}
|
}
|
||||||
|
|
||||||
// we've been given a file reference to a specific wheel METADATA file. note: this may be for a directory
|
// we've been given a file reference to a specific wheel METADATA file. note: this may be for a directory
|
||||||
@ -74,7 +75,7 @@ func (c *PackageCataloger) catalogEggOrWheel(resolver scope.Resolver, metadataRe
|
|||||||
recordPath := filepath.Join(filepath.Dir(string(metadataRef.Path)), "RECORD")
|
recordPath := filepath.Join(filepath.Dir(string(metadataRef.Path)), "RECORD")
|
||||||
recordRef, err := resolver.RelativeFileByPath(metadataRef, recordPath)
|
recordRef, err := resolver.RelativeFileByPath(metadataRef, recordPath)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, nil, err
|
||||||
}
|
}
|
||||||
|
|
||||||
if recordRef != nil {
|
if recordRef != nil {
|
||||||
@ -82,20 +83,56 @@ func (c *PackageCataloger) catalogEggOrWheel(resolver scope.Resolver, metadataRe
|
|||||||
|
|
||||||
recordContents, err := resolver.FileContentsByRef(*recordRef)
|
recordContents, err := resolver.FileContentsByRef(*recordRef)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, nil, err
|
||||||
}
|
}
|
||||||
|
|
||||||
// parse the record contents
|
// parse the record contents
|
||||||
records, err := parseWheelOrEggRecord(strings.NewReader(recordContents))
|
records, err := parseWheelOrEggRecord(strings.NewReader(recordContents))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, nil, err
|
||||||
}
|
}
|
||||||
|
|
||||||
// append the record files list to the metadata
|
// append the record files list to the metadata
|
||||||
metadata.Files = records
|
metadata.Files = records
|
||||||
}
|
}
|
||||||
|
|
||||||
// assemble the package
|
// a top_level.txt file specifies the python top-level packages (provided by this python package) installed into site-packages
|
||||||
|
parentDir := filepath.Dir(string(metadataRef.Path))
|
||||||
|
topLevelPath := filepath.Join(parentDir, "top_level.txt")
|
||||||
|
topLevelRef, err := resolver.RelativeFileByPath(metadataRef, topLevelPath)
|
||||||
|
if err != nil {
|
||||||
|
return nil, nil, err
|
||||||
|
}
|
||||||
|
if topLevelRef == nil {
|
||||||
|
return nil, nil, fmt.Errorf("missing python package top_level.txt (package=%q)", string(metadataRef.Path))
|
||||||
|
}
|
||||||
|
|
||||||
|
topLevelContents, err := resolver.FileContentsByRef(*topLevelRef)
|
||||||
|
if err != nil {
|
||||||
|
return nil, nil, err
|
||||||
|
}
|
||||||
|
// nolint:prealloc
|
||||||
|
var topLevelPackages []string
|
||||||
|
scanner := bufio.NewScanner(strings.NewReader(topLevelContents))
|
||||||
|
for scanner.Scan() {
|
||||||
|
topLevelPackages = append(topLevelPackages, scanner.Text())
|
||||||
|
}
|
||||||
|
|
||||||
|
if err := scanner.Err(); err != nil {
|
||||||
|
return nil, nil, fmt.Errorf("could not read python package top_level.txt: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
metadata.TopLevelPackages = topLevelPackages
|
||||||
|
|
||||||
|
return &metadata, sources, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *PackageCataloger) catalogEggOrWheel(resolver scope.Resolver, metadataRef file.Reference) (*pkg.Package, error) {
|
||||||
|
|
||||||
|
metadata, sources, err := c.assembleEggOrWheelMetadata(resolver, metadataRef)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
var licenses []string
|
var licenses []string
|
||||||
if metadata.License != "" {
|
if metadata.License != "" {
|
||||||
@ -111,6 +148,6 @@ func (c *PackageCataloger) catalogEggOrWheel(resolver scope.Resolver, metadataRe
|
|||||||
Language: pkg.Python,
|
Language: pkg.Python,
|
||||||
Type: pkg.PythonPkg,
|
Type: pkg.PythonPkg,
|
||||||
MetadataType: pkg.PythonPackageMetadataType,
|
MetadataType: pkg.PythonPackageMetadataType,
|
||||||
Metadata: metadata,
|
Metadata: *metadata,
|
||||||
}, nil
|
}, nil
|
||||||
}
|
}
|
||||||
|
|||||||
@ -5,6 +5,7 @@ import (
|
|||||||
"io"
|
"io"
|
||||||
"io/ioutil"
|
"io/ioutil"
|
||||||
"os"
|
"os"
|
||||||
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
"github.com/anchore/stereoscope/pkg/file"
|
"github.com/anchore/stereoscope/pkg/file"
|
||||||
@ -16,29 +17,65 @@ import (
|
|||||||
type pythonTestResolverMock struct {
|
type pythonTestResolverMock struct {
|
||||||
metadataReader io.Reader
|
metadataReader io.Reader
|
||||||
recordReader io.Reader
|
recordReader io.Reader
|
||||||
|
topLevelReader io.Reader
|
||||||
metadataRef *file.Reference
|
metadataRef *file.Reference
|
||||||
recordRef *file.Reference
|
recordRef *file.Reference
|
||||||
|
topLevelRef *file.Reference
|
||||||
contents map[file.Reference]string
|
contents map[file.Reference]string
|
||||||
}
|
}
|
||||||
|
|
||||||
func newTestResolver(recordReader, metadataReader io.Reader) *pythonTestResolverMock {
|
func newTestResolver(metaPath, recordPath, topPath string) *pythonTestResolverMock {
|
||||||
|
metadataReader, err := os.Open(metaPath)
|
||||||
|
if err != nil {
|
||||||
|
panic(fmt.Errorf("failed to open metadata: %+v", err))
|
||||||
|
}
|
||||||
|
|
||||||
|
var recordReader io.Reader
|
||||||
|
if recordPath != "" {
|
||||||
|
recordReader, err = os.Open(recordPath)
|
||||||
|
if err != nil {
|
||||||
|
panic(fmt.Errorf("failed to open record: %+v", err))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
var topLevelReader io.Reader
|
||||||
|
if topPath != "" {
|
||||||
|
topLevelReader, err = os.Open(topPath)
|
||||||
|
if err != nil {
|
||||||
|
panic(fmt.Errorf("failed to open top level: %+v", err))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
var recordRef *file.Reference
|
var recordRef *file.Reference
|
||||||
if recordReader != nil {
|
if recordReader != nil {
|
||||||
ref := file.NewFileReference("record-path")
|
ref := file.NewFileReference("test-fixtures/dist-info/RECORD")
|
||||||
recordRef = &ref
|
recordRef = &ref
|
||||||
}
|
}
|
||||||
metadataRef := file.NewFileReference("metadata-path")
|
var topLevelRef *file.Reference
|
||||||
|
if topLevelReader != nil {
|
||||||
|
ref := file.NewFileReference("test-fixtures/dist-info/top_level.txt")
|
||||||
|
topLevelRef = &ref
|
||||||
|
}
|
||||||
|
metadataRef := file.NewFileReference("test-fixtures/dist-info/METADATA")
|
||||||
return &pythonTestResolverMock{
|
return &pythonTestResolverMock{
|
||||||
recordReader: recordReader,
|
recordReader: recordReader,
|
||||||
metadataReader: metadataReader,
|
metadataReader: metadataReader,
|
||||||
|
topLevelReader: topLevelReader,
|
||||||
metadataRef: &metadataRef,
|
metadataRef: &metadataRef,
|
||||||
recordRef: recordRef,
|
recordRef: recordRef,
|
||||||
|
topLevelRef: topLevelRef,
|
||||||
contents: make(map[file.Reference]string),
|
contents: make(map[file.Reference]string),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func (r *pythonTestResolverMock) FileContentsByRef(ref file.Reference) (string, error) {
|
func (r *pythonTestResolverMock) FileContentsByRef(ref file.Reference) (string, error) {
|
||||||
switch ref.Path {
|
switch ref.Path {
|
||||||
|
case r.topLevelRef.Path:
|
||||||
|
b, err := ioutil.ReadAll(r.topLevelReader)
|
||||||
|
if err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
return string(b), nil
|
||||||
case r.metadataRef.Path:
|
case r.metadataRef.Path:
|
||||||
b, err := ioutil.ReadAll(r.metadataReader)
|
b, err := ioutil.ReadAll(r.metadataReader)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@ -66,12 +103,14 @@ func (r *pythonTestResolverMock) FilesByPath(_ ...file.Path) ([]file.Reference,
|
|||||||
func (r *pythonTestResolverMock) FilesByGlob(_ ...string) ([]file.Reference, error) {
|
func (r *pythonTestResolverMock) FilesByGlob(_ ...string) ([]file.Reference, error) {
|
||||||
return nil, fmt.Errorf("not implemented")
|
return nil, fmt.Errorf("not implemented")
|
||||||
}
|
}
|
||||||
func (r *pythonTestResolverMock) RelativeFileByPath(reference file.Reference, _ string) (*file.Reference, error) {
|
func (r *pythonTestResolverMock) RelativeFileByPath(_ file.Reference, path string) (*file.Reference, error) {
|
||||||
switch reference.Path {
|
switch {
|
||||||
case r.metadataRef.Path:
|
case strings.Contains(path, "RECORD"):
|
||||||
return r.recordRef, nil
|
return r.recordRef, nil
|
||||||
|
case strings.Contains(path, "top_level.txt"):
|
||||||
|
return r.topLevelRef, nil
|
||||||
default:
|
default:
|
||||||
return nil, fmt.Errorf("invalid value given")
|
return nil, fmt.Errorf("invalid RelativeFileByPath value given: %q", path)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -79,11 +118,13 @@ func TestPythonPackageWheelCataloger(t *testing.T) {
|
|||||||
tests := []struct {
|
tests := []struct {
|
||||||
MetadataFixture string
|
MetadataFixture string
|
||||||
RecordFixture string
|
RecordFixture string
|
||||||
|
TopLevelFixture string
|
||||||
ExpectedPackage pkg.Package
|
ExpectedPackage pkg.Package
|
||||||
}{
|
}{
|
||||||
{
|
{
|
||||||
MetadataFixture: "test-fixtures/egg-info/PKG-INFO",
|
MetadataFixture: "test-fixtures/egg-info/PKG-INFO",
|
||||||
RecordFixture: "test-fixtures/egg-info/RECORD",
|
RecordFixture: "test-fixtures/egg-info/RECORD",
|
||||||
|
TopLevelFixture: "test-fixtures/egg-info/top_level.txt",
|
||||||
ExpectedPackage: pkg.Package{
|
ExpectedPackage: pkg.Package{
|
||||||
Name: "requests",
|
Name: "requests",
|
||||||
Version: "2.22.0",
|
Version: "2.22.0",
|
||||||
@ -93,12 +134,13 @@ func TestPythonPackageWheelCataloger(t *testing.T) {
|
|||||||
FoundBy: "python-package-cataloger",
|
FoundBy: "python-package-cataloger",
|
||||||
MetadataType: pkg.PythonPackageMetadataType,
|
MetadataType: pkg.PythonPackageMetadataType,
|
||||||
Metadata: pkg.PythonPackageMetadata{
|
Metadata: pkg.PythonPackageMetadata{
|
||||||
Name: "requests",
|
Name: "requests",
|
||||||
Version: "2.22.0",
|
Version: "2.22.0",
|
||||||
License: "Apache 2.0",
|
License: "Apache 2.0",
|
||||||
Platform: "UNKNOWN",
|
Platform: "UNKNOWN",
|
||||||
Author: "Kenneth Reitz",
|
Author: "Kenneth Reitz",
|
||||||
AuthorEmail: "me@kennethreitz.org",
|
AuthorEmail: "me@kennethreitz.org",
|
||||||
|
SitePackagesRootPath: "test-fixtures",
|
||||||
Files: []pkg.PythonFileRecord{
|
Files: []pkg.PythonFileRecord{
|
||||||
{Path: "requests-2.22.0.dist-info/INSTALLER", Digest: pkg.Digest{"sha256", "zuuue4knoyJ-UwPPXg8fezS7VCrXJQrAP7zeNuwvFQg"}, Size: "4"},
|
{Path: "requests-2.22.0.dist-info/INSTALLER", Digest: pkg.Digest{"sha256", "zuuue4knoyJ-UwPPXg8fezS7VCrXJQrAP7zeNuwvFQg"}, Size: "4"},
|
||||||
{Path: "requests/__init__.py", Digest: pkg.Digest{"sha256", "PnKCgjcTq44LaAMzB-7--B2FdewRrE8F_vjZeaG9NhA"}, Size: "3921"},
|
{Path: "requests/__init__.py", Digest: pkg.Digest{"sha256", "PnKCgjcTq44LaAMzB-7--B2FdewRrE8F_vjZeaG9NhA"}, Size: "3921"},
|
||||||
@ -107,12 +149,14 @@ func TestPythonPackageWheelCataloger(t *testing.T) {
|
|||||||
{Path: "requests/__version__.py", Digest: pkg.Digest{"sha256", "Bm-GFstQaFezsFlnmEMrJDe8JNROz9n2XXYtODdvjjc"}, Size: "436"},
|
{Path: "requests/__version__.py", Digest: pkg.Digest{"sha256", "Bm-GFstQaFezsFlnmEMrJDe8JNROz9n2XXYtODdvjjc"}, Size: "436"},
|
||||||
{Path: "requests/utils.py", Digest: pkg.Digest{"sha256", "LtPJ1db6mJff2TJSJWKi7rBpzjPS3mSOrjC9zRhoD3A"}, Size: "30049"},
|
{Path: "requests/utils.py", Digest: pkg.Digest{"sha256", "LtPJ1db6mJff2TJSJWKi7rBpzjPS3mSOrjC9zRhoD3A"}, Size: "30049"},
|
||||||
},
|
},
|
||||||
|
TopLevelPackages: []string{"requests"},
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
MetadataFixture: "test-fixtures/dist-info/METADATA",
|
MetadataFixture: "test-fixtures/dist-info/METADATA",
|
||||||
RecordFixture: "test-fixtures/dist-info/RECORD",
|
RecordFixture: "test-fixtures/dist-info/RECORD",
|
||||||
|
TopLevelFixture: "test-fixtures/dist-info/top_level.txt",
|
||||||
ExpectedPackage: pkg.Package{
|
ExpectedPackage: pkg.Package{
|
||||||
Name: "Pygments",
|
Name: "Pygments",
|
||||||
Version: "2.6.1",
|
Version: "2.6.1",
|
||||||
@ -122,12 +166,13 @@ func TestPythonPackageWheelCataloger(t *testing.T) {
|
|||||||
FoundBy: "python-package-cataloger",
|
FoundBy: "python-package-cataloger",
|
||||||
MetadataType: pkg.PythonPackageMetadataType,
|
MetadataType: pkg.PythonPackageMetadataType,
|
||||||
Metadata: pkg.PythonPackageMetadata{
|
Metadata: pkg.PythonPackageMetadata{
|
||||||
Name: "Pygments",
|
Name: "Pygments",
|
||||||
Version: "2.6.1",
|
Version: "2.6.1",
|
||||||
License: "BSD License",
|
License: "BSD License",
|
||||||
Platform: "any",
|
Platform: "any",
|
||||||
Author: "Georg Brandl",
|
Author: "Georg Brandl",
|
||||||
AuthorEmail: "georg@python.org",
|
AuthorEmail: "georg@python.org",
|
||||||
|
SitePackagesRootPath: "test-fixtures",
|
||||||
Files: []pkg.PythonFileRecord{
|
Files: []pkg.PythonFileRecord{
|
||||||
{Path: "../../../bin/pygmentize", Digest: pkg.Digest{"sha256", "dDhv_U2jiCpmFQwIRHpFRLAHUO4R1jIJPEvT_QYTFp8"}, Size: "220"},
|
{Path: "../../../bin/pygmentize", Digest: pkg.Digest{"sha256", "dDhv_U2jiCpmFQwIRHpFRLAHUO4R1jIJPEvT_QYTFp8"}, Size: "220"},
|
||||||
{Path: "Pygments-2.6.1.dist-info/AUTHORS", Digest: pkg.Digest{"sha256", "PVpa2_Oku6BGuiUvutvuPnWGpzxqFy2I8-NIrqCvqUY"}, Size: "8449"},
|
{Path: "Pygments-2.6.1.dist-info/AUTHORS", Digest: pkg.Digest{"sha256", "PVpa2_Oku6BGuiUvutvuPnWGpzxqFy2I8-NIrqCvqUY"}, Size: "8449"},
|
||||||
@ -135,12 +180,15 @@ func TestPythonPackageWheelCataloger(t *testing.T) {
|
|||||||
{Path: "pygments/__pycache__/__init__.cpython-38.pyc"},
|
{Path: "pygments/__pycache__/__init__.cpython-38.pyc"},
|
||||||
{Path: "pygments/util.py", Digest: pkg.Digest{"sha256", "586xXHiJGGZxqk5PMBu3vBhE68DLuAe5MBARWrSPGxA"}, Size: "10778"},
|
{Path: "pygments/util.py", Digest: pkg.Digest{"sha256", "586xXHiJGGZxqk5PMBu3vBhE68DLuAe5MBARWrSPGxA"}, Size: "10778"},
|
||||||
},
|
},
|
||||||
|
TopLevelPackages: []string{"pygments", "something_else"},
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
// in casses where the metadata file is available and the record is not we should still record there is a package
|
// in cases where the metadata file is available and the record is not we should still record there is a package
|
||||||
|
// additionally empty top_level.txt files should not result in an error
|
||||||
MetadataFixture: "test-fixtures/partial.dist-info/METADATA",
|
MetadataFixture: "test-fixtures/partial.dist-info/METADATA",
|
||||||
|
TopLevelFixture: "test-fixtures/partial.dist-info/top_level.txt",
|
||||||
ExpectedPackage: pkg.Package{
|
ExpectedPackage: pkg.Package{
|
||||||
Name: "Pygments",
|
Name: "Pygments",
|
||||||
Version: "2.6.1",
|
Version: "2.6.1",
|
||||||
@ -150,12 +198,13 @@ func TestPythonPackageWheelCataloger(t *testing.T) {
|
|||||||
FoundBy: "python-package-cataloger",
|
FoundBy: "python-package-cataloger",
|
||||||
MetadataType: pkg.PythonPackageMetadataType,
|
MetadataType: pkg.PythonPackageMetadataType,
|
||||||
Metadata: pkg.PythonPackageMetadata{
|
Metadata: pkg.PythonPackageMetadata{
|
||||||
Name: "Pygments",
|
Name: "Pygments",
|
||||||
Version: "2.6.1",
|
Version: "2.6.1",
|
||||||
License: "BSD License",
|
License: "BSD License",
|
||||||
Platform: "any",
|
Platform: "any",
|
||||||
Author: "Georg Brandl",
|
Author: "Georg Brandl",
|
||||||
AuthorEmail: "georg@python.org",
|
AuthorEmail: "georg@python.org",
|
||||||
|
SitePackagesRootPath: "test-fixtures",
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
@ -163,20 +212,7 @@ func TestPythonPackageWheelCataloger(t *testing.T) {
|
|||||||
|
|
||||||
for _, test := range tests {
|
for _, test := range tests {
|
||||||
t.Run(test.MetadataFixture, func(t *testing.T) {
|
t.Run(test.MetadataFixture, func(t *testing.T) {
|
||||||
metadata, err := os.Open(test.MetadataFixture)
|
resolver := newTestResolver(test.MetadataFixture, test.RecordFixture, test.TopLevelFixture)
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("failed to open record: %+v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
var record io.Reader
|
|
||||||
if test.RecordFixture != "" {
|
|
||||||
record, err = os.Open(test.RecordFixture)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("failed to open record: %+v", err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
resolver := newTestResolver(record, metadata)
|
|
||||||
|
|
||||||
// note that the source is the record ref created by the resolver mock... attach the expected values
|
// note that the source is the record ref created by the resolver mock... attach the expected values
|
||||||
test.ExpectedPackage.Source = []file.Reference{*resolver.metadataRef}
|
test.ExpectedPackage.Source = []file.Reference{*resolver.metadataRef}
|
||||||
|
|||||||
@ -4,8 +4,11 @@ import (
|
|||||||
"bufio"
|
"bufio"
|
||||||
"fmt"
|
"fmt"
|
||||||
"io"
|
"io"
|
||||||
|
"path/filepath"
|
||||||
"strings"
|
"strings"
|
||||||
|
|
||||||
|
"github.com/anchore/stereoscope/pkg/file"
|
||||||
|
|
||||||
"github.com/mitchellh/mapstructure"
|
"github.com/mitchellh/mapstructure"
|
||||||
|
|
||||||
"github.com/anchore/syft/syft/pkg"
|
"github.com/anchore/syft/syft/pkg"
|
||||||
@ -13,7 +16,7 @@ import (
|
|||||||
|
|
||||||
// parseWheelOrEggMetadata takes a Python Egg or Wheel (which share the same format and values for our purposes),
|
// parseWheelOrEggMetadata takes a Python Egg or Wheel (which share the same format and values for our purposes),
|
||||||
// returning all Python packages listed.
|
// returning all Python packages listed.
|
||||||
func parseWheelOrEggMetadata(reader io.Reader) (pkg.PythonPackageMetadata, error) {
|
func parseWheelOrEggMetadata(path file.Path, reader io.Reader) (pkg.PythonPackageMetadata, error) {
|
||||||
fields := make(map[string]string)
|
fields := make(map[string]string)
|
||||||
var key string
|
var key string
|
||||||
|
|
||||||
@ -68,5 +71,10 @@ func parseWheelOrEggMetadata(reader io.Reader) (pkg.PythonPackageMetadata, error
|
|||||||
return pkg.PythonPackageMetadata{}, fmt.Errorf("unable to parse APK metadata: %w", err)
|
return pkg.PythonPackageMetadata{}, fmt.Errorf("unable to parse APK metadata: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// add additional metadata not stored in the egg/wheel metadata file
|
||||||
|
|
||||||
|
sitePackagesRoot := filepath.Clean(filepath.Join(filepath.Dir(string(path)), ".."))
|
||||||
|
metadata.SitePackagesRootPath = sitePackagesRoot
|
||||||
|
|
||||||
return metadata, nil
|
return metadata, nil
|
||||||
}
|
}
|
||||||
|
|||||||
@ -4,6 +4,8 @@ import (
|
|||||||
"os"
|
"os"
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
|
"github.com/anchore/stereoscope/pkg/file"
|
||||||
|
|
||||||
"github.com/anchore/syft/syft/pkg"
|
"github.com/anchore/syft/syft/pkg"
|
||||||
"github.com/go-test/deep"
|
"github.com/go-test/deep"
|
||||||
)
|
)
|
||||||
@ -16,23 +18,25 @@ func TestParseWheelEggMetadata(t *testing.T) {
|
|||||||
{
|
{
|
||||||
Fixture: "test-fixtures/egg-info/PKG-INFO",
|
Fixture: "test-fixtures/egg-info/PKG-INFO",
|
||||||
ExpectedMetadata: pkg.PythonPackageMetadata{
|
ExpectedMetadata: pkg.PythonPackageMetadata{
|
||||||
Name: "requests",
|
Name: "requests",
|
||||||
Version: "2.22.0",
|
Version: "2.22.0",
|
||||||
License: "Apache 2.0",
|
License: "Apache 2.0",
|
||||||
Platform: "UNKNOWN",
|
Platform: "UNKNOWN",
|
||||||
Author: "Kenneth Reitz",
|
Author: "Kenneth Reitz",
|
||||||
AuthorEmail: "me@kennethreitz.org",
|
AuthorEmail: "me@kennethreitz.org",
|
||||||
|
SitePackagesRootPath: "test-fixtures",
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
Fixture: "test-fixtures/dist-info/METADATA",
|
Fixture: "test-fixtures/dist-info/METADATA",
|
||||||
ExpectedMetadata: pkg.PythonPackageMetadata{
|
ExpectedMetadata: pkg.PythonPackageMetadata{
|
||||||
Name: "Pygments",
|
Name: "Pygments",
|
||||||
Version: "2.6.1",
|
Version: "2.6.1",
|
||||||
License: "BSD License",
|
License: "BSD License",
|
||||||
Platform: "any",
|
Platform: "any",
|
||||||
Author: "Georg Brandl",
|
Author: "Georg Brandl",
|
||||||
AuthorEmail: "georg@python.org",
|
AuthorEmail: "georg@python.org",
|
||||||
|
SitePackagesRootPath: "test-fixtures",
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
@ -44,7 +48,7 @@ func TestParseWheelEggMetadata(t *testing.T) {
|
|||||||
t.Fatalf("failed to open fixture: %+v", err)
|
t.Fatalf("failed to open fixture: %+v", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
actual, err := parseWheelOrEggMetadata(fixture)
|
actual, err := parseWheelOrEggMetadata(file.Path(test.Fixture), fixture)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("failed to parse: %+v", err)
|
t.Fatalf("failed to parse: %+v", err)
|
||||||
}
|
}
|
||||||
|
|||||||
@ -0,0 +1,2 @@
|
|||||||
|
pygments
|
||||||
|
something_else
|
||||||
@ -0,0 +1 @@
|
|||||||
|
requests
|
||||||
@ -5,6 +5,7 @@ type Digest struct {
|
|||||||
Value string `json:"value"`
|
Value string `json:"value"`
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// PythonFileRecord represents a single entry within a RECORD file for a python wheel or egg package
|
||||||
type PythonFileRecord struct {
|
type PythonFileRecord struct {
|
||||||
Path string `json:"path"`
|
Path string `json:"path"`
|
||||||
Digest Digest `json:"digest"`
|
Digest Digest `json:"digest"`
|
||||||
@ -13,11 +14,13 @@ type PythonFileRecord struct {
|
|||||||
|
|
||||||
// PythonPackageMetadata represents all captured data for a python egg or wheel package.
|
// PythonPackageMetadata represents all captured data for a python egg or wheel package.
|
||||||
type PythonPackageMetadata struct {
|
type PythonPackageMetadata struct {
|
||||||
Name string `json:"name" mapstruct:"Name"`
|
Name string `json:"name" mapstruct:"Name"`
|
||||||
Version string `json:"version" mapstruct:"Version"`
|
Version string `json:"version" mapstruct:"Version"`
|
||||||
License string `json:"license" mapstruct:"License"`
|
License string `json:"license" mapstruct:"License"`
|
||||||
Author string `json:"author" mapstruct:"Author"`
|
Author string `json:"author" mapstruct:"Author"`
|
||||||
AuthorEmail string `json:"authorEmail" mapstruct:"Authoremail"`
|
AuthorEmail string `json:"authorEmail" mapstruct:"Authoremail"`
|
||||||
Platform string `json:"platform" mapstruct:"Platform"`
|
Platform string `json:"platform" mapstruct:"Platform"`
|
||||||
Files []PythonFileRecord `json:"files,omitempty"`
|
Files []PythonFileRecord `json:"files,omitempty"`
|
||||||
|
SitePackagesRootPath string `json:"sitePackagesRootPath"`
|
||||||
|
TopLevelPackages []string `json:"topLevelPackages,omitempty"`
|
||||||
}
|
}
|
||||||
|
|||||||
@ -5,6 +5,7 @@ import (
|
|||||||
"io/ioutil"
|
"io/ioutil"
|
||||||
"os"
|
"os"
|
||||||
"path"
|
"path"
|
||||||
|
"path/filepath"
|
||||||
|
|
||||||
"github.com/anchore/stereoscope/pkg/file"
|
"github.com/anchore/stereoscope/pkg/file"
|
||||||
"github.com/anchore/syft/internal/log"
|
"github.com/anchore/syft/internal/log"
|
||||||
@ -18,7 +19,7 @@ type DirectoryResolver struct {
|
|||||||
|
|
||||||
// Stringer to represent a directory path data source
|
// Stringer to represent a directory path data source
|
||||||
func (s DirectoryResolver) String() string {
|
func (s DirectoryResolver) String() string {
|
||||||
return fmt.Sprintf("dir://%s", s.Path)
|
return fmt.Sprintf("dir:%s", s.Path)
|
||||||
}
|
}
|
||||||
|
|
||||||
// FilesByPath returns all file.References that match the given paths from the directory.
|
// FilesByPath returns all file.References that match the given paths from the directory.
|
||||||
@ -26,15 +27,19 @@ func (s DirectoryResolver) FilesByPath(userPaths ...file.Path) ([]file.Reference
|
|||||||
var references = make([]file.Reference, 0)
|
var references = make([]file.Reference, 0)
|
||||||
|
|
||||||
for _, userPath := range userPaths {
|
for _, userPath := range userPaths {
|
||||||
resolvedPath := path.Join(s.Path, string(userPath))
|
userStrPath := string(userPath)
|
||||||
_, err := os.Stat(resolvedPath)
|
|
||||||
|
if filepath.IsAbs(userStrPath) {
|
||||||
|
// a path relative to root should be prefixed with the resolvers directory path, otherwise it should be left as is
|
||||||
|
userStrPath = path.Join(s.Path, userStrPath)
|
||||||
|
}
|
||||||
|
_, err := os.Stat(userStrPath)
|
||||||
if os.IsNotExist(err) {
|
if os.IsNotExist(err) {
|
||||||
continue
|
continue
|
||||||
} else if err != nil {
|
} else if err != nil {
|
||||||
log.Errorf("path (%s) is not valid: %v", resolvedPath, err)
|
log.Errorf("path (%s) is not valid: %v", userStrPath, err)
|
||||||
}
|
}
|
||||||
filePath := file.Path(resolvedPath)
|
references = append(references, file.NewFileReference(file.Path(userStrPath)))
|
||||||
references = append(references, file.NewFileReference(filePath))
|
|
||||||
}
|
}
|
||||||
|
|
||||||
return references, nil
|
return references, nil
|
||||||
|
|||||||
@ -1,7 +1,6 @@
|
|||||||
package resolvers
|
package resolvers
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"path"
|
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
"github.com/anchore/stereoscope/pkg/file"
|
"github.com/anchore/stereoscope/pkg/file"
|
||||||
@ -10,24 +9,49 @@ import (
|
|||||||
func TestDirectoryResolver_FilesByPath(t *testing.T) {
|
func TestDirectoryResolver_FilesByPath(t *testing.T) {
|
||||||
cases := []struct {
|
cases := []struct {
|
||||||
name string
|
name string
|
||||||
|
root string
|
||||||
input string
|
input string
|
||||||
|
expected string
|
||||||
refCount int
|
refCount int
|
||||||
}{
|
}{
|
||||||
{
|
{
|
||||||
name: "finds a file",
|
name: "finds a file (relative)",
|
||||||
input: "image-symlinks/file-1.txt",
|
root: "./test-fixtures/",
|
||||||
|
input: "test-fixtures/image-symlinks/file-1.txt",
|
||||||
|
expected: "test-fixtures/image-symlinks/file-1.txt",
|
||||||
refCount: 1,
|
refCount: 1,
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
name: "managed non-existing files",
|
name: "finds a file with relative indirection",
|
||||||
input: "image-symlinks/bogus.txt",
|
root: "./test-fixtures/../test-fixtures",
|
||||||
|
input: "test-fixtures/image-symlinks/file-1.txt",
|
||||||
|
expected: "test-fixtures/image-symlinks/file-1.txt",
|
||||||
|
refCount: 1,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
// note: this is asserting the old behavior is not supported
|
||||||
|
name: "relative lookup with wrong path fails",
|
||||||
|
root: "./test-fixtures/",
|
||||||
|
input: "image-symlinks/file-1.txt",
|
||||||
refCount: 0,
|
refCount: 0,
|
||||||
},
|
},
|
||||||
|
{
|
||||||
|
name: "managed non-existing files (relative)",
|
||||||
|
root: "./test-fixtures/",
|
||||||
|
input: "test-fixtures/image-symlinks/bogus.txt",
|
||||||
|
refCount: 0,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "finds a file (absolute)",
|
||||||
|
root: "./test-fixtures/",
|
||||||
|
input: "/image-symlinks/file-1.txt",
|
||||||
|
expected: "test-fixtures/image-symlinks/file-1.txt",
|
||||||
|
refCount: 1,
|
||||||
|
},
|
||||||
}
|
}
|
||||||
for _, c := range cases {
|
for _, c := range cases {
|
||||||
t.Run(c.name, func(t *testing.T) {
|
t.Run(c.name, func(t *testing.T) {
|
||||||
resolver := DirectoryResolver{"test-fixtures"}
|
resolver := DirectoryResolver{c.root}
|
||||||
expected := path.Join("test-fixtures", c.input)
|
|
||||||
refs, err := resolver.FilesByPath(file.Path(c.input))
|
refs, err := resolver.FilesByPath(file.Path(c.input))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("could not use resolver: %+v, %+v", err, refs)
|
t.Fatalf("could not use resolver: %+v, %+v", err, refs)
|
||||||
@ -38,8 +62,8 @@ func TestDirectoryResolver_FilesByPath(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
for _, actual := range refs {
|
for _, actual := range refs {
|
||||||
if actual.Path != file.Path(expected) {
|
if actual.Path != file.Path(c.expected) {
|
||||||
t.Errorf("bad resolve path: '%s'!='%s'", actual.Path, c.input)
|
t.Errorf("bad resolve path: '%s'!='%s'", actual.Path, c.expected)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
@ -54,17 +78,17 @@ func TestDirectoryResolver_MultipleFilesByPath(t *testing.T) {
|
|||||||
}{
|
}{
|
||||||
{
|
{
|
||||||
name: "finds multiple files",
|
name: "finds multiple files",
|
||||||
input: []file.Path{file.Path("image-symlinks/file-1.txt"), file.Path("image-symlinks/file-2.txt")},
|
input: []file.Path{file.Path("test-fixtures/image-symlinks/file-1.txt"), file.Path("test-fixtures/image-symlinks/file-2.txt")},
|
||||||
refCount: 2,
|
refCount: 2,
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
name: "skips non-existing files",
|
name: "skips non-existing files",
|
||||||
input: []file.Path{file.Path("image-symlinks/bogus.txt"), file.Path("image-symlinks/file-1.txt")},
|
input: []file.Path{file.Path("test-fixtures/image-symlinks/bogus.txt"), file.Path("test-fixtures/image-symlinks/file-1.txt")},
|
||||||
refCount: 1,
|
refCount: 1,
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
name: "does not return anything for non-existing directories",
|
name: "does not return anything for non-existing directories",
|
||||||
input: []file.Path{file.Path("non-existing/bogus.txt"), file.Path("non-existing/file-1.txt")},
|
input: []file.Path{file.Path("test-fixtures/non-existing/bogus.txt"), file.Path("test-fixtures/non-existing/file-1.txt")},
|
||||||
refCount: 0,
|
refCount: 0,
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
@ -93,17 +117,17 @@ func TestDirectoryResolver_MultipleFileContentsByRef(t *testing.T) {
|
|||||||
}{
|
}{
|
||||||
{
|
{
|
||||||
name: "gets multiple file contents",
|
name: "gets multiple file contents",
|
||||||
input: []file.Path{file.Path("image-symlinks/file-1.txt"), file.Path("image-symlinks/file-2.txt")},
|
input: []file.Path{file.Path("test-fixtures/image-symlinks/file-1.txt"), file.Path("test-fixtures/image-symlinks/file-2.txt")},
|
||||||
refCount: 2,
|
refCount: 2,
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
name: "skips non-existing files",
|
name: "skips non-existing files",
|
||||||
input: []file.Path{file.Path("image-symlinks/bogus.txt"), file.Path("image-symlinks/file-1.txt")},
|
input: []file.Path{file.Path("test-fixtures/image-symlinks/bogus.txt"), file.Path("test-fixtures/image-symlinks/file-1.txt")},
|
||||||
refCount: 1,
|
refCount: 1,
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
name: "does not return anything for non-existing directories",
|
name: "does not return anything for non-existing directories",
|
||||||
input: []file.Path{file.Path("non-existing/bogus.txt"), file.Path("non-existing/file-1.txt")},
|
input: []file.Path{file.Path("test-fixtures/non-existing/bogus.txt"), file.Path("test-fixtures/non-existing/file-1.txt")},
|
||||||
refCount: 0,
|
refCount: 0,
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
|||||||
@ -61,13 +61,13 @@ func TestDirectoryScope(t *testing.T) {
|
|||||||
{
|
{
|
||||||
desc: "path detected",
|
desc: "path detected",
|
||||||
input: "test-fixtures",
|
input: "test-fixtures",
|
||||||
inputPaths: []file.Path{file.Path("path-detected")},
|
inputPaths: []file.Path{file.Path("test-fixtures/path-detected")},
|
||||||
expRefs: 1,
|
expRefs: 1,
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
desc: "no files-by-path detected",
|
desc: "no files-by-path detected",
|
||||||
input: "test-fixtures",
|
input: "test-fixtures",
|
||||||
inputPaths: []file.Path{file.Path("no-path-detected")},
|
inputPaths: []file.Path{file.Path("test-fixtures/no-path-detected")},
|
||||||
expRefs: 0,
|
expRefs: 0,
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
@ -105,13 +105,13 @@ func TestMultipleFileContentsByRefContents(t *testing.T) {
|
|||||||
{
|
{
|
||||||
input: "test-fixtures/path-detected",
|
input: "test-fixtures/path-detected",
|
||||||
desc: "empty file",
|
desc: "empty file",
|
||||||
path: "empty",
|
path: "test-fixtures/path-detected/empty",
|
||||||
expected: "",
|
expected: "",
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
input: "test-fixtures/path-detected",
|
input: "test-fixtures/path-detected",
|
||||||
desc: "file has contents",
|
desc: "file has contents",
|
||||||
path: ".vimrc",
|
path: "test-fixtures/path-detected/.vimrc",
|
||||||
expected: "\" A .vimrc file\n",
|
expected: "\" A .vimrc file\n",
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
@ -127,7 +127,7 @@ func TestMultipleFileContentsByRefContents(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
if len(refs) != 1 {
|
if len(refs) != 1 {
|
||||||
t.Errorf("expected a single ref to be generated but got: %d", len(refs))
|
t.Fatalf("expected a single ref to be generated but got: %d", len(refs))
|
||||||
}
|
}
|
||||||
ref := refs[0]
|
ref := refs[0]
|
||||||
|
|
||||||
|
|||||||
@ -0,0 +1 @@
|
|||||||
|
top-level-pkg
|
||||||
@ -0,0 +1 @@
|
|||||||
|
top-level-pkg
|
||||||
@ -0,0 +1 @@
|
|||||||
|
top-level-pkg
|
||||||
@ -0,0 +1 @@
|
|||||||
|
top-level-pkg
|
||||||
Loading…
x
Reference in New Issue
Block a user