oss/syft
2
0
Fork 0
mirror of https://github.com/anchore/syft.git synced 2026-11-05 00:46:38 +01:00
Code Issues Packages Projects Releases Wiki Activity

feat: update syft license concept to complex struct (#1743)

Browse Source 
this PR makes the following changes to update the underlying license model to have more expressive capabilities
it also provides some guarantee's surrounding the license values themselves

- Licenses are updated from string -> pkg.LicenseSet which contain pkg.License with the following fields:
- original `Value` read by syft
- If it's possible to construct licenses will always have a valid SPDX expression for downstream consumption
- the above is run against a generated list of SPDX license ID to try and find the correct ID
- SPDX concluded vs declared is added to the new struct
- URL source for license is added to the new struct
- Location source is added to the new struct to show where the expression was pulled from
This commit is contained in:
Christopher Angelo Phillips
2023-05-15 16:23:39 -04:00
committed by GitHub
parent 8046f09562
commit 42fa9e4965
126 changed files with 5165 additions and 1503 deletions
Download Patch File Download Diff File Expand all files Collapse all files
syft/pkg/cataloger/python/cataloger_test.go
+40 -35
View File
@@ -40,18 +40,19 @@ func Test_PackageCataloger(t *testing.T) {
"test-fixtures/egg-info/top_level.txt",
},
expectedPackage: pkg.Package{
Name: "requests",
Version: "2.22.0",
PURL: "pkg:pypi/requests@2.22.0",
Type: pkg.PythonPkg,
Language: pkg.Python,
Licenses: []string{"Apache 2.0"},
Name: "requests",
Version: "2.22.0",
PURL: "pkg:pypi/requests@2.22.0",
Type: pkg.PythonPkg,
Language: pkg.Python,
Licenses: pkg.NewLicenseSet(
pkg.NewLicenseFromLocations("Apache 2.0", source.NewLocation("test-fixtures/egg-info/PKG-INFO")),
),
FoundBy: "python-package-cataloger",
MetadataType: pkg.PythonPackageMetadataType,
Metadata: pkg.PythonPackageMetadata{
Name: "requests",
Version: "2.22.0",
License: "Apache 2.0",
Platform: "UNKNOWN",
Author: "Kenneth Reitz",
AuthorEmail: "me@kennethreitz.org",
@@ -77,18 +78,19 @@ func Test_PackageCataloger(t *testing.T) {
"test-fixtures/dist-info/direct_url.json",
},
expectedPackage: pkg.Package{
Name: "Pygments",
Version: "2.6.1",
PURL: "pkg:pypi/Pygments@2.6.1?vcs_url=git+https://github.com/python-test/test.git%40aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
Type: pkg.PythonPkg,
Language: pkg.Python,
Licenses: []string{"BSD License"},
Name: "Pygments",
Version: "2.6.1",
PURL: "pkg:pypi/Pygments@2.6.1?vcs_url=git+https://github.com/python-test/test.git%40aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
Type: pkg.PythonPkg,
Language: pkg.Python,
Licenses: pkg.NewLicenseSet(
pkg.NewLicenseFromLocations("BSD License", source.NewLocation("test-fixtures/dist-info/METADATA")),
),
FoundBy: "python-package-cataloger",
MetadataType: pkg.PythonPackageMetadataType,
Metadata: pkg.PythonPackageMetadata{
Name: "Pygments",
Version: "2.6.1",
License: "BSD License",
Platform: "any",
Author: "Georg Brandl",
AuthorEmail: "georg@python.org",
@@ -114,18 +116,19 @@ func Test_PackageCataloger(t *testing.T) {
"test-fixtures/malformed-record/dist-info/RECORD",
},
expectedPackage: pkg.Package{
Name: "Pygments",
Version: "2.6.1",
PURL: "pkg:pypi/Pygments@2.6.1",
Type: pkg.PythonPkg,
Language: pkg.Python,
Licenses: []string{"BSD License"},
Name: "Pygments",
Version: "2.6.1",
PURL: "pkg:pypi/Pygments@2.6.1",
Type: pkg.PythonPkg,
Language: pkg.Python,
Licenses: pkg.NewLicenseSet(
pkg.NewLicenseFromLocations("BSD License", source.NewLocation("test-fixtures/malformed-record/dist-info/METADATA")),
),
FoundBy: "python-package-cataloger",
MetadataType: pkg.PythonPackageMetadataType,
Metadata: pkg.PythonPackageMetadata{
Name: "Pygments",
Version: "2.6.1",
License: "BSD License",
Platform: "any",
Author: "Georg Brandl",
AuthorEmail: "georg@python.org",
@@ -145,18 +148,19 @@ func Test_PackageCataloger(t *testing.T) {
name: "partial dist-info directory",
fixtures: []string{"test-fixtures/partial.dist-info/METADATA"},
expectedPackage: pkg.Package{
Name: "Pygments",
Version: "2.6.1",
PURL: "pkg:pypi/Pygments@2.6.1",
Type: pkg.PythonPkg,
Language: pkg.Python,
Licenses: []string{"BSD License"},
Name: "Pygments",
Version: "2.6.1",
PURL: "pkg:pypi/Pygments@2.6.1",
Type: pkg.PythonPkg,
Language: pkg.Python,
Licenses: pkg.NewLicenseSet(
pkg.NewLicenseFromLocations("BSD License", source.NewLocation("test-fixtures/partial.dist-info/METADATA")),
),
FoundBy: "python-package-cataloger",
MetadataType: pkg.PythonPackageMetadataType,
Metadata: pkg.PythonPackageMetadata{
Name: "Pygments",
Version: "2.6.1",
License: "BSD License",
Platform: "any",
Author: "Georg Brandl",
AuthorEmail: "georg@python.org",
@@ -168,18 +172,19 @@ func Test_PackageCataloger(t *testing.T) {
name: "egg-info regular file",
fixtures: []string{"test-fixtures/test.egg-info"},
expectedPackage: pkg.Package{
Name: "requests",
Version: "2.22.0",
PURL: "pkg:pypi/requests@2.22.0",
Type: pkg.PythonPkg,
Language: pkg.Python,
Licenses: []string{"Apache 2.0"},
Name: "requests",
Version: "2.22.0",
PURL: "pkg:pypi/requests@2.22.0",
Type: pkg.PythonPkg,
Language: pkg.Python,
Licenses: pkg.NewLicenseSet(
pkg.NewLicenseFromLocations("Apache 2.0", source.NewLocation("test-fixtures/test.egg-info")),
),
FoundBy: "python-package-cataloger",
MetadataType: pkg.PythonPackageMetadataType,
Metadata: pkg.PythonPackageMetadata{
Name: "requests",
Version: "2.22.0",
License: "Apache 2.0",
Platform: "UNKNOWN",
Author: "Kenneth Reitz",
AuthorEmail: "me@kennethreitz.org",
syft/pkg/cataloger/python/package.go
+5 -9
View File
@@ -57,25 +57,21 @@ func newPackageForRequirementsWithMetadata(name, version string, metadata pkg.Py
return p
}
func newPackageForPackage(m pkg.PythonPackageMetadata, sources ...source.Location) pkg.Package {
var licenses []string
if m.License != "" {
licenses = []string{m.License}
}
func newPackageForPackage(m parsedData, sources ...source.Location) pkg.Package {
p := pkg.Package{
Name: m.Name,
Version: m.Version,
PURL: packageURL(m.Name, m.Version, &m),
PURL: packageURL(m.Name, m.Version, &m.PythonPackageMetadata),
Locations: source.NewLocationSet(sources...),
Licenses: licenses,
Licenses: pkg.NewLicenseSet(pkg.NewLicensesFromLocation(m.LicenseLocation, m.Licenses)...),
Language: pkg.Python,
Type: pkg.PythonPkg,
MetadataType: pkg.PythonPackageMetadataType,
Metadata: m,
Metadata: m.PythonPackageMetadata,
}
p.SetID()
return p
}
syft/pkg/cataloger/python/parse_wheel_egg.go
+13 -13
View File
@@ -17,21 +17,21 @@ import (
// parseWheelOrEgg takes the primary metadata file reference and returns the python package it represents.
func parseWheelOrEgg(resolver source.FileResolver, _ *generic.Environment, reader source.LocationReadCloser) ([]pkg.Package, []artifact.Relationship, error) {
metadata, sources, err := assembleEggOrWheelMetadata(resolver, reader.Location)
pd, sources, err := assembleEggOrWheelMetadata(resolver, reader.Location)
if err != nil {
return nil, nil, err
}
if metadata == nil {
if pd == nil {
return nil, nil, nil
}
// This can happen for Python 2.7 where it is reported from an egg-info, but Python is
// the actual runtime, it isn't a "package". The special-casing here allows to skip it
if metadata.Name == "Python" {
if pd.Name == "Python" {
return nil, nil, nil
}
pkgs := []pkg.Package{newPackageForPackage(*metadata, sources...)}
pkgs := []pkg.Package{newPackageForPackage(*pd, sources...)}
return pkgs, nil, nil
}
@@ -160,7 +160,7 @@ func fetchDirectURLData(resolver source.FileResolver, metadataLocation source.Lo
}
// assembleEggOrWheelMetadata discovers and accumulates python package metadata from multiple file sources and returns a single metadata object as well as a list of files where the metadata was derived from.
func assembleEggOrWheelMetadata(resolver source.FileResolver, metadataLocation source.Location) (*pkg.PythonPackageMetadata, []source.Location, error) {
func assembleEggOrWheelMetadata(resolver source.FileResolver, metadataLocation source.Location) (*parsedData, []source.Location, error) {
var sources = []source.Location{
metadataLocation.WithAnnotation(pkg.EvidenceAnnotationKey, pkg.PrimaryEvidenceAnnotation),
}
@@ -171,12 +171,12 @@ func assembleEggOrWheelMetadata(resolver source.FileResolver, metadataLocation s
}
defer internal.CloseAndLogError(metadataContents, metadataLocation.VirtualPath)
metadata, err := parseWheelOrEggMetadata(metadataLocation.RealPath, metadataContents)
pd, err := parseWheelOrEggMetadata(metadataLocation.RealPath, metadataContents)
if err != nil {
return nil, nil, err
}
if metadata.Name == "" {
if pd.Name == "" {
return nil, nil, nil
}
@@ -186,14 +186,14 @@ func assembleEggOrWheelMetadata(resolver source.FileResolver, metadataLocation s
return nil, nil, err
}
if len(r) == 0 {
r, s, err = fetchInstalledFiles(resolver, metadataLocation, metadata.SitePackagesRootPath)
r, s, err = fetchInstalledFiles(resolver, metadataLocation, pd.SitePackagesRootPath)
if err != nil {
return nil, nil, err
}
}
sources = append(sources, s...)
metadata.Files = r
pd.Files = r
// attach any top-level package names found for the given wheel/egg installation
p, s, err := fetchTopLevelPackages(resolver, metadataLocation)
@@ -201,15 +201,15 @@ func assembleEggOrWheelMetadata(resolver source.FileResolver, metadataLocation s
return nil, nil, err
}
sources = append(sources, s...)
metadata.TopLevelPackages = p
pd.TopLevelPackages = p
// attach any direct-url package data found for the given wheel/egg installation
d, s, err := fetchDirectURLData(resolver, metadataLocation)
if err != nil {
return nil, nil, err
}
sources = append(sources, s...)
metadata.DirectURLOrigin = d
return &metadata, sources, nil
sources = append(sources, s...)
pd.DirectURLOrigin = d
return &pd, sources, nil
}
syft/pkg/cataloger/python/parse_wheel_egg_metadata.go
+18 -8
View File
@@ -12,11 +12,18 @@ import (
"github.com/anchore/syft/internal/file"
"github.com/anchore/syft/internal/log"
"github.com/anchore/syft/syft/pkg"
"github.com/anchore/syft/syft/source"
)
type parsedData struct {
Licenses string `mapstructure:"License"`
LicenseLocation source.Location
pkg.PythonPackageMetadata `mapstructure:",squash"`
}
// parseWheelOrEggMetadata takes a Python Egg or Wheel (which share the same format and values for our purposes),
// returning all Python packages listed.
func parseWheelOrEggMetadata(path string, reader io.Reader) (pkg.PythonPackageMetadata, error) {
func parseWheelOrEggMetadata(path string, reader io.Reader) (parsedData, error) {
fields := make(map[string]string)
var key string
@@ -43,7 +50,7 @@ func parseWheelOrEggMetadata(path string, reader io.Reader) (pkg.PythonPackageMe
// a field-body continuation
updatedValue, err := handleFieldBodyContinuation(key, line, fields)
if err != nil {
return pkg.PythonPackageMetadata{}, err
return parsedData{}, err
}
fields[key] = updatedValue
@@ -62,19 +69,22 @@ func parseWheelOrEggMetadata(path string, reader io.Reader) (pkg.PythonPackageMe
}
if err := scanner.Err(); err != nil {
return pkg.PythonPackageMetadata{}, fmt.Errorf("failed to parse python wheel/egg: %w", err)
return parsedData{}, fmt.Errorf("failed to parse python wheel/egg: %w", err)
}
var metadata pkg.PythonPackageMetadata
if err := mapstructure.Decode(fields, &metadata); err != nil {
return pkg.PythonPackageMetadata{}, fmt.Errorf("unable to parse APK metadata: %w", err)
var pd parsedData
if err := mapstructure.Decode(fields, &pd); err != nil {
return pd, fmt.Errorf("unable to parse APK metadata: %w", err)
}
// add additional metadata not stored in the egg/wheel metadata file
metadata.SitePackagesRootPath = determineSitePackagesRootPath(path)
pd.SitePackagesRootPath = determineSitePackagesRootPath(path)
if pd.Licenses != "" {
pd.LicenseLocation = source.NewLocation(path)
}
return metadata, nil
return pd, nil
}
// isEggRegularFile determines if the specified path is the regular file variant
syft/pkg/cataloger/python/parse_wheel_egg_metadata_test.go
+33 -22
View File
@@ -7,35 +7,42 @@ import (
"github.com/go-test/deep"
"github.com/anchore/syft/syft/pkg"
"github.com/anchore/syft/syft/source"
)
func TestParseWheelEggMetadata(t *testing.T) {
tests := []struct {
Fixture string
ExpectedMetadata pkg.PythonPackageMetadata
ExpectedMetadata parsedData
}{
{
Fixture: "test-fixtures/egg-info/PKG-INFO",
ExpectedMetadata: pkg.PythonPackageMetadata{
Name: "requests",
Version: "2.22.0",
License: "Apache 2.0",
Platform: "UNKNOWN",
Author: "Kenneth Reitz",
AuthorEmail: "me@kennethreitz.org",
SitePackagesRootPath: "test-fixtures",
ExpectedMetadata: parsedData{
"Apache 2.0",
source.NewLocation("test-fixtures/egg-info/PKG-INFO"),
pkg.PythonPackageMetadata{
Name: "requests",
Version: "2.22.0",
Platform: "UNKNOWN",
Author: "Kenneth Reitz",
AuthorEmail: "me@kennethreitz.org",
SitePackagesRootPath: "test-fixtures",
},
},
},
{
Fixture: "test-fixtures/dist-info/METADATA",
ExpectedMetadata: pkg.PythonPackageMetadata{
Name: "Pygments",
Version: "2.6.1",
License: "BSD License",
Platform: "any",
Author: "Georg Brandl",
AuthorEmail: "georg@python.org",
SitePackagesRootPath: "test-fixtures",
ExpectedMetadata: parsedData{
"BSD License",
source.NewLocation("test-fixtures/dist-info/METADATA"),
pkg.PythonPackageMetadata{
Name: "Pygments",
Version: "2.6.1",
Platform: "any",
Author: "Georg Brandl",
AuthorEmail: "georg@python.org",
SitePackagesRootPath: "test-fixtures",
},
},
},
}
@@ -122,14 +129,18 @@ func TestDetermineSitePackagesRootPath(t *testing.T) {
func TestParseWheelEggMetadataInvalid(t *testing.T) {
tests := []struct {
Fixture string
ExpectedMetadata pkg.PythonPackageMetadata
ExpectedMetadata parsedData
}{
{
Fixture: "test-fixtures/egg-info/PKG-INFO-INVALID",
ExpectedMetadata: pkg.PythonPackageMetadata{
Name: "mxnet",
Version: "1.8.0",
SitePackagesRootPath: "test-fixtures",
ExpectedMetadata: parsedData{
"",
source.Location{},
pkg.PythonPackageMetadata{
Name: "mxnet",
Version: "1.8.0",
SitePackagesRootPath: "test-fixtures",
},
},
},
}