oss/syft
2
0
Fork 0
mirror of https://github.com/anchore/syft.git synced 2026-11-01 06:56:40 +01:00
Code Issues Packages Projects Releases Wiki Activity

feat: 3626 add option enable license content; disable by default (#3631)

Browse Source 
---------
Signed-off-by: Christopher Phillips <32073428+spiffcs@users.noreply.github.com>
This commit is contained in:
Christopher Angelo Phillips
2025-02-05 20:41:03 +00:00
committed by GitHub
parent 7bab6e9851
commit e584c9f416
25 changed files with 436 additions and 98 deletions
Download Patch File Download Diff File Expand all files Collapse all files
internal/licenses/context.go
+2 -2
View File
@@ -10,9 +10,9 @@ func SetContextLicenseScanner(ctx context.Context, s Scanner) context.Context {
return context.WithValue(ctx, licenseScannerKey{}, s)
}
func ContextLicenseScanner(ctx context.Context) Scanner {
func ContextLicenseScanner(ctx context.Context) (Scanner, error) {
if s, ok := ctx.Value(licenseScannerKey{}).(Scanner); ok {
return s
return s, nil
}
return NewDefaultScanner()
}
internal/licenses/scanner.go
+57 -39
View File
@@ -2,66 +2,84 @@ package licenses
import (
"context"
"fmt"
"io"
"github.com/google/licensecheck"
"github.com/anchore/syft/internal/log"
"github.com/anchore/syft/syft/file"
"github.com/anchore/syft/syft/pkg"
)
const coverageThreshold = 75 // determined by experimentation
const (
DefaultCoverageThreshold = 75 // determined by experimentation
DefaultIncludeLicenseContent = false
)
type Scanner interface {
IdentifyLicenseIDs(context.Context, io.Reader) ([]string, []byte, error)
FileSearch(context.Context, file.LocationReadCloser) ([]file.License, error)
PkgSearch(context.Context, file.LocationReadCloser) ([]pkg.License, error)
}
var _ Scanner = (*scanner)(nil)
type scanner struct {
coverageThreshold float64 // between 0 and 100
scanner func([]byte) licensecheck.Coverage
coverageThreshold float64 // between 0 and 100
includeLicenseContent bool
scanner func([]byte) licensecheck.Coverage
}
type ScannerConfig struct {
CoverageThreshold float64
IncludeLicenseContent bool
Scanner func([]byte) licensecheck.Coverage
}
type Option func(*scanner)
func WithCoverage(coverage float64) Option {
return func(s *scanner) {
s.coverageThreshold = coverage
}
}
func WithIncludeLicenseContent(includeLicenseContent bool) Option {
return func(s *scanner) {
s.includeLicenseContent = includeLicenseContent
}
}
// NewDefaultScanner returns a scanner that uses a new instance of the default licensecheck package scanner.
func NewDefaultScanner() Scanner {
func NewDefaultScanner(o ...Option) (Scanner, error) {
s, err := licensecheck.NewScanner(licensecheck.BuiltinLicenses())
if err != nil {
log.WithFields("error", err).Trace("unable to create default license scanner")
s = nil
return nil, fmt.Errorf("unable to create default license scanner: %w", err)
}
newScanner := &scanner{
coverageThreshold: DefaultCoverageThreshold,
includeLicenseContent: DefaultIncludeLicenseContent,
scanner: s.Scan,
}
for _, opt := range o {
opt(newScanner)
}
return newScanner, nil
}
// NewScanner generates a license Scanner with the given ScannerConfig
// if config is nil NewDefaultScanner is used
func NewScanner(c *ScannerConfig) (Scanner, error) {
if c == nil {
return NewDefaultScanner()
}
return &scanner{
coverageThreshold: coverageThreshold,
scanner: s.Scan,
}
}
func NewScanner(scan func([]byte) licensecheck.Coverage, coverage float64) Scanner {
return scanner{
coverageThreshold: coverage,
scanner: scan,
}
}
func (s scanner) IdentifyLicenseIDs(_ context.Context, reader io.Reader) ([]string, []byte, error) {
if s.scanner == nil {
return nil, nil, nil
}
content, err := io.ReadAll(reader)
if err != nil {
return nil, nil, err
}
cov := s.scanner(content)
if cov.Percent < s.coverageThreshold {
// unknown or no licenses here?
// => return binary content
return nil, content, nil
}
var ids []string
for _, m := range cov.Match {
ids = append(ids, m.ID)
}
return ids, nil, nil
coverageThreshold: c.CoverageThreshold,
includeLicenseContent: c.IncludeLicenseContent,
scanner: c.Scanner,
}, nil
}
internal/licenses/scanner_test.go
+7 -6
View File
@@ -27,11 +27,11 @@ func TestIdentifyLicenseIDs(t *testing.T) {
expected: expectation{
yieldError: false,
ids: []string{"Apache-2.0"},
content: []byte{},
content: nil,
},
},
{
name: "custom license",
name: "custom license includes content for IdentifyLicenseIDs",
in: "test-fixtures/nvidia-software-and-cuda-supplement",
expected: expectation{
yieldError: false,
@@ -45,7 +45,7 @@ func TestIdentifyLicenseIDs(t *testing.T) {
t.Run(test.name, func(t *testing.T) {
content, err := os.ReadFile(test.in)
require.NoError(t, err)
ids, content, err := testScanner().IdentifyLicenseIDs(context.TODO(), bytes.NewReader(content))
ids, content, err := testScanner(false).IdentifyLicenseIDs(context.TODO(), bytes.NewReader(content))
if test.expected.yieldError {
require.Error(t, err)
} else {
@@ -66,10 +66,11 @@ func TestIdentifyLicenseIDs(t *testing.T) {
}
}
func testScanner() Scanner {
func testScanner(includeLicenseContent bool) Scanner {
return &scanner{
coverageThreshold: coverageThreshold,
scanner: licensecheck.Scan,
coverageThreshold: DefaultCoverageThreshold,
includeLicenseContent: includeLicenseContent,
scanner: licensecheck.Scan,
}
}
internal/licenses/search.go
+69 -4
View File
@@ -4,6 +4,7 @@ import (
"context"
"crypto/sha256"
"fmt"
"io"
"strings"
"github.com/anchore/syft/syft/file"
@@ -21,11 +22,35 @@ func getCustomLicenseContentHash(contents []byte) string {
return fmt.Sprintf("%x", hash[:])
}
// Search scans the contents of a license file to attempt to determine the type of license it is
func Search(ctx context.Context, scanner Scanner, reader file.LocationReadCloser) (licenses []pkg.License, err error) {
func (s *scanner) IdentifyLicenseIDs(_ context.Context, reader io.Reader) ([]string, []byte, error) {
if s.scanner == nil {
return nil, nil, nil
}
content, err := io.ReadAll(reader)
if err != nil {
return nil, nil, err
}
cov := s.scanner(content)
if cov.Percent < s.coverageThreshold {
// unknown or no licenses here
// => check return content to Search to process
return nil, content, nil
}
var ids []string
for _, m := range cov.Match {
ids = append(ids, m.ID)
}
return ids, nil, nil
}
// PkgSearch scans the contents of a license file to attempt to determine the type of license it is
func (s *scanner) PkgSearch(ctx context.Context, reader file.LocationReadCloser) (licenses []pkg.License, err error) {
licenses = make([]pkg.License, 0)
ids, content, err := scanner.IdentifyLicenseIDs(ctx, reader)
ids, content, err := s.IdentifyLicenseIDs(ctx, reader)
if err != nil {
return nil, err
}
@@ -48,7 +73,47 @@ func Search(ctx context.Context, scanner Scanner, reader file.LocationReadCloser
lic := pkg.NewLicenseFromLocations(unknownLicenseType, reader.Location)
lic.SPDXExpression = UnknownLicensePrefix + getCustomLicenseContentHash(content)
lic.Contents = string(content)
if s.includeLicenseContent {
lic.Contents = string(content)
}
lic.Type = license.Declared
licenses = append(licenses, lic)
}
return licenses, nil
}
// FileSearch scans the contents of a license file to attempt to determine the type of license it is
func (s *scanner) FileSearch(ctx context.Context, reader file.LocationReadCloser) (licenses []file.License, err error) {
licenses = make([]file.License, 0)
ids, content, err := s.IdentifyLicenseIDs(ctx, reader)
if err != nil {
return nil, err
}
// IdentifyLicenseIDs can only return a list of ID or content
// These return values are mutually exclusive.
// If the scanner threshold for matching scores < 75% then we return the license full content
if len(ids) > 0 {
for _, id := range ids {
lic := file.NewLicense(id)
lic.Type = license.Concluded
licenses = append(licenses, lic)
}
} else if len(content) > 0 {
// harmonize line endings to unix compatible first:
// 1. \r\n => \n (Windows => UNIX)
// 2. \r => \n (Macintosh => UNIX)
content = []byte(strings.ReplaceAll(strings.ReplaceAll(string(content), "\r\n", "\n"), "\r", "\n"))
lic := file.NewLicense(unknownLicenseType)
lic.SPDXExpression = UnknownLicensePrefix + getCustomLicenseContentHash(content)
if s.includeLicenseContent {
lic.Contents = string(content)
}
lic.Type = license.Declared
licenses = append(licenses, lic)
internal/licenses/search_test.go
+99 -28
View File
@@ -27,47 +27,28 @@ func newBytesReadCloser(data []byte) *bytesReadCloser {
}
}
func TestSearch(t *testing.T) {
func TestSearchFileLicenses(t *testing.T) {
type expectation struct {
yieldError bool
licenses []pkg.License
licenses []file.License
}
testLocation := file.NewLocation("LICENSE")
tests := []struct {
name string
in string
expected expectation
name string
in string
includeUnkownLicenseContent bool
expected expectation
}{
{
name: "apache license 2.0",
in: "test-fixtures/apache-license-2.0",
expected: expectation{
yieldError: false,
licenses: []pkg.License{
licenses: []file.License{
{
Value: "Apache-2.0",
SPDXExpression: "Apache-2.0",
Type: "concluded",
URLs: nil,
Locations: file.NewLocationSet(testLocation),
Contents: "",
},
},
},
},
{
name: "custom license",
in: "test-fixtures/nvidia-software-and-cuda-supplement",
expected: expectation{
yieldError: false,
licenses: []pkg.License{
{
Value: "UNKNOWN",
SPDXExpression: "UNKNOWN_eebcea3ab1d1a28e671de90119ffcfb35fe86951e4af1b17af52b7a82fcf7d0a",
Type: "declared",
URLs: nil,
Locations: file.NewLocationSet(testLocation),
Contents: string(mustOpen("test-fixtures/nvidia-software-and-cuda-supplement")),
},
},
},
@@ -76,9 +57,11 @@ func TestSearch(t *testing.T) {
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
ctx := context.TODO()
content, err := os.ReadFile(test.in)
require.NoError(t, err)
result, err := Search(context.TODO(), testScanner(), file.NewLocationReadCloser(file.NewLocation("LICENSE"), io.NopCloser(bytes.NewReader(content))))
s := testScanner(false)
result, err := s.FileSearch(ctx, file.NewLocationReadCloser(file.NewLocation("LICENSE"), io.NopCloser(bytes.NewReader(content))))
if test.expected.yieldError {
require.Error(t, err)
} else {
@@ -93,3 +76,91 @@ func TestSearch(t *testing.T) {
})
}
}
func TestSearchPkgLicenses(t *testing.T) {
type expectation struct {
wantErr require.ErrorAssertionFunc
licenses []pkg.License
}
testLocation := file.NewLocation("LICENSE")
tests := []struct {
name string
in string
includeUnkownLicenseContent bool
expected expectation
}{
{
name: "apache license 2.0",
in: "test-fixtures/apache-license-2.0",
expected: expectation{
licenses: []pkg.License{
{
Value: "Apache-2.0",
SPDXExpression: "Apache-2.0",
Type: "concluded",
URLs: nil,
Locations: file.NewLocationSet(testLocation),
Contents: "",
},
},
wantErr: nil,
},
},
{
name: "custom license no content by default",
in: "test-fixtures/nvidia-software-and-cuda-supplement",
expected: expectation{
licenses: []pkg.License{
{
Value: "UNKNOWN",
SPDXExpression: "UNKNOWN_eebcea3ab1d1a28e671de90119ffcfb35fe86951e4af1b17af52b7a82fcf7d0a",
Type: "declared",
URLs: nil,
Locations: file.NewLocationSet(testLocation),
Contents: "",
},
},
wantErr: nil,
},
},
{
name: "custom license with content when scanner has content config",
in: "test-fixtures/nvidia-software-and-cuda-supplement",
includeUnkownLicenseContent: true,
expected: expectation{
licenses: []pkg.License{
{
Value: "UNKNOWN",
SPDXExpression: "UNKNOWN_eebcea3ab1d1a28e671de90119ffcfb35fe86951e4af1b17af52b7a82fcf7d0a",
Type: "declared",
URLs: nil,
Locations: file.NewLocationSet(testLocation),
Contents: string(mustOpen("test-fixtures/nvidia-software-and-cuda-supplement")),
},
},
wantErr: nil,
},
},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
ctx := context.TODO()
content, err := os.ReadFile(test.in)
require.NoError(t, err)
s := testScanner(test.includeUnkownLicenseContent)
result, err := s.PkgSearch(ctx, file.NewLocationReadCloser(file.NewLocation("LICENSE"), io.NopCloser(bytes.NewReader(content))))
if test.expected.wantErr != nil {
test.expected.wantErr(t, err)
}
require.NoError(t, err)
require.Len(t, result, len(test.expected.licenses))
if len(test.expected.licenses) > 0 {
require.Equal(t, test.expected.licenses, result)
}
})
}
}
internal/task/cataloging_config.go
+2
View File
@@ -11,6 +11,7 @@ type CatalogingFactoryConfig struct {
SearchConfig cataloging.SearchConfig
RelationshipsConfig cataloging.RelationshipsConfig
DataGenerationConfig cataloging.DataGenerationConfig
LicenseConfig cataloging.LicenseConfig
PackagesConfig pkgcataloging.Config
FilesConfig filecataloging.Config
}
@@ -21,6 +22,7 @@ func DefaultCatalogingFactoryConfig() CatalogingFactoryConfig {
SearchConfig: cataloging.DefaultSearchConfig(),
RelationshipsConfig: cataloging.DefaultRelationshipsConfig(),
DataGenerationConfig: cataloging.DefaultDataGenerationConfig(),
LicenseConfig: cataloging.DefaultLicenseConfig(),
PackagesConfig: pkgcataloging.DefaultConfig(),
FilesConfig: filecataloging.DefaultConfig(),
}