detect license ID from full text when incidentally provided as a value (#3876)

---------
Signed-off-by: Christopher Phillips <32073428+spiffcs@users.noreply.github.com>
Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>
Co-authored-by: Alex Goodman <wagoodman@users.noreply.github.com>
This commit is contained in:
Christopher Angelo Phillips
2025-05-13 16:37:18 -04:00
committed by GitHub
co-authored by Alex Goodman
parent b4d717fb30
commit f77d503892
117 changed files with 2082 additions and 1327 deletions
@@ -1,6 +1,7 @@
package helpers
import (
"context"
"encoding/base64"
"strings"
@@ -58,11 +59,11 @@ func decodeLicenses(c *cyclonedx.Component) []pkg.License {
// these fields are mutually exclusive in the spec
switch {
case l.License != nil && l.License.ID != "":
licenses = append(licenses, pkg.NewLicenseFromURLs(l.License.ID, l.License.URL))
licenses = append(licenses, pkg.NewLicenseFromURLsWithContext(context.TODO(), l.License.ID, l.License.URL))
case l.License != nil && l.License.Name != "":
licenses = append(licenses, pkg.NewLicenseFromURLs(l.License.Name, l.License.URL))
licenses = append(licenses, pkg.NewLicenseFromURLsWithContext(context.TODO(), l.License.Name, l.License.URL))
case l.Expression != "":
licenses = append(licenses, pkg.NewLicense(l.Expression))
licenses = append(licenses, pkg.NewLicenseWithContext(context.TODO(), l.Expression))
default:
}
}
@@ -1,6 +1,7 @@
package helpers
import (
"context"
"testing"
"github.com/CycloneDX/cyclonedx-go"
@@ -12,6 +13,7 @@ import (
)
func Test_encodeLicense(t *testing.T) {
ctx := context.TODO()
tests := []struct {
name string
input pkg.Package
@@ -25,7 +27,7 @@ func Test_encodeLicense(t *testing.T) {
name: "no SPDX licenses",
input: pkg.Package{
Licenses: pkg.NewLicenseSet(
pkg.NewLicense("RandomLicense"),
pkg.NewLicenseWithContext(ctx, "RandomLicense"),
),
},
expected: &cyclonedx.Licenses{
@@ -40,8 +42,8 @@ func Test_encodeLicense(t *testing.T) {
name: "single SPDX ID and Non SPDX ID",
input: pkg.Package{
Licenses: pkg.NewLicenseSet(
pkg.NewLicense("mit"),
pkg.NewLicense("FOOBAR"),
pkg.NewLicenseWithContext(ctx, "mit"),
pkg.NewLicenseWithContext(ctx, "FOOBAR"),
),
},
expected: &cyclonedx.Licenses{
@@ -61,7 +63,7 @@ func Test_encodeLicense(t *testing.T) {
name: "with complex SPDX license expression",
input: pkg.Package{
Licenses: pkg.NewLicenseSet(
pkg.NewLicense("MIT AND GPL-3.0-only"),
pkg.NewLicenseWithContext(ctx, "MIT AND GPL-3.0-only"),
),
},
expected: &cyclonedx.Licenses{
@@ -74,8 +76,8 @@ func Test_encodeLicense(t *testing.T) {
name: "with multiple complex SPDX license expression",
input: pkg.Package{
Licenses: pkg.NewLicenseSet(
pkg.NewLicense("MIT AND GPL-3.0-only"),
pkg.NewLicense("MIT AND GPL-3.0-only WITH Classpath-exception-2.0"),
pkg.NewLicenseWithContext(ctx, "MIT AND GPL-3.0-only"),
pkg.NewLicenseWithContext(ctx, "MIT AND GPL-3.0-only WITH Classpath-exception-2.0"),
),
},
expected: &cyclonedx.Licenses{
@@ -88,9 +90,9 @@ func Test_encodeLicense(t *testing.T) {
name: "with multiple URLs and expressions",
input: pkg.Package{
Licenses: pkg.NewLicenseSet(
pkg.NewLicenseFromURLs("MIT", "https://opensource.org/licenses/MIT", "https://spdx.org/licenses/MIT.html"),
pkg.NewLicense("MIT AND GPL-3.0-only"),
pkg.NewLicenseFromURLs("FakeLicense", "htts://someurl.com"),
pkg.NewLicenseFromURLsWithContext(ctx, "MIT", "https://opensource.org/licenses/MIT", "https://spdx.org/licenses/MIT.html"),
pkg.NewLicenseWithContext(ctx, "MIT AND GPL-3.0-only"),
pkg.NewLicenseFromURLsWithContext(ctx, "FakeLicense", "htts://someurl.com"),
),
},
expected: &cyclonedx.Licenses{
@@ -123,8 +125,8 @@ func Test_encodeLicense(t *testing.T) {
name: "with multiple values licenses are deduplicated",
input: pkg.Package{
Licenses: pkg.NewLicenseSet(
pkg.NewLicense("Apache-2"),
pkg.NewLicense("Apache-2.0"),
pkg.NewLicenseWithContext(ctx, "Apache-2"),
pkg.NewLicenseWithContext(ctx, "Apache-2.0"),
),
},
expected: &cyclonedx.Licenses{
@@ -139,9 +141,9 @@ func Test_encodeLicense(t *testing.T) {
name: "with multiple URLs and single with no URLs",
input: pkg.Package{
Licenses: pkg.NewLicenseSet(
pkg.NewLicense("MIT"),
pkg.NewLicenseFromURLs("MIT", "https://opensource.org/licenses/MIT", "https://spdx.org/licenses/MIT.html"),
pkg.NewLicense("MIT AND GPL-3.0-only"),
pkg.NewLicenseWithContext(ctx, "MIT"),
pkg.NewLicenseFromURLsWithContext(ctx, "MIT", "https://opensource.org/licenses/MIT", "https://spdx.org/licenses/MIT.html"),
pkg.NewLicenseWithContext(ctx, "MIT AND GPL-3.0-only"),
),
},
expected: &cyclonedx.Licenses{
@@ -167,7 +169,7 @@ func Test_encodeLicense(t *testing.T) {
{
name: "single parenthesized SPDX expression",
input: pkg.Package{
Licenses: pkg.NewLicenseSet(pkg.NewLicensesFromValues("(MIT OR Apache-2.0)")...),
Licenses: pkg.NewLicenseSet(pkg.NewLicensesFromValuesWithContext(ctx, "(MIT OR Apache-2.0)")...),
},
expected: &cyclonedx.Licenses{
{
@@ -179,7 +181,7 @@ func Test_encodeLicense(t *testing.T) {
name: "single license AND to parenthesized SPDX expression",
// (LGPL-3.0-or-later OR GPL-2.0-or-later OR (LGPL-3.0-or-later AND GPL-2.0-or-later)) AND GFDL-1.3-invariants-or-later
input: pkg.Package{
Licenses: pkg.NewLicenseSet(pkg.NewLicensesFromValues("(LGPL-3.0-or-later OR GPL-2.0-or-later OR (LGPL-3.0-or-later AND GPL-2.0-or-later)) AND GFDL-1.3-invariants-or-later")...),
Licenses: pkg.NewLicenseSet(pkg.NewLicensesFromValuesWithContext(ctx, "(LGPL-3.0-or-later OR GPL-2.0-or-later OR (LGPL-3.0-or-later AND GPL-2.0-or-later)) AND GFDL-1.3-invariants-or-later")...),
},
expected: &cyclonedx.Licenses{
{
@@ -248,7 +250,6 @@ func TestDecodeLicenses(t *testing.T) {
Value: "RandomLicense",
// CycloneDX specification doesn't give a field for determining the license type
Type: license.Declared,
URLs: []string{},
},
},
},
@@ -268,7 +269,6 @@ func TestDecodeLicenses(t *testing.T) {
Value: "MIT",
SPDXExpression: "MIT",
Type: license.Declared,
URLs: []string{},
},
},
},
@@ -1,16 +1,17 @@
package helpers
import (
"crypto/sha256"
"fmt"
"sort"
"strings"
"github.com/spdx/tools-golang/spdx"
"github.com/anchore/syft/internal/spdxlicense"
"github.com/anchore/syft/syft/license"
"github.com/anchore/syft/syft/pkg"
)
func License(p pkg.Package) (concluded, declared string) {
func License(p pkg.Package) (concluded, declared string, otherLicenses []spdx.OtherLicense) {
// source: https://spdx.github.io/spdx-spec/v2.3/package-information/#713-concluded-license-field
// The options to populate this field are limited to:
// A valid SPDX License Expression as defined in Annex D;
@@ -21,15 +22,15 @@ func License(p pkg.Package) (concluded, declared string) {
// (iii) the SPDX file creator has intentionally provided no information (no meaning should be implied by doing so).
if p.Licenses.Empty() {
return NOASSERTION, NOASSERTION
return NOASSERTION, NOASSERTION, nil
}
// take all licenses and assume an AND expression;
// for information about license expressions see:
// https://spdx.github.io/spdx-spec/v2.3/SPDX-license-expressions/
pc, pd := ParseLicenses(p.Licenses.ToSlice())
pc, pd, ol := ParseLicenses(p.Licenses.ToSlice())
return joinLicenses(pc), joinLicenses(pd)
return joinLicenses(pc), joinLicenses(pd), ol
}
func joinLicenses(licenses []SPDXLicense) string {
@@ -58,14 +59,32 @@ func joinLicenses(licenses []SPDXLicense) string {
}
type SPDXLicense struct {
ID string
Value string
FullText string
// Valid SPDX ID OR License Value (should have LicenseRef- prefix and be sanitized)
// OR combination of the above as a valid SPDX License Expression as defined in Annex D.
// https://spdx.github.io/spdx-spec/v2.3/SPDX-license-expressions/
ID string
// If the SPDX license is not on the SPDX License List
LicenseName string
FullText string // 0..1 (Mandatory, one) if there is a License Identifier assigned (LicenseRef).
URLs []string
}
func ParseLicenses(raw []pkg.License) (concluded, declared []SPDXLicense) {
func ParseLicenses(raw []pkg.License) (concluded, declared []SPDXLicense, otherLicenses []spdx.OtherLicense) {
for _, l := range raw {
candidate := createSPDXLicense(l)
// isCustomLicense determines if the candidate falls under https://spdx.github.io/spdx-spec/v2.3/other-licensing-information-detected/#
// of the SPDX spec, where:
// - we should not have a complex SPDX expression
// - if a single license, it should not be a known license (on the SPDX license list)
if l.SPDXExpression == "" && strings.Contains(candidate.ID, spdxlicense.LicenseRefPrefix) {
otherLicenses = append(otherLicenses, spdx.OtherLicense{
LicenseIdentifier: candidate.ID,
ExtractedText: candidate.FullText,
LicenseName: candidate.LicenseName,
LicenseCrossReferences: candidate.URLs,
})
}
switch l.Type {
case license.Concluded:
concluded = append(concluded, candidate)
@@ -74,35 +93,70 @@ func ParseLicenses(raw []pkg.License) (concluded, declared []SPDXLicense) {
}
}
return concluded, declared
return concluded, declared, otherLicenses
}
func createSPDXLicense(l pkg.License) SPDXLicense {
candidate := SPDXLicense{
ID: generateLicenseID(l),
FullText: l.Contents,
// source: https://spdx.github.io/spdx-spec/v2.3/other-licensing-information-detected/#102-extracted-text-field
// we need to populate this field in the spdx document if we have a license ref
// 0..1 (Mandatory, one) if there is a License Identifier assigned (LicenseRef).
ft := NOASSERTION
if l.Contents != "" {
ft = l.Contents
}
if l.SPDXExpression == "" {
candidate.Value = l.Value
return SPDXLicense{
ID: generateLicenseID(l),
LicenseName: l.Value,
FullText: ft,
URLs: l.URLs,
}
return candidate
}
// generateLicenseID generates a license ID for the given license, which is either the license value or the SPDX expression.
func generateLicenseID(l pkg.License) string {
if l.SPDXExpression != "" {
return l.SPDXExpression
}
if l.Value != "" {
return spdxlicense.LicenseRefPrefix + SanitizeElementID(l.Value)
// syft format includes the algo for the sha in the values
// we can strip this and just make LicenseRef-<sum> for spdx consumption
id := strings.ReplaceAll(l.Value, "sha256:", "")
if !strings.HasPrefix(id, "LicenseRef-") {
id = "LicenseRef-" + id
}
return licenseSum(l.Contents)
return SanitizeElementID(id)
}
func licenseSum(s string) string {
if len(s) <= 64 {
return spdxlicense.LicenseRefPrefix + SanitizeElementID(s)
type SPDXOtherLicenseSet struct {
set map[string]spdx.OtherLicense
}
func NewSPDXOtherLicenseSet() *SPDXOtherLicenseSet {
return &SPDXOtherLicenseSet{
set: make(map[string]spdx.OtherLicense),
}
hash := sha256.Sum256([]byte(s))
return fmt.Sprintf("%s%x", spdxlicense.LicenseRefPrefix, hash)
}
func (s *SPDXOtherLicenseSet) Add(licenses ...spdx.OtherLicense) {
for _, l := range licenses {
s.set[l.LicenseIdentifier] = l
}
}
type ByLicenseIdentifier []spdx.OtherLicense
func (o ByLicenseIdentifier) Len() int { return len(o) }
func (o ByLicenseIdentifier) Swap(i, j int) { o[i], o[j] = o[j], o[i] }
func (o ByLicenseIdentifier) Less(i, j int) bool {
return o[i].LicenseIdentifier < o[j].LicenseIdentifier
}
func (s *SPDXOtherLicenseSet) ToSlice() []spdx.OtherLicense {
values := make([]spdx.OtherLicense, 0, len(s.set))
for _, v := range s.set {
values = append(values, v)
}
sort.Sort(ByLicenseIdentifier(values))
return values
}
@@ -1,9 +1,10 @@
package helpers
import (
"strings"
"context"
"testing"
"github.com/google/go-cmp/cmp"
"github.com/stretchr/testify/assert"
"github.com/anchore/syft/internal/spdxlicense"
@@ -11,6 +12,7 @@ import (
)
func Test_License(t *testing.T) {
ctx := context.TODO()
type expected struct {
concluded string
declared string
@@ -31,7 +33,7 @@ func Test_License(t *testing.T) {
{
name: "no SPDX licenses",
input: pkg.Package{
Licenses: pkg.NewLicenseSet(pkg.NewLicense("made-up")),
Licenses: pkg.NewLicenseSet(pkg.NewLicenseWithContext(ctx, "made-up")),
},
expected: expected{
concluded: "NOASSERTION",
@@ -41,7 +43,7 @@ func Test_License(t *testing.T) {
{
name: "with SPDX license",
input: pkg.Package{
Licenses: pkg.NewLicenseSet(pkg.NewLicense("MIT")),
Licenses: pkg.NewLicenseSet(pkg.NewLicenseWithContext(ctx, "MIT")),
},
expected: struct {
concluded string
@@ -55,8 +57,8 @@ func Test_License(t *testing.T) {
name: "with SPDX license expression",
input: pkg.Package{
Licenses: pkg.NewLicenseSet(
pkg.NewLicense("MIT"),
pkg.NewLicense("GPL-3.0-only"),
pkg.NewLicenseWithContext(ctx, "MIT"),
pkg.NewLicenseWithContext(ctx, "GPL-3.0-only"),
),
},
expected: expected{
@@ -69,9 +71,9 @@ func Test_License(t *testing.T) {
name: "includes valid LicenseRef-",
input: pkg.Package{
Licenses: pkg.NewLicenseSet(
pkg.NewLicense("one thing first"),
pkg.NewLicense("two things/#$^second"),
pkg.NewLicense("MIT"),
pkg.NewLicenseWithContext(ctx, "one thing first"),
pkg.NewLicenseWithContext(ctx, "two things/#$^second"),
pkg.NewLicenseWithContext(ctx, "MIT"),
),
},
expected: expected{
@@ -84,9 +86,9 @@ func Test_License(t *testing.T) {
name: "join parentheses correctly",
input: pkg.Package{
Licenses: pkg.NewLicenseSet(
pkg.NewLicense("one thing first"),
pkg.NewLicense("MIT AND GPL-3.0-only"),
pkg.NewLicense("MIT OR APACHE-2.0"),
pkg.NewLicenseWithContext(ctx, "one thing first"),
pkg.NewLicenseWithContext(ctx, "MIT AND GPL-3.0-only"),
pkg.NewLicenseWithContext(ctx, "MIT OR APACHE-2.0"),
),
},
expected: expected{
@@ -98,7 +100,7 @@ func Test_License(t *testing.T) {
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
c, d := License(test.input)
c, d, _ := License(test.input)
assert.Equal(t, test.expected.concluded, c)
assert.Equal(t, test.expected.declared, d)
})
@@ -123,11 +125,13 @@ func TestGenerateLicenseID(t *testing.T) {
{
name: "Uses value if no SPDX expression",
license: pkg.License{
Value: "MIT",
Value: "my-sweet-custom-license",
},
expected: spdxlicense.LicenseRefPrefix + "MIT",
expected: spdxlicense.LicenseRefPrefix + "my-sweet-custom-license",
},
{
// note: this is an oversight of the SPDX spec. It does NOT allow "+" in the ID even though they are
// significant to the licenses in the expressions below
name: "Long value is sanitized correctly",
license: pkg.License{
Value: "LGPLv2+ and LGPLv2+ with exceptions and GPLv2+ and GPLv2+ with exceptions and BSD and Inner-Net and ISC and Public Domain and GFDL",
@@ -135,13 +139,6 @@ func TestGenerateLicenseID(t *testing.T) {
expected: spdxlicense.LicenseRefPrefix +
"LGPLv2--and-LGPLv2--with-exceptions-and-GPLv2--and-GPLv2--with-exceptions-and-BSD-and-Inner-Net-and-ISC-and-Public-Domain-and-GFDL",
},
{
name: "Uses hash of contents when nothing else is provided",
license: pkg.License{
Contents: "This is a very long custom license text that should be hashed because it's more than 64 characters long.",
},
expected: "", // We'll verify it starts with the correct prefix
},
}
for _, tt := range tests {
@@ -160,34 +157,92 @@ func TestGenerateLicenseID(t *testing.T) {
func Test_joinLicenses(t *testing.T) {
tests := []struct {
name string
args []string
args []SPDXLicense
want string
}{
{
name: "multiple licenses",
args: []string{"MIT", "GPL-3.0-only"},
args: []SPDXLicense{{ID: "MIT"}, {ID: "GPL-3.0-only"}},
want: "MIT AND GPL-3.0-only",
},
{
name: "multiple licenses with complex expressions",
args: []string{"MIT AND Apache", "GPL-3.0-only"},
args: []SPDXLicense{{ID: "MIT AND Apache"}, {ID: "GPL-3.0-only"}},
want: "(MIT AND Apache) AND GPL-3.0-only",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
assert.Equalf(t, tt.want, joinLicenses(toSpdxLicenses(tt.args)), "joinLicenses(%v)", tt.args)
assert.Equalf(t, tt.want, joinLicenses(tt.args), "joinLicenses(%v)", tt.args)
})
}
}
func toSpdxLicenses(ids []string) (licenses []SPDXLicense) {
for _, l := range ids {
license := SPDXLicense{ID: l}
if strings.HasPrefix(l, spdxlicense.LicenseRefPrefix) {
license.Value = l
}
licenses = append(licenses, license)
func TestCreateSPDXLicenseAndGenerateLicenseID(t *testing.T) {
tests := []struct {
name string
input pkg.License
expected SPDXLicense
}{
{
name: "SPDX expression used as ID",
input: pkg.License{
SPDXExpression: "MIT",
Value: "MIT",
Contents: "",
},
expected: SPDXLicense{
ID: "MIT",
LicenseName: "MIT",
FullText: "NOASSERTION",
},
},
{
name: "LicenseRef with contents",
input: pkg.License{
Value: "sha256:123abc",
Contents: "license contents here",
},
expected: SPDXLicense{
ID: "LicenseRef-123abc",
LicenseName: "sha256:123abc",
FullText: "license contents here",
},
},
{
name: "LicenseRef without contents",
input: pkg.License{
Value: "custom-license",
Contents: "",
},
expected: SPDXLicense{
ID: "LicenseRef-custom-license",
LicenseName: "custom-license",
FullText: "NOASSERTION",
},
},
{
name: "URL is passed through",
input: pkg.License{
SPDXExpression: "MIT",
URLs: []string{
"https://example.com/license",
},
},
expected: SPDXLicense{
ID: "MIT",
FullText: "NOASSERTION",
URLs: []string{"https://example.com/license"},
},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
license := createSPDXLicense(tt.input)
if d := cmp.Diff(tt.expected, license); d != "" {
t.Errorf("createSPDXLicense() mismatch (-want +got):\n%s", d)
}
})
}
return licenses
}
@@ -1,6 +1,7 @@
package testutil
import (
"context"
"os"
"path/filepath"
"testing"
@@ -98,7 +99,7 @@ func DirectoryInputWithAuthorField(t testing.TB) sbom.SBOM {
func newDirectoryCatalog() *pkg.Collection {
catalog := pkg.NewCollection()
ctx := context.TODO()
// populate catalog with test data
catalog.Add(pkg.Package{
Name: "package-1",
@@ -110,7 +111,7 @@ func newDirectoryCatalog() *pkg.Collection {
),
Language: pkg.Python,
Licenses: pkg.NewLicenseSet(
pkg.NewLicense("MIT"),
pkg.NewLicenseWithContext(ctx, "MIT"),
),
Metadata: pkg.PythonPackage{
Name: "package-1",
@@ -149,7 +150,7 @@ func newDirectoryCatalog() *pkg.Collection {
func newDirectoryCatalogWithAuthorField() *pkg.Collection {
catalog := pkg.NewCollection()
ctx := context.TODO()
// populate catalog with test data
catalog.Add(pkg.Package{
Name: "package-1",
@@ -161,7 +162,7 @@ func newDirectoryCatalogWithAuthorField() *pkg.Collection {
),
Language: pkg.Python,
Licenses: pkg.NewLicenseSet(
pkg.NewLicense("MIT"),
pkg.NewLicenseWithContext(ctx, "MIT"),
),
Metadata: pkg.PythonPackage{
Name: "package-1",
+3 -2
View File
@@ -1,6 +1,7 @@
package testutil
import (
"context"
"os"
"path/filepath"
"testing"
@@ -98,7 +99,7 @@ func populateImageCatalog(catalog *pkg.Collection, img *image.Image) {
// TODO: this helper function is coupled to the image-simple fixture, which seems like a bad idea
_, ref1, _ := img.SquashedTree().File("/somefile-1.txt", filetree.FollowBasenameLinks)
_, ref2, _ := img.SquashedTree().File("/somefile-2.txt", filetree.FollowBasenameLinks)
ctx := context.TODO()
// populate catalog with test data
if ref1 != nil {
catalog.Add(pkg.Package{
@@ -111,7 +112,7 @@ func populateImageCatalog(catalog *pkg.Collection, img *image.Image) {
FoundBy: "the-cataloger-1",
Language: pkg.Python,
Licenses: pkg.NewLicenseSet(
pkg.NewLicense("MIT"),
pkg.NewLicenseWithContext(ctx, "MIT"),
),
Metadata: pkg.PythonPackage{
Name: "package-1",