detect license ID from full text when incidentally provided as a value (#3876)

---------
Signed-off-by: Christopher Phillips <32073428+spiffcs@users.noreply.github.com>
Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>
Co-authored-by: Alex Goodman <wagoodman@users.noreply.github.com>
This commit is contained in:
Christopher Angelo Phillips
2025-05-13 16:37:18 -04:00
committed by GitHub
co-authored by Alex Goodman
parent b4d717fb30
commit f77d503892
117 changed files with 2082 additions and 1327 deletions
@@ -5,7 +5,6 @@ import (
"crypto/sha1"
"fmt"
"path"
"regexp"
"slices"
"sort"
"strings"
@@ -15,7 +14,6 @@ import (
"github.com/spdx/tools-golang/spdx"
"github.com/anchore/packageurl-go"
internallicenses "github.com/anchore/syft/internal/licenses"
"github.com/anchore/syft/internal/log"
"github.com/anchore/syft/internal/mimetype"
"github.com/anchore/syft/internal/relationship"
@@ -50,7 +48,7 @@ func ToFormatModel(s sbom.SBOM) *spdx.Document {
name, namespace := helpers.DocumentNameAndNamespace(s.Source, s.Descriptor)
rels := relationship.NewIndex(s.Relationships...)
packages := toPackages(rels, s.Artifacts.Packages, s)
packages, otherLicenses := toPackages(rels, s.Artifacts.Packages, s)
allRelationships := toRelationships(rels.All())
@@ -156,7 +154,7 @@ func ToFormatModel(s sbom.SBOM) *spdx.Document {
Packages: packages,
Files: toFiles(s),
Relationships: allRelationships,
OtherLicenses: toOtherLicenses(s.Artifacts.Packages),
OtherLicenses: convertOtherLicense(otherLicenses),
}
}
@@ -319,16 +317,18 @@ func toSPDXID(identifiable artifact.Identifiable) spdx.ElementID {
// packages populates all Package Information from the package Collection (see https://spdx.github.io/spdx-spec/3-package-information/)
//
//nolint:funlen
func toPackages(rels *relationship.Index, catalog *pkg.Collection, sbom sbom.SBOM) (results []*spdx.Package) {
func toPackages(rels *relationship.Index, catalog *pkg.Collection, sbom sbom.SBOM) (results []*spdx.Package, otherLicenses []spdx.OtherLicense) {
otherLicenseSet := helpers.NewSPDXOtherLicenseSet()
for _, p := range catalog.Sorted() {
// name should be guaranteed to be unique, but semantically useful and stable
// name should be guaranteed to be unique but semantically useful and stable
id := toSPDXID(p)
// If the Concluded License is not the same as the Declared License, a written explanation should be provided
// in the Comments on License field (section 7.16). With respect to NOASSERTION, a written explanation in
// the Comments on License field (section 7.16) is preferred.
// extract these correctly to the spdx license format
concluded, declared := helpers.License(p)
concluded, declared, ol := helpers.License(p)
otherLicenseSet.Add(ol...)
// two ways to get filesAnalyzed == true:
// 1. syft has generated a sha1 digest for the package itself - usually in the java cataloger
@@ -487,7 +487,7 @@ func toPackages(rels *relationship.Index, catalog *pkg.Collection, sbom sbom.SBO
PackageAttributionTexts: nil,
})
}
return results
return results, otherLicenseSet.ToSlice()
}
func toPackageChecksums(p pkg.Package) ([]spdx.Checksum, bool) {
@@ -740,81 +740,14 @@ func toFileTypes(metadata *file.Metadata) (ty []string) {
return ty
}
// other licenses are for licenses from the pkg.Package that do not have a valid SPDX Expression
// OR are an expression that is a single `License-Ref-*`
func toOtherLicenses(catalog *pkg.Collection) []*spdx.OtherLicense {
licenses := map[string]helpers.SPDXLicense{}
for p := range catalog.Enumerate() {
declaredLicenses, concludedLicenses := helpers.ParseLicenses(p.Licenses.ToSlice())
for _, l := range declaredLicenses {
if l.Value != "" {
licenses[l.ID] = l
}
if l.ID != "" && isLicenseRef(l.ID) {
licenses[l.ID] = l
}
}
for _, l := range concludedLicenses {
if l.Value != "" {
licenses[l.ID] = l
}
if l.ID != "" && isLicenseRef(l.ID) {
licenses[l.ID] = l
}
}
}
var result []*spdx.OtherLicense
var ids []string
for licenseID := range licenses {
ids = append(ids, licenseID)
}
slices.Sort(ids)
for _, id := range ids {
license := licenses[id]
value := license.Value
fullText := license.FullText
// handle cases where LicenseRef needs to be included in hasExtractedLicensingInfos
if license.Value == "" {
value, _ = strings.CutPrefix(license.ID, "LicenseRef-")
}
other := &spdx.OtherLicense{
LicenseIdentifier: license.ID,
}
if fullText != "" {
other.ExtractedText = fullText
} else {
other.ExtractedText = value
}
customPrefix := spdxlicense.LicenseRefPrefix + helpers.SanitizeElementID(internallicenses.UnknownLicensePrefix)
if strings.HasPrefix(license.ID, customPrefix) {
other.LicenseName = strings.TrimPrefix(license.ID, customPrefix)
other.LicenseComment = strings.Trim(internallicenses.UnknownLicensePrefix, "-_")
}
result = append(result, other)
}
return result
}
var licenseRefRegEx = regexp.MustCompile(`^LicenseRef-[A-Za-z0-9_-]+$`)
// isSingularLicenseRef checks if the string is a singular LicenseRef-* identifier
func isLicenseRef(s string) bool {
// Match the input string against the regex
return licenseRefRegEx.MatchString(s)
}
// TODO: handle SPDX excludes file case
// f file is an "excludes" file, skip it /* exclude SPDX analysis file(s) */
// see: https://spdx.github.io/spdx-spec/v2.3/package-information/#79-package-verification-code-field
// the above link contains the SPDX algorithm for a package verification code
func newPackageVerificationCode(rels *relationship.Index, p pkg.Package, sbom sbom.SBOM) *spdx.PackageVerificationCode {
// key off of the contains relationship;
// spdx validator will fail if a package claims to contain a file but no sha1 provided
// if a sha1 for a file is provided then the validator will fail if the package does not have
// key off of the spdx contains relationship;
// spdx validator will fail if a package claims to contain a file, but no sha1 provided
// if a sha1 for a file is provided, then the validator will fail if the package does not have
// a package verification code
coordinates := rels.Coordinates(p, artifact.ContainsRelationship)
var digests []file.Digest
@@ -887,3 +820,15 @@ func convertAbsoluteToRelative(absPath string) (string, error) {
return relPath, nil
}
func convertOtherLicense(otherLicenses []spdx.OtherLicense) []*spdx.OtherLicense {
if len(otherLicenses) == 0 {
return nil
}
result := make([]*spdx.OtherLicense, 0, len(otherLicenses))
for i := range otherLicenses {
result = append(result, &otherLicenses[i])
}
return result
}
@@ -1,8 +1,10 @@
package spdxhelpers
import (
"context"
"fmt"
"regexp"
"strings"
"testing"
"github.com/google/go-cmp/cmp"
@@ -736,7 +738,7 @@ func Test_H1Digest(t *testing.T) {
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
catalog := pkg.NewCollection(test.pkg)
pkgs := toPackages(relationship.NewIndex(), catalog, s)
pkgs, _ := toPackages(relationship.NewIndex(), catalog, s)
require.Len(t, pkgs, 1)
for _, p := range pkgs {
if test.expectedDigest == "" {
@@ -753,29 +755,31 @@ func Test_H1Digest(t *testing.T) {
}
func Test_OtherLicenses(t *testing.T) {
ctx := context.Background()
tests := []struct {
name string
pkg pkg.Package
expected []*spdx.OtherLicense
expected []spdx.OtherLicense
}{
{
name: "no licenseRef",
pkg: pkg.Package{
Licenses: pkg.NewLicenseSet(),
},
expected: nil,
expected: []spdx.OtherLicense{},
},
{
name: "single licenseRef",
pkg: pkg.Package{
Licenses: pkg.NewLicenseSet(
pkg.NewLicense("foobar"),
pkg.NewLicenseWithContext(ctx, "foobar"),
),
},
expected: []*spdx.OtherLicense{
expected: []spdx.OtherLicense{
{
LicenseIdentifier: "LicenseRef-foobar",
ExtractedText: "foobar",
LicenseName: "foobar",
ExtractedText: "NOASSERTION",
},
},
},
@@ -783,18 +787,20 @@ func Test_OtherLicenses(t *testing.T) {
name: "multiple licenseRef",
pkg: pkg.Package{
Licenses: pkg.NewLicenseSet(
pkg.NewLicense("internal made up license name"),
pkg.NewLicense("new apple license 2.0"),
pkg.NewLicenseWithContext(ctx, "internal made up license name"),
pkg.NewLicenseWithContext(ctx, "new apple license 2.0"),
),
},
expected: []*spdx.OtherLicense{
expected: []spdx.OtherLicense{
{
LicenseIdentifier: "LicenseRef-internal-made-up-license-name",
ExtractedText: "internal made up license name",
ExtractedText: "NOASSERTION",
LicenseName: "internal made up license name",
},
{
LicenseIdentifier: "LicenseRef-new-apple-license-2.0",
ExtractedText: "new apple license 2.0",
ExtractedText: "NOASSERTION",
LicenseName: "new apple license 2.0",
},
},
},
@@ -802,31 +808,27 @@ func Test_OtherLicenses(t *testing.T) {
name: "LicenseRef as a valid spdx expression",
pkg: pkg.Package{
Licenses: pkg.NewLicenseSet(
pkg.NewLicense("LicenseRef-Fedora-Public-Domain"),
pkg.NewLicenseWithContext(ctx, "LicenseRef-Fedora-Public-Domain"),
),
},
expected: []*spdx.OtherLicense{
{
LicenseIdentifier: "LicenseRef-Fedora-Public-Domain",
ExtractedText: "Fedora-Public-Domain",
},
},
expected: []spdx.OtherLicense{},
},
{
name: "LicenseRef as a valid spdx expression does not otherize compound spdx expressions",
pkg: pkg.Package{
Licenses: pkg.NewLicenseSet(
pkg.NewLicense("(MIT AND LicenseRef-Fedora-Public-Domain)"),
pkg.NewLicenseWithContext(ctx, "(MIT AND LicenseRef-Fedora-Public-Domain)"),
),
},
expected: nil,
expected: []spdx.OtherLicense{},
},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
catalog := pkg.NewCollection(test.pkg)
otherLicenses := toOtherLicenses(catalog)
rels := relationship.NewIndex()
_, otherLicenses := toPackages(rels, catalog, sbom.SBOM{})
require.Len(t, otherLicenses, len(test.expected))
require.Equal(t, test.expected, otherLicenses)
})
@@ -902,18 +904,19 @@ func Test_toSPDXID(t *testing.T) {
}
func Test_otherLicenses(t *testing.T) {
ctx := context.TODO()
pkg1 := pkg.Package{
Name: "first-pkg",
Version: "1.1",
Licenses: pkg.NewLicenseSet(
pkg.NewLicense("MIT"),
pkg.NewLicenseWithContext(ctx, "MIT"),
),
}
pkg2 := pkg.Package{
Name: "second-pkg",
Version: "2.2",
Licenses: pkg.NewLicenseSet(
pkg.NewLicense("non spdx license"),
pkg.NewLicenseWithContext(ctx, "non spdx license"),
),
}
bigText := `
@@ -923,7 +926,7 @@ func Test_otherLicenses(t *testing.T) {
Name: "third-pkg",
Version: "3.3",
Licenses: pkg.NewLicenseSet(
pkg.NewLicense(bigText),
pkg.NewLicenseWithContext(ctx, bigText),
),
}
@@ -938,22 +941,25 @@ func Test_otherLicenses(t *testing.T) {
expected: nil,
},
{
name: "other licenses includes original text",
name: "other licenses must include some original text",
packages: []pkg.Package{pkg2},
expected: []*spdx.OtherLicense{
{
LicenseIdentifier: "LicenseRef-non-spdx-license",
ExtractedText: "non spdx license",
LicenseName: "non spdx license",
ExtractedText: "NOASSERTION",
},
},
},
{
name: "big licenses get hashed",
name: "big licenses get hashed and space is trimmed",
packages: []pkg.Package{pkg3},
expected: []*spdx.OtherLicense{
{
LicenseIdentifier: "LicenseRef-e9a1e42833d3e456f147052f4d312101bd171a0798893169fe596ca6b55c049e",
ExtractedText: bigText,
LicenseIdentifier: "LicenseRef-3f17782eef51ae86f18fdd6832f5918e2b40f688b52c9adc07ba6ec1024ef408",
// Carries through the syft-json license value when we shasum large texts
LicenseName: "LicenseRef-sha256:3f17782eef51ae86f18fdd6832f5918e2b40f688b52c9adc07ba6ec1024ef408",
ExtractedText: strings.TrimSpace(bigText),
},
},
},
@@ -1,6 +1,7 @@
package spdxhelpers
import (
"context"
"errors"
"fmt"
"net/url"
@@ -535,14 +536,14 @@ func parseSPDXLicenses(p *spdx.Package) []pkg.License {
// concluded
if p.PackageLicenseConcluded != helpers.NOASSERTION && p.PackageLicenseConcluded != helpers.NONE && p.PackageLicenseConcluded != "" {
l := pkg.NewLicense(cleanSPDXID(p.PackageLicenseConcluded))
l := pkg.NewLicenseWithContext(context.TODO(), cleanSPDXID(p.PackageLicenseConcluded))
l.Type = license.Concluded
licenses = append(licenses, l)
}
// declared
if p.PackageLicenseDeclared != helpers.NOASSERTION && p.PackageLicenseDeclared != helpers.NONE && p.PackageLicenseDeclared != "" {
l := pkg.NewLicense(cleanSPDXID(p.PackageLicenseDeclared))
l := pkg.NewLicenseWithContext(context.TODO(), cleanSPDXID(p.PackageLicenseDeclared))
l.Type = license.Declared
licenses = append(licenses, l)
}
@@ -24,7 +24,7 @@
},
"components": [
{
"bom-ref": "4dd25c6ee16b729a",
"bom-ref": "f04d218ff5ff50db",
"type": "library",
"name": "package-1",
"version": "1.0.1",
@@ -25,7 +25,7 @@
},
"components": [
{
"bom-ref": "72567175418f73f8",
"bom-ref": "2f52f617f1548337",
"type": "library",
"name": "package-1",
"version": "1.0.1",
@@ -16,7 +16,7 @@
</component>
</metadata>
<components>
<component bom-ref="4dd25c6ee16b729a" type="library">
<component bom-ref="f04d218ff5ff50db" type="library">
<name>package-1</name>
<version>1.0.1</version>
<licenses>
@@ -17,7 +17,7 @@
</component>
</metadata>
<components>
<component bom-ref="72567175418f73f8" type="library">
<component bom-ref="2f52f617f1548337" type="library">
<name>package-1</name>
<version>1.0.1</version>
<licenses>
@@ -1,6 +1,7 @@
package helpers
import (
"context"
"encoding/base64"
"strings"
@@ -58,11 +59,11 @@ func decodeLicenses(c *cyclonedx.Component) []pkg.License {
// these fields are mutually exclusive in the spec
switch {
case l.License != nil && l.License.ID != "":
licenses = append(licenses, pkg.NewLicenseFromURLs(l.License.ID, l.License.URL))
licenses = append(licenses, pkg.NewLicenseFromURLsWithContext(context.TODO(), l.License.ID, l.License.URL))
case l.License != nil && l.License.Name != "":
licenses = append(licenses, pkg.NewLicenseFromURLs(l.License.Name, l.License.URL))
licenses = append(licenses, pkg.NewLicenseFromURLsWithContext(context.TODO(), l.License.Name, l.License.URL))
case l.Expression != "":
licenses = append(licenses, pkg.NewLicense(l.Expression))
licenses = append(licenses, pkg.NewLicenseWithContext(context.TODO(), l.Expression))
default:
}
}
@@ -1,6 +1,7 @@
package helpers
import (
"context"
"testing"
"github.com/CycloneDX/cyclonedx-go"
@@ -12,6 +13,7 @@ import (
)
func Test_encodeLicense(t *testing.T) {
ctx := context.TODO()
tests := []struct {
name string
input pkg.Package
@@ -25,7 +27,7 @@ func Test_encodeLicense(t *testing.T) {
name: "no SPDX licenses",
input: pkg.Package{
Licenses: pkg.NewLicenseSet(
pkg.NewLicense("RandomLicense"),
pkg.NewLicenseWithContext(ctx, "RandomLicense"),
),
},
expected: &cyclonedx.Licenses{
@@ -40,8 +42,8 @@ func Test_encodeLicense(t *testing.T) {
name: "single SPDX ID and Non SPDX ID",
input: pkg.Package{
Licenses: pkg.NewLicenseSet(
pkg.NewLicense("mit"),
pkg.NewLicense("FOOBAR"),
pkg.NewLicenseWithContext(ctx, "mit"),
pkg.NewLicenseWithContext(ctx, "FOOBAR"),
),
},
expected: &cyclonedx.Licenses{
@@ -61,7 +63,7 @@ func Test_encodeLicense(t *testing.T) {
name: "with complex SPDX license expression",
input: pkg.Package{
Licenses: pkg.NewLicenseSet(
pkg.NewLicense("MIT AND GPL-3.0-only"),
pkg.NewLicenseWithContext(ctx, "MIT AND GPL-3.0-only"),
),
},
expected: &cyclonedx.Licenses{
@@ -74,8 +76,8 @@ func Test_encodeLicense(t *testing.T) {
name: "with multiple complex SPDX license expression",
input: pkg.Package{
Licenses: pkg.NewLicenseSet(
pkg.NewLicense("MIT AND GPL-3.0-only"),
pkg.NewLicense("MIT AND GPL-3.0-only WITH Classpath-exception-2.0"),
pkg.NewLicenseWithContext(ctx, "MIT AND GPL-3.0-only"),
pkg.NewLicenseWithContext(ctx, "MIT AND GPL-3.0-only WITH Classpath-exception-2.0"),
),
},
expected: &cyclonedx.Licenses{
@@ -88,9 +90,9 @@ func Test_encodeLicense(t *testing.T) {
name: "with multiple URLs and expressions",
input: pkg.Package{
Licenses: pkg.NewLicenseSet(
pkg.NewLicenseFromURLs("MIT", "https://opensource.org/licenses/MIT", "https://spdx.org/licenses/MIT.html"),
pkg.NewLicense("MIT AND GPL-3.0-only"),
pkg.NewLicenseFromURLs("FakeLicense", "htts://someurl.com"),
pkg.NewLicenseFromURLsWithContext(ctx, "MIT", "https://opensource.org/licenses/MIT", "https://spdx.org/licenses/MIT.html"),
pkg.NewLicenseWithContext(ctx, "MIT AND GPL-3.0-only"),
pkg.NewLicenseFromURLsWithContext(ctx, "FakeLicense", "htts://someurl.com"),
),
},
expected: &cyclonedx.Licenses{
@@ -123,8 +125,8 @@ func Test_encodeLicense(t *testing.T) {
name: "with multiple values licenses are deduplicated",
input: pkg.Package{
Licenses: pkg.NewLicenseSet(
pkg.NewLicense("Apache-2"),
pkg.NewLicense("Apache-2.0"),
pkg.NewLicenseWithContext(ctx, "Apache-2"),
pkg.NewLicenseWithContext(ctx, "Apache-2.0"),
),
},
expected: &cyclonedx.Licenses{
@@ -139,9 +141,9 @@ func Test_encodeLicense(t *testing.T) {
name: "with multiple URLs and single with no URLs",
input: pkg.Package{
Licenses: pkg.NewLicenseSet(
pkg.NewLicense("MIT"),
pkg.NewLicenseFromURLs("MIT", "https://opensource.org/licenses/MIT", "https://spdx.org/licenses/MIT.html"),
pkg.NewLicense("MIT AND GPL-3.0-only"),
pkg.NewLicenseWithContext(ctx, "MIT"),
pkg.NewLicenseFromURLsWithContext(ctx, "MIT", "https://opensource.org/licenses/MIT", "https://spdx.org/licenses/MIT.html"),
pkg.NewLicenseWithContext(ctx, "MIT AND GPL-3.0-only"),
),
},
expected: &cyclonedx.Licenses{
@@ -167,7 +169,7 @@ func Test_encodeLicense(t *testing.T) {
{
name: "single parenthesized SPDX expression",
input: pkg.Package{
Licenses: pkg.NewLicenseSet(pkg.NewLicensesFromValues("(MIT OR Apache-2.0)")...),
Licenses: pkg.NewLicenseSet(pkg.NewLicensesFromValuesWithContext(ctx, "(MIT OR Apache-2.0)")...),
},
expected: &cyclonedx.Licenses{
{
@@ -179,7 +181,7 @@ func Test_encodeLicense(t *testing.T) {
name: "single license AND to parenthesized SPDX expression",
// (LGPL-3.0-or-later OR GPL-2.0-or-later OR (LGPL-3.0-or-later AND GPL-2.0-or-later)) AND GFDL-1.3-invariants-or-later
input: pkg.Package{
Licenses: pkg.NewLicenseSet(pkg.NewLicensesFromValues("(LGPL-3.0-or-later OR GPL-2.0-or-later OR (LGPL-3.0-or-later AND GPL-2.0-or-later)) AND GFDL-1.3-invariants-or-later")...),
Licenses: pkg.NewLicenseSet(pkg.NewLicensesFromValuesWithContext(ctx, "(LGPL-3.0-or-later OR GPL-2.0-or-later OR (LGPL-3.0-or-later AND GPL-2.0-or-later)) AND GFDL-1.3-invariants-or-later")...),
},
expected: &cyclonedx.Licenses{
{
@@ -248,7 +250,6 @@ func TestDecodeLicenses(t *testing.T) {
Value: "RandomLicense",
// CycloneDX specification doesn't give a field for determining the license type
Type: license.Declared,
URLs: []string{},
},
},
},
@@ -268,7 +269,6 @@ func TestDecodeLicenses(t *testing.T) {
Value: "MIT",
SPDXExpression: "MIT",
Type: license.Declared,
URLs: []string{},
},
},
},
@@ -1,16 +1,17 @@
package helpers
import (
"crypto/sha256"
"fmt"
"sort"
"strings"
"github.com/spdx/tools-golang/spdx"
"github.com/anchore/syft/internal/spdxlicense"
"github.com/anchore/syft/syft/license"
"github.com/anchore/syft/syft/pkg"
)
func License(p pkg.Package) (concluded, declared string) {
func License(p pkg.Package) (concluded, declared string, otherLicenses []spdx.OtherLicense) {
// source: https://spdx.github.io/spdx-spec/v2.3/package-information/#713-concluded-license-field
// The options to populate this field are limited to:
// A valid SPDX License Expression as defined in Annex D;
@@ -21,15 +22,15 @@ func License(p pkg.Package) (concluded, declared string) {
// (iii) the SPDX file creator has intentionally provided no information (no meaning should be implied by doing so).
if p.Licenses.Empty() {
return NOASSERTION, NOASSERTION
return NOASSERTION, NOASSERTION, nil
}
// take all licenses and assume an AND expression;
// for information about license expressions see:
// https://spdx.github.io/spdx-spec/v2.3/SPDX-license-expressions/
pc, pd := ParseLicenses(p.Licenses.ToSlice())
pc, pd, ol := ParseLicenses(p.Licenses.ToSlice())
return joinLicenses(pc), joinLicenses(pd)
return joinLicenses(pc), joinLicenses(pd), ol
}
func joinLicenses(licenses []SPDXLicense) string {
@@ -58,14 +59,32 @@ func joinLicenses(licenses []SPDXLicense) string {
}
type SPDXLicense struct {
ID string
Value string
FullText string
// Valid SPDX ID OR License Value (should have LicenseRef- prefix and be sanitized)
// OR combination of the above as a valid SPDX License Expression as defined in Annex D.
// https://spdx.github.io/spdx-spec/v2.3/SPDX-license-expressions/
ID string
// If the SPDX license is not on the SPDX License List
LicenseName string
FullText string // 0..1 (Mandatory, one) if there is a License Identifier assigned (LicenseRef).
URLs []string
}
func ParseLicenses(raw []pkg.License) (concluded, declared []SPDXLicense) {
func ParseLicenses(raw []pkg.License) (concluded, declared []SPDXLicense, otherLicenses []spdx.OtherLicense) {
for _, l := range raw {
candidate := createSPDXLicense(l)
// isCustomLicense determines if the candidate falls under https://spdx.github.io/spdx-spec/v2.3/other-licensing-information-detected/#
// of the SPDX spec, where:
// - we should not have a complex SPDX expression
// - if a single license, it should not be a known license (on the SPDX license list)
if l.SPDXExpression == "" && strings.Contains(candidate.ID, spdxlicense.LicenseRefPrefix) {
otherLicenses = append(otherLicenses, spdx.OtherLicense{
LicenseIdentifier: candidate.ID,
ExtractedText: candidate.FullText,
LicenseName: candidate.LicenseName,
LicenseCrossReferences: candidate.URLs,
})
}
switch l.Type {
case license.Concluded:
concluded = append(concluded, candidate)
@@ -74,35 +93,70 @@ func ParseLicenses(raw []pkg.License) (concluded, declared []SPDXLicense) {
}
}
return concluded, declared
return concluded, declared, otherLicenses
}
func createSPDXLicense(l pkg.License) SPDXLicense {
candidate := SPDXLicense{
ID: generateLicenseID(l),
FullText: l.Contents,
// source: https://spdx.github.io/spdx-spec/v2.3/other-licensing-information-detected/#102-extracted-text-field
// we need to populate this field in the spdx document if we have a license ref
// 0..1 (Mandatory, one) if there is a License Identifier assigned (LicenseRef).
ft := NOASSERTION
if l.Contents != "" {
ft = l.Contents
}
if l.SPDXExpression == "" {
candidate.Value = l.Value
return SPDXLicense{
ID: generateLicenseID(l),
LicenseName: l.Value,
FullText: ft,
URLs: l.URLs,
}
return candidate
}
// generateLicenseID generates a license ID for the given license, which is either the license value or the SPDX expression.
func generateLicenseID(l pkg.License) string {
if l.SPDXExpression != "" {
return l.SPDXExpression
}
if l.Value != "" {
return spdxlicense.LicenseRefPrefix + SanitizeElementID(l.Value)
// syft format includes the algo for the sha in the values
// we can strip this and just make LicenseRef-<sum> for spdx consumption
id := strings.ReplaceAll(l.Value, "sha256:", "")
if !strings.HasPrefix(id, "LicenseRef-") {
id = "LicenseRef-" + id
}
return licenseSum(l.Contents)
return SanitizeElementID(id)
}
func licenseSum(s string) string {
if len(s) <= 64 {
return spdxlicense.LicenseRefPrefix + SanitizeElementID(s)
type SPDXOtherLicenseSet struct {
set map[string]spdx.OtherLicense
}
func NewSPDXOtherLicenseSet() *SPDXOtherLicenseSet {
return &SPDXOtherLicenseSet{
set: make(map[string]spdx.OtherLicense),
}
hash := sha256.Sum256([]byte(s))
return fmt.Sprintf("%s%x", spdxlicense.LicenseRefPrefix, hash)
}
func (s *SPDXOtherLicenseSet) Add(licenses ...spdx.OtherLicense) {
for _, l := range licenses {
s.set[l.LicenseIdentifier] = l
}
}
type ByLicenseIdentifier []spdx.OtherLicense
func (o ByLicenseIdentifier) Len() int { return len(o) }
func (o ByLicenseIdentifier) Swap(i, j int) { o[i], o[j] = o[j], o[i] }
func (o ByLicenseIdentifier) Less(i, j int) bool {
return o[i].LicenseIdentifier < o[j].LicenseIdentifier
}
func (s *SPDXOtherLicenseSet) ToSlice() []spdx.OtherLicense {
values := make([]spdx.OtherLicense, 0, len(s.set))
for _, v := range s.set {
values = append(values, v)
}
sort.Sort(ByLicenseIdentifier(values))
return values
}
@@ -1,9 +1,10 @@
package helpers
import (
"strings"
"context"
"testing"
"github.com/google/go-cmp/cmp"
"github.com/stretchr/testify/assert"
"github.com/anchore/syft/internal/spdxlicense"
@@ -11,6 +12,7 @@ import (
)
func Test_License(t *testing.T) {
ctx := context.TODO()
type expected struct {
concluded string
declared string
@@ -31,7 +33,7 @@ func Test_License(t *testing.T) {
{
name: "no SPDX licenses",
input: pkg.Package{
Licenses: pkg.NewLicenseSet(pkg.NewLicense("made-up")),
Licenses: pkg.NewLicenseSet(pkg.NewLicenseWithContext(ctx, "made-up")),
},
expected: expected{
concluded: "NOASSERTION",
@@ -41,7 +43,7 @@ func Test_License(t *testing.T) {
{
name: "with SPDX license",
input: pkg.Package{
Licenses: pkg.NewLicenseSet(pkg.NewLicense("MIT")),
Licenses: pkg.NewLicenseSet(pkg.NewLicenseWithContext(ctx, "MIT")),
},
expected: struct {
concluded string
@@ -55,8 +57,8 @@ func Test_License(t *testing.T) {
name: "with SPDX license expression",
input: pkg.Package{
Licenses: pkg.NewLicenseSet(
pkg.NewLicense("MIT"),
pkg.NewLicense("GPL-3.0-only"),
pkg.NewLicenseWithContext(ctx, "MIT"),
pkg.NewLicenseWithContext(ctx, "GPL-3.0-only"),
),
},
expected: expected{
@@ -69,9 +71,9 @@ func Test_License(t *testing.T) {
name: "includes valid LicenseRef-",
input: pkg.Package{
Licenses: pkg.NewLicenseSet(
pkg.NewLicense("one thing first"),
pkg.NewLicense("two things/#$^second"),
pkg.NewLicense("MIT"),
pkg.NewLicenseWithContext(ctx, "one thing first"),
pkg.NewLicenseWithContext(ctx, "two things/#$^second"),
pkg.NewLicenseWithContext(ctx, "MIT"),
),
},
expected: expected{
@@ -84,9 +86,9 @@ func Test_License(t *testing.T) {
name: "join parentheses correctly",
input: pkg.Package{
Licenses: pkg.NewLicenseSet(
pkg.NewLicense("one thing first"),
pkg.NewLicense("MIT AND GPL-3.0-only"),
pkg.NewLicense("MIT OR APACHE-2.0"),
pkg.NewLicenseWithContext(ctx, "one thing first"),
pkg.NewLicenseWithContext(ctx, "MIT AND GPL-3.0-only"),
pkg.NewLicenseWithContext(ctx, "MIT OR APACHE-2.0"),
),
},
expected: expected{
@@ -98,7 +100,7 @@ func Test_License(t *testing.T) {
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
c, d := License(test.input)
c, d, _ := License(test.input)
assert.Equal(t, test.expected.concluded, c)
assert.Equal(t, test.expected.declared, d)
})
@@ -123,11 +125,13 @@ func TestGenerateLicenseID(t *testing.T) {
{
name: "Uses value if no SPDX expression",
license: pkg.License{
Value: "MIT",
Value: "my-sweet-custom-license",
},
expected: spdxlicense.LicenseRefPrefix + "MIT",
expected: spdxlicense.LicenseRefPrefix + "my-sweet-custom-license",
},
{
// note: this is an oversight of the SPDX spec. It does NOT allow "+" in the ID even though they are
// significant to the licenses in the expressions below
name: "Long value is sanitized correctly",
license: pkg.License{
Value: "LGPLv2+ and LGPLv2+ with exceptions and GPLv2+ and GPLv2+ with exceptions and BSD and Inner-Net and ISC and Public Domain and GFDL",
@@ -135,13 +139,6 @@ func TestGenerateLicenseID(t *testing.T) {
expected: spdxlicense.LicenseRefPrefix +
"LGPLv2--and-LGPLv2--with-exceptions-and-GPLv2--and-GPLv2--with-exceptions-and-BSD-and-Inner-Net-and-ISC-and-Public-Domain-and-GFDL",
},
{
name: "Uses hash of contents when nothing else is provided",
license: pkg.License{
Contents: "This is a very long custom license text that should be hashed because it's more than 64 characters long.",
},
expected: "", // We'll verify it starts with the correct prefix
},
}
for _, tt := range tests {
@@ -160,34 +157,92 @@ func TestGenerateLicenseID(t *testing.T) {
func Test_joinLicenses(t *testing.T) {
tests := []struct {
name string
args []string
args []SPDXLicense
want string
}{
{
name: "multiple licenses",
args: []string{"MIT", "GPL-3.0-only"},
args: []SPDXLicense{{ID: "MIT"}, {ID: "GPL-3.0-only"}},
want: "MIT AND GPL-3.0-only",
},
{
name: "multiple licenses with complex expressions",
args: []string{"MIT AND Apache", "GPL-3.0-only"},
args: []SPDXLicense{{ID: "MIT AND Apache"}, {ID: "GPL-3.0-only"}},
want: "(MIT AND Apache) AND GPL-3.0-only",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
assert.Equalf(t, tt.want, joinLicenses(toSpdxLicenses(tt.args)), "joinLicenses(%v)", tt.args)
assert.Equalf(t, tt.want, joinLicenses(tt.args), "joinLicenses(%v)", tt.args)
})
}
}
func toSpdxLicenses(ids []string) (licenses []SPDXLicense) {
for _, l := range ids {
license := SPDXLicense{ID: l}
if strings.HasPrefix(l, spdxlicense.LicenseRefPrefix) {
license.Value = l
}
licenses = append(licenses, license)
func TestCreateSPDXLicenseAndGenerateLicenseID(t *testing.T) {
tests := []struct {
name string
input pkg.License
expected SPDXLicense
}{
{
name: "SPDX expression used as ID",
input: pkg.License{
SPDXExpression: "MIT",
Value: "MIT",
Contents: "",
},
expected: SPDXLicense{
ID: "MIT",
LicenseName: "MIT",
FullText: "NOASSERTION",
},
},
{
name: "LicenseRef with contents",
input: pkg.License{
Value: "sha256:123abc",
Contents: "license contents here",
},
expected: SPDXLicense{
ID: "LicenseRef-123abc",
LicenseName: "sha256:123abc",
FullText: "license contents here",
},
},
{
name: "LicenseRef without contents",
input: pkg.License{
Value: "custom-license",
Contents: "",
},
expected: SPDXLicense{
ID: "LicenseRef-custom-license",
LicenseName: "custom-license",
FullText: "NOASSERTION",
},
},
{
name: "URL is passed through",
input: pkg.License{
SPDXExpression: "MIT",
URLs: []string{
"https://example.com/license",
},
},
expected: SPDXLicense{
ID: "MIT",
FullText: "NOASSERTION",
URLs: []string{"https://example.com/license"},
},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
license := createSPDXLicense(tt.input)
if d := cmp.Diff(tt.expected, license); d != "" {
t.Errorf("createSPDXLicense() mismatch (-want +got):\n%s", d)
}
})
}
return licenses
}
@@ -1,6 +1,7 @@
package testutil
import (
"context"
"os"
"path/filepath"
"testing"
@@ -98,7 +99,7 @@ func DirectoryInputWithAuthorField(t testing.TB) sbom.SBOM {
func newDirectoryCatalog() *pkg.Collection {
catalog := pkg.NewCollection()
ctx := context.TODO()
// populate catalog with test data
catalog.Add(pkg.Package{
Name: "package-1",
@@ -110,7 +111,7 @@ func newDirectoryCatalog() *pkg.Collection {
),
Language: pkg.Python,
Licenses: pkg.NewLicenseSet(
pkg.NewLicense("MIT"),
pkg.NewLicenseWithContext(ctx, "MIT"),
),
Metadata: pkg.PythonPackage{
Name: "package-1",
@@ -149,7 +150,7 @@ func newDirectoryCatalog() *pkg.Collection {
func newDirectoryCatalogWithAuthorField() *pkg.Collection {
catalog := pkg.NewCollection()
ctx := context.TODO()
// populate catalog with test data
catalog.Add(pkg.Package{
Name: "package-1",
@@ -161,7 +162,7 @@ func newDirectoryCatalogWithAuthorField() *pkg.Collection {
),
Language: pkg.Python,
Licenses: pkg.NewLicenseSet(
pkg.NewLicense("MIT"),
pkg.NewLicenseWithContext(ctx, "MIT"),
),
Metadata: pkg.PythonPackage{
Name: "package-1",
+3 -2
View File
@@ -1,6 +1,7 @@
package testutil
import (
"context"
"os"
"path/filepath"
"testing"
@@ -98,7 +99,7 @@ func populateImageCatalog(catalog *pkg.Collection, img *image.Image) {
// TODO: this helper function is coupled to the image-simple fixture, which seems like a bad idea
_, ref1, _ := img.SquashedTree().File("/somefile-1.txt", filetree.FollowBasenameLinks)
_, ref2, _ := img.SquashedTree().File("/somefile-2.txt", filetree.FollowBasenameLinks)
ctx := context.TODO()
// populate catalog with test data
if ref1 != nil {
catalog.Add(pkg.Package{
@@ -111,7 +112,7 @@ func populateImageCatalog(catalog *pkg.Collection, img *image.Image) {
FoundBy: "the-cataloger-1",
Language: pkg.Python,
Licenses: pkg.NewLicenseSet(
pkg.NewLicense("MIT"),
pkg.NewLicenseWithContext(ctx, "MIT"),
),
Metadata: pkg.PythonPackage{
Name: "package-1",
@@ -15,7 +15,7 @@
"packages": [
{
"name": "package-1",
"SPDXID": "SPDXRef-Package-python-package-1-4dd25c6ee16b729a",
"SPDXID": "SPDXRef-Package-python-package-1-f04d218ff5ff50db",
"versionInfo": "1.0.1",
"supplier": "NOASSERTION",
"downloadLocation": "NOASSERTION",
@@ -76,7 +76,7 @@
"relationships": [
{
"spdxElementId": "SPDXRef-DocumentRoot-Directory-some-path",
"relatedSpdxElement": "SPDXRef-Package-python-package-1-4dd25c6ee16b729a",
"relatedSpdxElement": "SPDXRef-Package-python-package-1-f04d218ff5ff50db",
"relationshipType": "CONTAINS"
},
{
@@ -15,7 +15,7 @@
"packages": [
{
"name": "package-1",
"SPDXID": "SPDXRef-Package-python-package-1-72567175418f73f8",
"SPDXID": "SPDXRef-Package-python-package-1-2f52f617f1548337",
"versionInfo": "1.0.1",
"supplier": "NOASSERTION",
"downloadLocation": "NOASSERTION",
@@ -90,7 +90,7 @@
"relationships": [
{
"spdxElementId": "SPDXRef-DocumentRoot-Image-user-image-input",
"relatedSpdxElement": "SPDXRef-Package-python-package-1-72567175418f73f8",
"relatedSpdxElement": "SPDXRef-Package-python-package-1-2f52f617f1548337",
"relationshipType": "CONTAINS"
},
{
@@ -15,7 +15,7 @@
"packages": [
{
"name": "package-1",
"SPDXID": "SPDXRef-Package-python-package-1-72567175418f73f8",
"SPDXID": "SPDXRef-Package-python-package-1-2f52f617f1548337",
"versionInfo": "1.0.1",
"supplier": "NOASSERTION",
"downloadLocation": "NOASSERTION",
@@ -199,38 +199,38 @@
],
"relationships": [
{
"spdxElementId": "SPDXRef-Package-python-package-1-72567175418f73f8",
"spdxElementId": "SPDXRef-Package-python-package-1-2f52f617f1548337",
"relatedSpdxElement": "SPDXRef-File-f1-5265a4dde3edbf7c",
"relationshipType": "CONTAINS"
},
{
"spdxElementId": "SPDXRef-Package-python-package-1-72567175418f73f8",
"spdxElementId": "SPDXRef-Package-python-package-1-2f52f617f1548337",
"relatedSpdxElement": "SPDXRef-File-z1-f5-839d99ee67d9d174",
"relationshipType": "CONTAINS"
},
{
"spdxElementId": "SPDXRef-Package-python-package-1-72567175418f73f8",
"spdxElementId": "SPDXRef-Package-python-package-1-2f52f617f1548337",
"relatedSpdxElement": "SPDXRef-File-a1-f6-9c2f7510199b17f6",
"relationshipType": "CONTAINS"
},
{
"spdxElementId": "SPDXRef-Package-python-package-1-72567175418f73f8",
"spdxElementId": "SPDXRef-Package-python-package-1-2f52f617f1548337",
"relatedSpdxElement": "SPDXRef-File-d2-f4-c641caa71518099f",
"relationshipType": "CONTAINS"
},
{
"spdxElementId": "SPDXRef-Package-python-package-1-72567175418f73f8",
"spdxElementId": "SPDXRef-Package-python-package-1-2f52f617f1548337",
"relatedSpdxElement": "SPDXRef-File-d1-f3-c6f5b29dca12661f",
"relationshipType": "CONTAINS"
},
{
"spdxElementId": "SPDXRef-Package-python-package-1-72567175418f73f8",
"spdxElementId": "SPDXRef-Package-python-package-1-2f52f617f1548337",
"relatedSpdxElement": "SPDXRef-File-f2-f9e49132a4b96ccd",
"relationshipType": "CONTAINS"
},
{
"spdxElementId": "SPDXRef-DocumentRoot-Image-user-image-input",
"relatedSpdxElement": "SPDXRef-Package-python-package-1-72567175418f73f8",
"relatedSpdxElement": "SPDXRef-Package-python-package-1-2f52f617f1548337",
"relationshipType": "CONTAINS"
},
{
@@ -91,7 +91,7 @@ ExternalRef: PACKAGE-MANAGER purl pkg:deb/debian/package-2@2.0.1
##### Package: package-1
PackageName: package-1
SPDXID: SPDXRef-Package-python-package-1-72567175418f73f8
SPDXID: SPDXRef-Package-python-package-1-2f52f617f1548337
PackageVersion: 1.0.1
PackageSupplier: NOASSERTION
PackageDownloadLocation: NOASSERTION
@@ -105,13 +105,13 @@ ExternalRef: PACKAGE-MANAGER purl a-purl-1
##### Relationships
Relationship: SPDXRef-Package-python-package-1-72567175418f73f8 CONTAINS SPDXRef-File-f1-5265a4dde3edbf7c
Relationship: SPDXRef-Package-python-package-1-72567175418f73f8 CONTAINS SPDXRef-File-z1-f5-839d99ee67d9d174
Relationship: SPDXRef-Package-python-package-1-72567175418f73f8 CONTAINS SPDXRef-File-a1-f6-9c2f7510199b17f6
Relationship: SPDXRef-Package-python-package-1-72567175418f73f8 CONTAINS SPDXRef-File-d2-f4-c641caa71518099f
Relationship: SPDXRef-Package-python-package-1-72567175418f73f8 CONTAINS SPDXRef-File-d1-f3-c6f5b29dca12661f
Relationship: SPDXRef-Package-python-package-1-72567175418f73f8 CONTAINS SPDXRef-File-f2-f9e49132a4b96ccd
Relationship: SPDXRef-DocumentRoot-Image-user-image-input CONTAINS SPDXRef-Package-python-package-1-72567175418f73f8
Relationship: SPDXRef-Package-python-package-1-2f52f617f1548337 CONTAINS SPDXRef-File-f1-5265a4dde3edbf7c
Relationship: SPDXRef-Package-python-package-1-2f52f617f1548337 CONTAINS SPDXRef-File-z1-f5-839d99ee67d9d174
Relationship: SPDXRef-Package-python-package-1-2f52f617f1548337 CONTAINS SPDXRef-File-a1-f6-9c2f7510199b17f6
Relationship: SPDXRef-Package-python-package-1-2f52f617f1548337 CONTAINS SPDXRef-File-d2-f4-c641caa71518099f
Relationship: SPDXRef-Package-python-package-1-2f52f617f1548337 CONTAINS SPDXRef-File-d1-f3-c6f5b29dca12661f
Relationship: SPDXRef-Package-python-package-1-2f52f617f1548337 CONTAINS SPDXRef-File-f2-f9e49132a4b96ccd
Relationship: SPDXRef-DocumentRoot-Image-user-image-input CONTAINS SPDXRef-Package-python-package-1-2f52f617f1548337
Relationship: SPDXRef-DocumentRoot-Image-user-image-input CONTAINS SPDXRef-Package-deb-package-2-4b756c6f6fb127a3
Relationship: SPDXRef-DOCUMENT DESCRIBES SPDXRef-DocumentRoot-Image-user-image-input
@@ -38,7 +38,7 @@ ExternalRef: PACKAGE-MANAGER purl pkg:deb/debian/package-2@2.0.1
##### Package: package-1
PackageName: package-1
SPDXID: SPDXRef-Package-python-package-1-4dd25c6ee16b729a
SPDXID: SPDXRef-Package-python-package-1-f04d218ff5ff50db
PackageVersion: 1.0.1
PackageSupplier: NOASSERTION
PackageDownloadLocation: NOASSERTION
@@ -52,7 +52,7 @@ ExternalRef: PACKAGE-MANAGER purl a-purl-2
##### Relationships
Relationship: SPDXRef-DocumentRoot-Directory-some-path CONTAINS SPDXRef-Package-python-package-1-4dd25c6ee16b729a
Relationship: SPDXRef-DocumentRoot-Directory-some-path CONTAINS SPDXRef-Package-python-package-1-f04d218ff5ff50db
Relationship: SPDXRef-DocumentRoot-Directory-some-path CONTAINS SPDXRef-Package-deb-package-2-39392bb5e270f669
Relationship: SPDXRef-DOCUMENT DESCRIBES SPDXRef-DocumentRoot-Directory-some-path
@@ -41,7 +41,7 @@ ExternalRef: PACKAGE-MANAGER purl pkg:deb/debian/package-2@2.0.1
##### Package: package-1
PackageName: package-1
SPDXID: SPDXRef-Package-python-package-1-72567175418f73f8
SPDXID: SPDXRef-Package-python-package-1-2f52f617f1548337
PackageVersion: 1.0.1
PackageSupplier: NOASSERTION
PackageDownloadLocation: NOASSERTION
@@ -55,7 +55,7 @@ ExternalRef: PACKAGE-MANAGER purl a-purl-1
##### Relationships
Relationship: SPDXRef-DocumentRoot-Image-user-image-input CONTAINS SPDXRef-Package-python-package-1-72567175418f73f8
Relationship: SPDXRef-DocumentRoot-Image-user-image-input CONTAINS SPDXRef-Package-python-package-1-2f52f617f1548337
Relationship: SPDXRef-DocumentRoot-Image-user-image-input CONTAINS SPDXRef-Package-deb-package-2-4b756c6f6fb127a3
Relationship: SPDXRef-DOCUMENT DESCRIBES SPDXRef-DocumentRoot-Image-user-image-input
+3 -2
View File
@@ -2,6 +2,7 @@ package syftjson
import (
"bytes"
"context"
"flag"
"strings"
"testing"
@@ -132,7 +133,7 @@ func TestImageEncoder(t *testing.T) {
func TestEncodeFullJSONDocument(t *testing.T) {
catalog := pkg.NewCollection()
ctx := context.TODO()
p1 := pkg.Package{
Name: "package-1",
Version: "1.0.1",
@@ -144,7 +145,7 @@ func TestEncodeFullJSONDocument(t *testing.T) {
Type: pkg.PythonPkg,
FoundBy: "the-cataloger-1",
Language: pkg.Python,
Licenses: pkg.NewLicenseSet(pkg.NewLicense("MIT")),
Licenses: pkg.NewLicenseSet(pkg.NewLicenseWithContext(ctx, "MIT")),
Metadata: pkg.PythonPackage{
Name: "package-1",
Version: "1.0.1",
@@ -1,7 +1,7 @@
{
"artifacts": [
{
"id": "4dd25c6ee16b729a",
"id": "f04d218ff5ff50db",
"name": "package-1",
"version": "1.0.1",
"type": "python",
@@ -1,7 +1,7 @@
{
"artifacts": [
{
"id": "fba4ca04d4906f25",
"id": "951845d9a8d6b5b2",
"name": "package-1",
"version": "1.0.1",
"type": "python",
@@ -1,7 +1,7 @@
{
"artifacts": [
{
"id": "72567175418f73f8",
"id": "2f52f617f1548337",
"name": "package-1",
"version": "1.0.1",
"type": "python",