feat: update licenses to including license content when SPDX expressions are unable to be determined (#3366)

---------
Signed-off-by: HeyeOpenSource <opensource@heye-international.com>
Signed-off-by: Christopher Phillips <32073428+spiffcs@users.noreply.github.com>
Co-authored-by: Christopher Phillips <32073428+spiffcs@users.noreply.github.com>

syft/format/common/spdxhelpers/to_format_model.go

@@ -15,6 +15,7 @@ import (
 	"github.com/spdx/tools-golang/spdx"
 
 	"github.com/anchore/packageurl-go"
+	internallicenses "github.com/anchore/syft/internal/licenses"
 	"github.com/anchore/syft/internal/log"
 	"github.com/anchore/syft/internal/mimetype"
 	"github.com/anchore/syft/internal/relationship"
@@ -762,10 +763,16 @@ func toOtherLicenses(catalog *pkg.Collection) []*spdx.OtherLicense {
 		if license.Value == "" {
 			value, _ = strings.CutPrefix(license.ID, "LicenseRef-")
 		}
-		result = append(result, &spdx.OtherLicense{
+		other := &spdx.OtherLicense{
 			LicenseIdentifier: license.ID,
 			ExtractedText:     value,
-		})
+		}
+		customPrefix := spdxlicense.LicenseRefPrefix + helpers.SanitizeElementID(internallicenses.UnknownLicensePrefix)
+		if strings.HasPrefix(license.ID, customPrefix) {
+			other.LicenseName = strings.TrimPrefix(license.ID, customPrefix)
+			other.LicenseComment = strings.Trim(internallicenses.UnknownLicensePrefix, "-_")
+		}
+		result = append(result, other)
 	}
 	return result
 }

syft/format/internal/cyclonedxutil/helpers/licenses.go

@@ -1,10 +1,12 @@
 package helpers
 
 import (
+	"encoding/base64"
 	"strings"
 
 	"github.com/CycloneDX/cyclonedx-go"
 
+	"github.com/anchore/syft/internal/licenses"
 	"github.com/anchore/syft/internal/spdxlicense"
 	"github.com/anchore/syft/syft/pkg"
 )
@@ -110,7 +112,7 @@ func separateLicenses(p pkg.Package) (spdx, other cyclonedx.Licenses, expression
 			continue
 		}
 
-		if l.SPDXExpression != "" {
+		if l.SPDXExpression != "" && !strings.HasPrefix(l.SPDXExpression, licenses.UnknownLicensePrefix) {
 			// COMPLEX EXPRESSION CASE
 			ex = append(ex, l.SPDXExpression)
 			continue
@@ -118,17 +120,43 @@ func separateLicenses(p pkg.Package) (spdx, other cyclonedx.Licenses, expression
 
 		// license string that are not valid spdx expressions or ids
 		// we only use license Name here since we cannot guarantee that the license is a valid SPDX expression
-		if len(l.URLs) > 0 {
+		if len(l.URLs) > 0 && !strings.HasPrefix(l.SPDXExpression, licenses.UnknownLicensePrefix) {
 			processLicenseURLs(l, "", &otherc)
 			continue
 		}
-		otherc = append(otherc, cyclonedx.LicenseChoice{
+
+		otherc = append(otherc, processCustomLicense(l)...)
+	}
+	return spdxc, otherc, ex
+}
+
+func processCustomLicense(l pkg.License) cyclonedx.Licenses {
+	result := cyclonedx.Licenses{}
+	if strings.HasPrefix(l.SPDXExpression, licenses.UnknownLicensePrefix) {
+		cyclonedxLicense := &cyclonedx.License{
+			Name: l.SPDXExpression,
+		}
+		if len(l.URLs) > 0 {
+			cyclonedxLicense.URL = l.URLs[0]
+		}
+		if len(l.Contents) > 0 {
+			cyclonedxLicense.Text = &cyclonedx.AttachedText{
+				Content: base64.StdEncoding.EncodeToString([]byte(l.Contents)),
+			}
+			cyclonedxLicense.Text.ContentType = "text/plain"
+			cyclonedxLicense.Text.Encoding = "base64"
+		}
+		result = append(result, cyclonedx.LicenseChoice{
+			License: cyclonedxLicense,
+		})
+	} else {
+		result = append(result, cyclonedx.LicenseChoice{
 			License: &cyclonedx.License{
 				Name: l.Value,
 			},
 		})
 	}
-	return spdxc, otherc, ex
+	return result
 }
 
 func processLicenseURLs(l pkg.License, spdxID string, populate *cyclonedx.Licenses) {

syft/format/internal/spdxutil/helpers/license.go

@@ -5,6 +5,7 @@ import (
 	"fmt"
 	"strings"
 
+	"github.com/anchore/syft/internal/licenses"
 	"github.com/anchore/syft/internal/spdxlicense"
 	"github.com/anchore/syft/syft/license"
 	"github.com/anchore/syft/syft/pkg"
@@ -69,19 +70,26 @@ func ParseLicenses(raw []pkg.License) (concluded, declared []SPDXLicense) {
 		}
 
 		candidate := SPDXLicense{}
-		if l.SPDXExpression != "" {
+		if l.SPDXExpression != "" && !strings.HasPrefix(l.SPDXExpression, licenses.UnknownLicensePrefix) {
 			candidate.ID = l.SPDXExpression
 		} else {
-			// we did not find a valid SPDX license ID so treat as separate license
-			if len(l.Value) <= 64 {
-				// if the license text is less than the size of the hash,
-				// just use it directly so the id is more readable
-				candidate.ID = spdxlicense.LicenseRefPrefix + SanitizeElementID(l.Value)
-			} else {
-				hash := sha256.Sum256([]byte(l.Value))
-				candidate.ID = fmt.Sprintf("%s%x", spdxlicense.LicenseRefPrefix, hash)
-			}
 			candidate.Value = l.Value
+			// we did not find a valid SPDX license ID so treat as separate license
+			if strings.HasPrefix(l.SPDXExpression, licenses.UnknownLicensePrefix) {
+				candidate.ID = spdxlicense.LicenseRefPrefix + SanitizeElementID(l.SPDXExpression)
+				if len(l.Contents) > 0 {
+					candidate.Value = l.Contents
+				}
+			} else {
+				if len(l.Value) <= 64 {
+					// if the license text is less than the size of the hash,
+					// just use it directly so the id is more readable
+					candidate.ID = spdxlicense.LicenseRefPrefix + SanitizeElementID(l.Value)
+				} else {
+					hash := sha256.Sum256([]byte(l.Value))
+					candidate.ID = fmt.Sprintf("%s%x", spdxlicense.LicenseRefPrefix, hash)
+				}
+			}
 		}
 
 		switch l.Type {

syft/format/syftjson/model/package.go

@@ -51,6 +51,7 @@ type License struct {
 	Type           license.Type    `json:"type"`
 	URLs           []string        `json:"urls"`
 	Locations      []file.Location `json:"locations"`
+	Contents       string          `json:"contents,omitempty"`
 }
 
 func newModelLicensesFromValues(licenses []string) (ml []License) {

syft/format/syftjson/to_format_model.go

@@ -234,6 +234,7 @@ func toLicenseModel(pkgLicenses []pkg.License) (modelLicenses []model.License) {
 			Type:           l.Type,
 			URLs:           urls,
 			Locations:      locations,
+			Contents:       l.Contents,
 		})
 	}
 	return

syft/format/syftjson/to_syft_model.go

@@ -164,6 +164,7 @@ func toSyftLicenses(m []model.License) (p []pkg.License) {
 			Type:           l.Type,
 			URLs:           l.URLs,
 			Locations:      file.NewLocationSet(l.Locations...),
+			Contents:       l.Contents,
 		})
 	}
 	return