feat: update syft license concept to complex struct (#1743)

this PR makes the following changes to update the underlying license model to have more expressive capabilities
it also provides some guarantee's surrounding the license values themselves

- Licenses are updated from string -> pkg.LicenseSet which contain pkg.License with the following fields:
- original `Value` read by syft
- If it's possible to construct licenses will always have a valid SPDX expression for downstream consumption
- the above is run against a generated list of SPDX license ID to try and find the correct ID
- SPDX concluded vs declared is added to the new struct
- URL source for license is added to the new struct
- Location source is added to the new struct to show where the expression was pulled from

syft/formats/common/spdxhelpers/license.go

@@ -4,10 +4,11 @@ import (
 	"strings"
 
 	"github.com/anchore/syft/internal/spdxlicense"
+	"github.com/anchore/syft/syft/license"
 	"github.com/anchore/syft/syft/pkg"
 )
 
-func License(p pkg.Package) string {
+func License(p pkg.Package) (concluded, declared string) {
 	// source: https://spdx.github.io/spdx-spec/3-package-information/#313-concluded-license
 	// The options to populate this field are limited to:
 	// A valid SPDX License Expression as defined in Appendix IV;
@@ -17,35 +18,70 @@ func License(p pkg.Package) string {
 	//   (ii) the SPDX file creator has made no attempt to determine this field; or
 	//   (iii) the SPDX file creator has intentionally provided no information (no meaning should be implied by doing so).
 
-	if len(p.Licenses) == 0 {
-		return NONE
+	if p.Licenses.Empty() {
+		return NOASSERTION, NOASSERTION
 	}
 
-	// take all licenses and assume an AND expression; for information about license expressions see https://spdx.github.io/spdx-spec/appendix-IV-SPDX-license-expressions/
-	parsedLicenses := parseLicenses(p.Licenses)
+	// take all licenses and assume an AND expression;
+	// for information about license expressions see:
+	// https://spdx.github.io/spdx-spec/v2.3/SPDX-license-expressions/
+	pc, pd := parseLicenses(p.Licenses.ToSlice())
 
-	for i, v := range parsedLicenses {
+	for i, v := range pc {
 		if strings.HasPrefix(v, spdxlicense.LicenseRefPrefix) {
-			parsedLicenses[i] = SanitizeElementID(v)
+			pc[i] = SanitizeElementID(v)
 		}
 	}
 
-	if len(parsedLicenses) == 0 {
+	for i, v := range pd {
+		if strings.HasPrefix(v, spdxlicense.LicenseRefPrefix) {
+			pd[i] = SanitizeElementID(v)
+		}
+	}
+
+	return joinLicenses(pc), joinLicenses(pd)
+}
+
+func joinLicenses(licenses []string) string {
+	if len(licenses) == 0 {
 		return NOASSERTION
 	}
 
-	return strings.Join(parsedLicenses, " AND ")
+	var newLicenses []string
+
+	for _, v := range licenses {
+		// check if license does not start or end with parens
+		if !strings.HasPrefix(v, "(") && !strings.HasSuffix(v, ")") {
+			// if license contains AND, OR, or WITH, then wrap in parens
+			if strings.Contains(v, " AND ") ||
+				strings.Contains(v, " OR ") ||
+				strings.Contains(v, " WITH ") {
+				newLicenses = append(newLicenses, "("+v+")")
+				continue
+			}
+		}
+		newLicenses = append(newLicenses, v)
+	}
+
+	return strings.Join(newLicenses, " AND ")
 }
 
-func parseLicenses(raw []string) (parsedLicenses []string) {
+func parseLicenses(raw []pkg.License) (concluded, declared []string) {
 	for _, l := range raw {
-		if value, exists := spdxlicense.ID(l); exists {
-			parsedLicenses = append(parsedLicenses, value)
+		var candidate string
+		if l.SPDXExpression != "" {
+			candidate = l.SPDXExpression
 		} else {
 			// we did not find a valid SPDX license ID so treat as separate license
-			otherLicense := spdxlicense.LicenseRefPrefix + l
-			parsedLicenses = append(parsedLicenses, otherLicense)
+			candidate = spdxlicense.LicenseRefPrefix + l.Value
+		}
+
+		switch l.Type {
+		case license.Concluded:
+			concluded = append(concluded, candidate)
+		case license.Declared:
+			declared = append(declared, candidate)
 		}
 	}
-	return
+	return concluded, declared
 }

syft/formats/common/spdxhelpers/license_test.go

@@ -9,77 +9,120 @@ import (
 )
 
 func Test_License(t *testing.T) {
+	type expected struct {
+		concluded string
+		declared  string
+	}
 	tests := []struct {
 		name     string
 		input    pkg.Package
-		expected string
+		expected expected
 	}{
 		{
-			name:     "no licenses",
-			input:    pkg.Package{},
-			expected: NONE,
+			name:  "no licenses",
+			input: pkg.Package{},
+			expected: expected{
+				concluded: "NOASSERTION",
+				declared:  "NOASSERTION",
+			},
 		},
 		{
 			name: "no SPDX licenses",
 			input: pkg.Package{
-				Licenses: []string{
-					"made-up",
-				},
+				Licenses: pkg.NewLicenseSet(pkg.NewLicense("made-up")),
+			},
+			expected: expected{
+				concluded: "NOASSERTION",
+				declared:  "LicenseRef-made-up",
 			},
-			expected: "LicenseRef-made-up",
 		},
 		{
 			name: "with SPDX license",
 			input: pkg.Package{
-				Licenses: []string{
-					"MIT",
-				},
+				Licenses: pkg.NewLicenseSet(pkg.NewLicense("MIT")),
+			},
+			expected: struct {
+				concluded string
+				declared  string
+			}{
+				concluded: "NOASSERTION",
+				declared:  "MIT",
 			},
-			expected: "MIT",
 		},
 		{
 			name: "with SPDX license expression",
 			input: pkg.Package{
-				Licenses: []string{
-					"MIT",
-					"GPL-3.0",
-				},
+				Licenses: pkg.NewLicenseSet(
+					pkg.NewLicense("MIT"),
+					pkg.NewLicense("GPL-3.0-only"),
+				),
 			},
-			expected: "MIT AND GPL-3.0-only",
-		},
-		{
-			name: "cap insensitive",
-			input: pkg.Package{
-				Licenses: []string{
-					"gpl-3.0",
-				},
+			expected: expected{
+				concluded: "NOASSERTION",
+				// because we sort licenses alphabetically GPL ends up at the start
+				declared: "GPL-3.0-only AND MIT",
 			},
-			expected: "GPL-3.0-only",
-		},
-		{
-			name: "debian to spdx conversion",
-			input: pkg.Package{
-				Licenses: []string{
-					"GPL-2",
-				},
-			},
-			expected: "GPL-2.0-only",
 		},
 		{
 			name: "includes valid LicenseRef-",
 			input: pkg.Package{
-				Licenses: []string{
-					"one thing first",
-					"two things/#$^second",
-					"MIT",
-				},
+				Licenses: pkg.NewLicenseSet(
+					pkg.NewLicense("one thing first"),
+					pkg.NewLicense("two things/#$^second"),
+					pkg.NewLicense("MIT"),
+				),
+			},
+			expected: expected{
+				concluded: "NOASSERTION",
+				// because we separate licenses between valid SPDX and non valid, valid ID always end at the front
+				declared: "MIT AND LicenseRef-one-thing-first AND LicenseRef-two-things----second",
+			},
+		},
+		{
+			name: "join parentheses correctly",
+			input: pkg.Package{
+				Licenses: pkg.NewLicenseSet(
+					pkg.NewLicense("one thing first"),
+					pkg.NewLicense("MIT AND GPL-3.0-only"),
+					pkg.NewLicense("MIT OR APACHE-2.0"),
+				),
+			},
+			expected: expected{
+				concluded: "NOASSERTION",
+				// because we separate licenses between valid SPDX and non valid, valid ID always end at the front
+				declared: "(MIT AND GPL-3.0-only) AND (MIT OR APACHE-2.0) AND LicenseRef-one-thing-first",
 			},
-			expected: "LicenseRef-one-thing-first AND LicenseRef-two-things----second AND MIT",
 		},
 	}
 	for _, test := range tests {
 		t.Run(test.name, func(t *testing.T) {
-			assert.Equal(t, test.expected, License(test.input))
+			c, d := License(test.input)
+			assert.Equal(t, test.expected.concluded, c)
+			assert.Equal(t, test.expected.declared, d)
+		})
+	}
+}
+
+func Test_joinLicenses(t *testing.T) {
+	tests := []struct {
+		name string
+		args []string
+		want string
+	}{
+		{
+			name: "multiple licenses",
+			args: []string{"MIT", "GPL-3.0-only"},
+			want: "MIT AND GPL-3.0-only",
+		},
+		{
+			name: "multiple licenses with complex expressions",
+			args: []string{"MIT AND Apache", "GPL-3.0-only"},
+			want: "(MIT AND Apache) AND GPL-3.0-only",
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			assert.Equalf(t, tt.want, joinLicenses(tt.args), "joinLicenses(%v)", tt.args)
 		})
 	}
 }

syft/formats/common/spdxhelpers/to_format_model.go

@@ -170,7 +170,8 @@ func toPackages(catalog *pkg.Collection, sbom sbom.SBOM) (results []*spdx.Packag
 		// If the Concluded License is not the same as the Declared License, a written explanation should be provided
 		// in the Comments on License field (section 7.16). With respect to NOASSERTION, a written explanation in
 		// the Comments on License field (section 7.16) is preferred.
-		license := License(p)
+		// extract these correctly to the spdx license format
+		concluded, declared := License(p)
 
 		// two ways to get filesAnalyzed == true:
 		// 1. syft has generated a sha1 digest for the package itself - usually in the java cataloger
@@ -274,7 +275,7 @@ func toPackages(catalog *pkg.Collection, sbom sbom.SBOM) (results []*spdx.Packag
 			// Cardinality: mandatory, one
 			// Purpose: Contain the license the SPDX file creator has concluded as governing the
 			// package or alternative values, if the governing license cannot be determined.
-			PackageLicenseConcluded: license,
+			PackageLicenseConcluded: concluded,
 
 			// 7.14: All Licenses Info from Files: SPDX License Expression, "NONE" or "NOASSERTION"
 			// Cardinality: mandatory, one or many if filesAnalyzed is true / omitted;
@@ -286,7 +287,7 @@ func toPackages(catalog *pkg.Collection, sbom sbom.SBOM) (results []*spdx.Packag
 			// Purpose: List the licenses that have been declared by the authors of the package.
 			// Any license information that does not originate from the package authors, e.g. license
 			// information from a third party repository, should not be included in this field.
-			PackageLicenseDeclared: license,
+			PackageLicenseDeclared: declared,
 
 			// 7.16: Comments on License
 			// Cardinality: optional, one
@@ -534,10 +535,18 @@ func toFileTypes(metadata *source.FileMetadata) (ty []string) {
 	return ty
 }
 
+// other licenses are for licenses from the pkg.Package that do not have an SPDXExpression
+// field. The spdxexpression field is only filled given a validated Value field.
 func toOtherLicenses(catalog *pkg.Collection) []*spdx.OtherLicense {
 	licenses := map[string]bool{}
 	for _, p := range catalog.Sorted() {
-		for _, license := range parseLicenses(p.Licenses) {
+		declaredLicenses, concludedLicenses := parseLicenses(p.Licenses.ToSlice())
+		for _, license := range declaredLicenses {
+			if strings.HasPrefix(license, spdxlicense.LicenseRefPrefix) {
+				licenses[license] = true
+			}
+		}
+		for _, license := range concludedLicenses {
 			if strings.HasPrefix(license, spdxlicense.LicenseRefPrefix) {
 				licenses[license] = true
 			}
@@ -549,12 +558,12 @@ func toOtherLicenses(catalog *pkg.Collection) []*spdx.OtherLicense {
 	sorted := maps.Keys(licenses)
 	slices.Sort(sorted)
 	for _, license := range sorted {
-		// separate the actual ID from the prefix
+		// separate the found value from the prefix
+		// this only contains licenses that are not found on the SPDX License List
 		name := strings.TrimPrefix(license, spdxlicense.LicenseRefPrefix)
 		result = append(result, &spdx.OtherLicense{
 			LicenseIdentifier: SanitizeElementID(license),
-			LicenseName:       name,
-			ExtractedText:     NONE, // we probably should have some extracted text here, but this is good enough for now
+			ExtractedText:     name,
 		})
 	}
 	return result

syft/formats/common/spdxhelpers/to_format_model_test.go

@@ -448,46 +448,40 @@ func Test_OtherLicenses(t *testing.T) {
 		{
 			name: "no licenseRef",
 			pkg: pkg.Package{
-				Licenses: []string{
-					"MIT",
-				},
+				Licenses: pkg.NewLicenseSet(),
 			},
 			expected: nil,
 		},
 		{
 			name: "single licenseRef",
 			pkg: pkg.Package{
-				Licenses: []string{
-					"un known",
-				},
+				Licenses: pkg.NewLicenseSet(
+					pkg.NewLicense("foobar"),
+				),
 			},
 			expected: []*spdx.OtherLicense{
 				{
-					LicenseIdentifier: "LicenseRef-un-known",
-					LicenseName:       "un known",
-					ExtractedText:     NONE,
+					LicenseIdentifier: "LicenseRef-foobar",
+					ExtractedText:     "foobar",
 				},
 			},
 		},
 		{
 			name: "multiple licenseRef",
 			pkg: pkg.Package{
-				Licenses: []string{
-					"un known",
-					"not known %s",
-					"MIT",
-				},
+				Licenses: pkg.NewLicenseSet(
+					pkg.NewLicense("internal made up license name"),
+					pkg.NewLicense("new apple license 2.0"),
+				),
 			},
 			expected: []*spdx.OtherLicense{
 				{
-					LicenseIdentifier: "LicenseRef-not-known--s",
-					LicenseName:       "not known %s",
-					ExtractedText:     NONE,
+					LicenseIdentifier: "LicenseRef-internal-made-up-license-name",
+					ExtractedText:     "internal made up license name",
 				},
 				{
-					LicenseIdentifier: "LicenseRef-un-known",
-					LicenseName:       "un known",
-					ExtractedText:     NONE,
+					LicenseIdentifier: "LicenseRef-new-apple-license-2.0",
+					ExtractedText:     "new apple license 2.0",
 				},
 			},
 		},

syft/formats/common/spdxhelpers/to_syft_model.go

@@ -14,6 +14,7 @@ import (
 	"github.com/anchore/syft/syft/cpe"
 	"github.com/anchore/syft/syft/file"
 	"github.com/anchore/syft/syft/formats/common/util"
+	"github.com/anchore/syft/syft/license"
 	"github.com/anchore/syft/syft/linux"
 	"github.com/anchore/syft/syft/pkg"
 	"github.com/anchore/syft/syft/sbom"
@@ -279,7 +280,7 @@ func toSyftPackage(p *spdx.Package) *pkg.Package {
 		Type:         info.typ,
 		Name:         p.PackageName,
 		Version:      p.PackageVersion,
-		Licenses:     parseLicense(p.PackageLicenseDeclared),
+		Licenses:     pkg.NewLicenseSet(parseSPDXLicenses(p)...),
 		CPEs:         extractCPEs(p),
 		PURL:         info.purl.String(),
 		Language:     info.lang,
@@ -292,6 +293,33 @@ func toSyftPackage(p *spdx.Package) *pkg.Package {
 	return &sP
 }
 
+func parseSPDXLicenses(p *spdx.Package) []pkg.License {
+	licenses := make([]pkg.License, 0)
+
+	// concluded
+	if p.PackageLicenseConcluded != NOASSERTION && p.PackageLicenseConcluded != NONE && p.PackageLicenseConcluded != "" {
+		l := pkg.NewLicense(cleanSPDXID(p.PackageLicenseConcluded))
+		l.Type = license.Concluded
+		licenses = append(licenses, l)
+	}
+
+	// declared
+	if p.PackageLicenseDeclared != NOASSERTION && p.PackageLicenseDeclared != NONE && p.PackageLicenseDeclared != "" {
+		l := pkg.NewLicense(cleanSPDXID(p.PackageLicenseDeclared))
+		l.Type = license.Declared
+		licenses = append(licenses, l)
+	}
+
+	return licenses
+}
+
+func cleanSPDXID(id string) string {
+	if strings.HasPrefix(id, "LicenseRef-") {
+		return strings.TrimPrefix(id, "LicenseRef-")
+	}
+	return id
+}
+
 //nolint:funlen
 func extractMetadata(p *spdx.Package, info pkgInfo) (pkg.MetadataType, interface{}) {
 	arch := info.qualifierValue(pkg.PURLQualifierArch)
@@ -317,7 +345,6 @@ func extractMetadata(p *spdx.Package, info pkgInfo) (pkg.MetadataType, interface
 			OriginPackage: upstreamName,
 			Maintainer:    supplier,
 			Version:       p.PackageVersion,
-			License:       p.PackageLicenseDeclared,
 			Architecture:  arch,
 			URL:           p.PackageHomePage,
 			Description:   p.PackageDescription,
@@ -330,17 +357,12 @@ func extractMetadata(p *spdx.Package, info pkgInfo) (pkg.MetadataType, interface
 		} else {
 			epoch = &converted
 		}
-		license := p.PackageLicenseDeclared
-		if license == "" {
-			license = p.PackageLicenseConcluded
-		}
 		return pkg.RpmMetadataType, pkg.RpmMetadata{
 			Name:      p.PackageName,
 			Version:   p.PackageVersion,
 			Epoch:     epoch,
 			Arch:      arch,
 			SourceRpm: upstreamValue,
-			License:   license,
 			Vendor:    originator,
 		}
 	case pkg.DebPkg:
@@ -400,10 +422,3 @@ func extractCPEs(p *spdx.Package) (cpes []cpe.CPE) {
 	}
 	return cpes
 }
-
-func parseLicense(l string) []string {
-	if l == NOASSERTION || l == NONE {
-		return nil
-	}
-	return strings.Split(l, " AND ")
-}