detect license ID from full text when incidentally provided as a value (#3876)

---------
Signed-off-by: Christopher Phillips <32073428+spiffcs@users.noreply.github.com>
Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>
Co-authored-by: Alex Goodman <wagoodman@users.noreply.github.com>

internal/task/package_task_factory.go

@@ -107,6 +107,11 @@ func finalizePkgCatalogerResults(cfg CatalogingFactoryConfig, resolver file.Path
 			}
 		}
 
+		// we want to know if the user wants to preserve license content or not in the final SBOM
+		// note: this looks incorrect, but pkg.License.Content is NOT used to compute the Package ID
+		// this does NOT change the reproducibility of the Package ID
+		applyLicenseContentRules(&p, cfg.LicenseConfig)
+
 		pkgs[i] = p
 	}
 	return pkgs, relationships
@@ -262,3 +267,29 @@ func packageFileOwnershipRelationships(p pkg.Package, resolver file.PathResolver
 	}
 	return relationships, nil
 }
+
+func applyLicenseContentRules(p *pkg.Package, cfg cataloging.LicenseConfig) {
+	if p.Licenses.Empty() {
+		return
+	}
+
+	licenses := p.Licenses.ToSlice()
+	for i := range licenses {
+		l := &licenses[i]
+		switch cfg.IncludeContent {
+		case cataloging.LicenseContentIncludeUnknown:
+			// we don't have an SPDX expression, which means we didn't find an SPDX license
+			// include the unknown licenses content in the final SBOM
+			if l.SPDXExpression != "" {
+				licenses[i].Contents = ""
+			}
+		case cataloging.LicenseContentExcludeAll:
+			// clear it all out
+			licenses[i].Contents = ""
+		case cataloging.LicenseContentIncludeAll:
+			// always include the content
+		}
+	}
+
+	p.Licenses = pkg.NewLicenseSet(licenses...)
+}

internal/task/package_task_factory_test.go

@@ -117,6 +117,114 @@ func TestFilterNonCompliantPackages(t *testing.T) {
 	assert.Equal(t, p2, droppedPkgs[0])
 }
 
+func TestApplyLicenseContentRules(t *testing.T) {
+	licenseWithSPDX := pkg.License{
+		SPDXExpression: "MIT",
+		Contents:       "MIT license content",
+	}
+	licenseWithoutSPDX := pkg.License{
+		Value:    "License-Not-A-SPDX-Expression",
+		Contents: "Non-SPDX license content",
+	}
+
+	tests := []struct {
+		name             string
+		inputLicenses    []pkg.License
+		cfg              cataloging.LicenseConfig
+		expectedLicenses []pkg.License
+	}{
+		{
+			name: "LicenseContentIncludeUnknown",
+			inputLicenses: []pkg.License{
+				licenseWithSPDX,
+				licenseWithoutSPDX,
+			},
+			cfg: cataloging.LicenseConfig{
+				IncludeContent: cataloging.LicenseContentIncludeUnknown,
+			},
+			expectedLicenses: []pkg.License{
+				{
+					SPDXExpression: "MIT",
+					Contents:       "", // content cleared for SPDX license
+				},
+				{
+					Value:    "License-Not-A-SPDX-Expression",
+					Contents: "Non-SPDX license content", // content preserved for non-SPDX
+				},
+			},
+		},
+		{
+			name: "LicenseContentExcludeAll",
+			inputLicenses: []pkg.License{
+				licenseWithSPDX,
+				licenseWithoutSPDX,
+			},
+			cfg: cataloging.LicenseConfig{
+				IncludeContent: cataloging.LicenseContentExcludeAll,
+			},
+			expectedLicenses: []pkg.License{
+				{
+					SPDXExpression: "MIT",
+					Contents:       "", // content cleared
+				},
+				{
+					Value:    "License-Not-A-SPDX-Expression",
+					Contents: "", // content cleared
+				},
+			},
+		},
+		{
+			name: "IncludeLicenseContentDefault",
+			inputLicenses: []pkg.License{
+				licenseWithSPDX,
+				licenseWithoutSPDX,
+			},
+			cfg: cataloging.LicenseConfig{
+				IncludeContent: cataloging.LicenseContentIncludeAll,
+			},
+			expectedLicenses: []pkg.License{
+				{
+					SPDXExpression: "MIT",
+					Contents:       "MIT license content", // content preserved
+				},
+				{
+					Value:    "License-Not-A-SPDX-Expression",
+					Contents: "Non-SPDX license content", // content preserved
+				},
+			},
+		},
+		{
+			name:          "Empty licenses",
+			inputLicenses: []pkg.License{},
+			cfg: cataloging.LicenseConfig{
+				IncludeContent: cataloging.LicenseContentIncludeAll,
+			},
+			expectedLicenses: []pkg.License{},
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			inputPkg := &pkg.Package{
+				Licenses: pkg.NewLicenseSet(tt.inputLicenses...),
+			}
+
+			inputPkg.SetID()
+			originalID := inputPkg.ID()
+
+			applyLicenseContentRules(inputPkg, tt.cfg)
+
+			assert.Equal(t, originalID, inputPkg.ID(), "package ID changed unexpectedly")
+
+			actualLicenses := inputPkg.Licenses.ToSlice()
+			expectedLicenses := pkg.NewLicenseSet(tt.expectedLicenses...).ToSlice()
+
+			assert.Equal(t, expectedLicenses, actualLicenses, "license contents do not match expected values")
+
+		})
+	}
+}
+
 func TestApplyComplianceRules_DropAndStub(t *testing.T) {
 	p := pkg.Package{Name: "", Version: ""}
 	p.SetID()