fix: match opensource.org license URLs across SPDX URL forms (#5361)

SPDX license list 3.29.0 rewrote every opensource.org seeAlso URL from
opensource.org/licenses/<ID> to opensource.org/license/<ID> (and e.g.
opensource.org/license/mit/ to opensource.org/license/MIT). Package metadata
in the wild (Maven POM <license><url>, npm, etc.) still uses the older forms,
so regenerating the list would stop LicenseByURL from resolving them.

Fall back to a normalized index of the opensource.org entries that ignores the
www. prefix, /licenses/ vs /license/, case, and a trailing slash. The index is
built from the generated urlToLicense map, so it works with both the current
and the updated license list without changes to the generator. URL forms that
would normalize to different licenses are left out rather than guessed.

Signed-off-by: Will Murphy <willmurphyscode@users.noreply.github.com>

internal/spdxlicense/license.go

@@ -44,6 +44,7 @@ type LicenseInfo struct {
 // LicenseByURL returns the license ID and name for a given URL from the SPDX license list.
 // The URL should match one of the URLs in the seeAlso field of an SPDX license.
 // The scheme (http:// or https://) is stripped before lookup, so both schemes match.
+// opensource.org URLs additionally match regardless of the form used (see normalizeOpenSourceOrgURL).
 func LicenseByURL(url string) (LicenseInfo, bool) {
 	url = strings.TrimSpace(url)
 	url = stripScheme(url)
@@ -52,9 +53,51 @@ func LicenseByURL(url string) (LicenseInfo, bool) {
 			ID: id,
 		}, true
 	}
+	if normalized, ok := normalizeOpenSourceOrgURL(url); ok {
+		if id, exists := openSourceOrgURLToLicense[normalized]; exists {
+			return LicenseInfo{
+				ID: id,
+			}, true
+		}
+	}
 	return LicenseInfo{}, false
 }
 
+// openSourceOrgURLToLicense indexes the opensource.org entries of urlToLicense by normalized URL.
+var openSourceOrgURLToLicense = buildOpenSourceOrgURLToLicense(urlToLicense)
+
+func buildOpenSourceOrgURLToLicense(urls map[string]string) map[string]string {
+	index := make(map[string]string)
+	ambiguous := make(map[string]bool)
+	for url, id := range urls {
+		normalized, ok := normalizeOpenSourceOrgURL(url)
+		if !ok || ambiguous[normalized] {
+			continue
+		}
+		if existing, exists := index[normalized]; exists && existing != id {
+			// don't guess between licenses that only differ by URL form
+			delete(index, normalized)
+			ambiguous[normalized] = true
+			continue
+		}
+		index[normalized] = id
+	}
+	return index
+}
+
+// normalizeOpenSourceOrgURL reduces a scheme-less opensource.org URL to a canonical form.
+// SPDX has changed these URLs over time (e.g. 3.29.0 replaced opensource.org/licenses/MIT and
+// opensource.org/license/mit/ with opensource.org/license/MIT), while package metadata in the wild
+// still uses the older forms. The www. prefix, /licenses/ vs /license/, case, and trailing slash are ignored.
+func normalizeOpenSourceOrgURL(url string) (string, bool) {
+	url = strings.TrimPrefix(strings.ToLower(url), "www.")
+	if !strings.HasPrefix(url, "opensource.org/") {
+		return "", false
+	}
+	url = strings.Replace(url, "opensource.org/licenses/", "opensource.org/license/", 1)
+	return strings.TrimSuffix(url, "/"), true
+}
+
 // stripScheme removes http:// or https:// prefix from a URL.
 // This allows a single map entry to match both schemes.
 func stripScheme(url string) string {

internal/spdxlicense/license_url_test.go

@@ -123,6 +123,96 @@ func TestLicenseByURL_AlternateScheme(t *testing.T) {
 	}
 }
 
+func TestLicenseByURL_OpenSourceOrgVariants(t *testing.T) {
+	// SPDX has changed the form of opensource.org URLs across list versions (e.g. 3.29.0 moved
+	// opensource.org/licenses/MIT to opensource.org/license/MIT); every form should keep resolving.
+	tests := []struct {
+		name   string
+		url    string
+		wantID string
+	}{
+		{name: "old plural path", url: "http://opensource.org/licenses/MIT", wantID: "MIT"},
+		{name: "new singular path", url: "https://opensource.org/license/MIT", wantID: "MIT"},
+		{name: "lowercase with trailing slash", url: "https://opensource.org/license/mit/", wantID: "MIT"},
+		{name: "plural path with trailing slash", url: "https://opensource.org/licenses/MIT/", wantID: "MIT"},
+		{name: "www prefix", url: "https://www.opensource.org/licenses/MIT", wantID: "MIT"},
+		{name: "uppercase host", url: "https://WWW.OpenSource.org/licenses/MIT", wantID: "MIT"},
+		{name: "old plural path BSD-3-Clause", url: "https://opensource.org/licenses/BSD-3-Clause", wantID: "BSD-3-Clause"},
+		{name: "new singular path BSD-3-Clause", url: "https://opensource.org/license/BSD-3-Clause", wantID: "BSD-3-Clause"},
+		{name: "lowercase BSD-3-Clause", url: "https://opensource.org/licenses/bsd-3-clause", wantID: "BSD-3-Clause"},
+		{name: "old plural path Apache-2.0", url: "http://opensource.org/licenses/Apache-2.0", wantID: "Apache-2.0"},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			info, found := LicenseByURL(tt.url)
+			if !found {
+				t.Fatalf("LicenseByURL(%q) found = false, want true", tt.url)
+			}
+			if info.ID != tt.wantID {
+				t.Errorf("LicenseByURL(%q) ID = %v, want %v", tt.url, info.ID, tt.wantID)
+			}
+		})
+	}
+}
+
+func TestLicenseByURL_NonOpenSourceOrgStaysExact(t *testing.T) {
+	// the relaxed matching is scoped to opensource.org; other hosts may have case-sensitive paths
+	if _, found := LicenseByURL("https://www.apache.org/licenses/license-2.0"); found {
+		t.Errorf("LicenseByURL() found = true for case-altered non-opensource.org URL, want false")
+	}
+	if _, found := LicenseByURL("https://opensource.org/licenses/not-a-real-license"); found {
+		t.Errorf("LicenseByURL() found = true for unknown opensource.org URL, want false")
+	}
+}
+
+func TestNormalizeOpenSourceOrgURL(t *testing.T) {
+	tests := []struct {
+		url    string
+		want   string
+		wantOK bool
+	}{
+		{url: "opensource.org/licenses/MIT", want: "opensource.org/license/mit", wantOK: true},
+		{url: "opensource.org/license/mit/", want: "opensource.org/license/mit", wantOK: true},
+		{url: "www.opensource.org/licenses/EPL-2.0", want: "opensource.org/license/epl-2.0", wantOK: true},
+		{url: "lists.opensource.org/pipermail/license-discuss", wantOK: false},
+		{url: "www.apache.org/licenses/LICENSE-2.0", wantOK: false},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.url, func(t *testing.T) {
+			got, ok := normalizeOpenSourceOrgURL(tt.url)
+			if ok != tt.wantOK {
+				t.Fatalf("normalizeOpenSourceOrgURL(%q) ok = %v, want %v", tt.url, ok, tt.wantOK)
+			}
+			if got != tt.want {
+				t.Errorf("normalizeOpenSourceOrgURL(%q) = %q, want %q", tt.url, got, tt.want)
+			}
+		})
+	}
+}
+
+func TestBuildOpenSourceOrgURLToLicense(t *testing.T) {
+	index := buildOpenSourceOrgURLToLicense(map[string]string{
+		"opensource.org/licenses/MIT": "MIT",
+		"opensource.org/license/mit/": "MIT",
+		"opensource.org/licenses/Foo": "Foo-1.0",
+		"opensource.org/license/foo":  "Foo-2.0",
+		"opensource.org/license/FOO/": "Foo-1.0",
+		"www.apache.org/licenses/Bar": "Bar",
+	})
+
+	if got := index["opensource.org/license/mit"]; got != "MIT" {
+		t.Errorf("index[mit] = %q, want MIT", got)
+	}
+	if got, exists := index["opensource.org/license/foo"]; exists {
+		t.Errorf("index[foo] = %q, want no entry for URL forms mapping to different licenses", got)
+	}
+	if len(index) != 1 {
+		t.Errorf("len(index) = %d, want 1 (non-opensource.org URLs excluded): %v", len(index), index)
+	}
+}
+
 func TestStripScheme(t *testing.T) {
 	tests := []struct {
 		name string