(#460) Extend license mapping for common SPDX license names (#509)

Fixes #460 
Signed-off-by: Christopher Angelo Phillips <christopher.phillips@anchore.com>

internal/presenter/packages/spdx_helpers_test.go

@@ -95,6 +95,15 @@ func Test_getSPDXLicense(t *testing.T) {
 			},
 			expected: "GPL-3.0",
 		},
+		{
+			name: "debian to spdx conversion",
+			input: pkg.Package{
+				Licenses: []string{
+					"GPL-2",
+				},
+			},
+			expected: "GPL-2.0",
+		},
 	}
 	for _, test := range tests {
 		t.Run(test.name, func(t *testing.T) {

internal/spdxlicense/generate/generate_license_list.go

@@ -0,0 +1,181 @@
+package main
+
+import (
+	"encoding/json"
+	"log"
+	"net/http"
+	"os"
+	"regexp"
+	"sort"
+	"strings"
+	"text/template"
+	"time"
+
+	"github.com/scylladb/go-set/strset"
+)
+
+// This program generates license_list.go.
+const (
+	source = "license_list.go"
+	url    = "https://spdx.org/licenses/licenses.json"
+)
+
+var tmp = template.Must(template.New("").Parse(`// Code generated by go generate; DO NOT EDIT.
+// This file was generated by robots at {{ .Timestamp }}
+// using data from {{ .URL }}
+package spdxlicense
+
+const Version = {{ printf "%q" .Version }}
+
+var licenseIDs = map[string]string{
+{{- range $k, $v := .LicenseIDs }}
+	{{ printf "%q" $k }}: {{ printf "%q" $v }},
+{{- end }}
+}
+`))
+
+var versionMatch = regexp.MustCompile(`-([0-9]+)\.?([0-9]+)?\.?([0-9]+)?\.?`)
+
+type LicenseList struct {
+	Version  string `json:"licenseListVersion"`
+	Licenses []struct {
+		ID          string   `json:"licenseId"`
+		Name        string   `json:"name"`
+		Text        string   `json:"licenseText"`
+		Deprecated  bool     `json:"isDeprecatedLicenseId"`
+		OSIApproved bool     `json:"isOsiApproved"`
+		SeeAlso     []string `json:"seeAlso"`
+	} `json:"licenses"`
+}
+
+func main() {
+	resp, err := http.Get(url)
+	if err != nil {
+		log.Fatalf("unable to get licenses list: %+v", err)
+	}
+
+	var result LicenseList
+	if err = json.NewDecoder(resp.Body).Decode(&result); err != nil {
+		log.Fatalf("unable to decode license list: %+v", err)
+	}
+	defer func() {
+		if err := resp.Body.Close(); err != nil {
+			log.Fatalf("unable to close body: %+v", err)
+		}
+	}()
+
+	f, err := os.Create(source)
+	if err != nil {
+		log.Fatalf("unable to create %q: %+v", source, err)
+	}
+	defer func() {
+		if err := f.Close(); err != nil {
+			log.Fatalf("unable to close %q: %+v", source, err)
+		}
+	}()
+
+	licenseIDs := processSPDXLicense(result)
+
+	err = tmp.Execute(f, struct {
+		Timestamp  time.Time
+		URL        string
+		Version    string
+		LicenseIDs map[string]string
+	}{
+		Timestamp:  time.Now(),
+		URL:        url,
+		Version:    result.Version,
+		LicenseIDs: licenseIDs,
+	})
+
+	if err != nil {
+		log.Fatalf("unable to generate template: %+v", err)
+	}
+}
+
+// Parsing the provided SPDX license list necessitates a two pass approach.
+// The first pass is only related to what SPDX considers the truth. These K:V pairs will never be overwritten.
+// The second pass attempts to generate known short/long version listings for each key.
+// For info on some short name conventions see this document:
+// https://www.debian.org/doc/packaging-manuals/copyright-format/1.0/#license-short-name.
+// The short long listing generation attempts to build all license permutations for a given key.
+// The new keys are then also associated with their relative SPDX value. If a key has already been entered
+// we know to ignore it since it came from the first pass which is considered the SPDX source of truth.
+// We also sort the licenses for the second pass so that cases like `GPL-1` associate to `GPL-1.0` and not `GPL-1.1`.
+func processSPDXLicense(result LicenseList) map[string]string {
+	// first pass build map
+	var licenseIDs = make(map[string]string)
+	for _, l := range result.Licenses {
+		cleanID := strings.ToLower(l.ID)
+		if _, exists := licenseIDs[cleanID]; exists {
+			log.Fatalf("duplicate license ID found: %q", cleanID)
+		}
+		licenseIDs[cleanID] = l.ID
+	}
+
+	sort.Slice(result.Licenses, func(i, j int) bool {
+		return result.Licenses[i].ID < result.Licenses[j].ID
+	})
+
+	// second pass build exceptions
+	// do not overwrite if already exists
+	for _, l := range result.Licenses {
+		var multipleID []string
+		cleanID := strings.ToLower(l.ID)
+		multipleID = append(multipleID, buildLicensePermutations(cleanID)...)
+		for _, id := range multipleID {
+			if _, exists := licenseIDs[id]; !exists {
+				licenseIDs[id] = l.ID
+			}
+		}
+	}
+
+	return licenseIDs
+}
+
+func buildLicensePermutations(license string) (perms []string) {
+	lv := findLicenseVersion(license)
+	vp := versionPermutations(lv)
+
+	version := strings.Join(lv, ".")
+	for _, p := range vp {
+		perms = append(perms, strings.Replace(license, version, p, 1))
+	}
+
+	return perms
+}
+
+func findLicenseVersion(license string) (version []string) {
+	versionList := versionMatch.FindAllStringSubmatch(license, -1)
+
+	if len(versionList) == 0 {
+		return version
+	}
+
+	for i, v := range versionList[0] {
+		if v != "" && i != 0 {
+			version = append(version, v)
+		}
+	}
+
+	return version
+}
+
+func versionPermutations(version []string) []string {
+	ver := append([]string(nil), version...)
+	perms := strset.New()
+	for i := 1; i <= 3; i++ {
+		if len(ver) < i+1 {
+			ver = append(ver, "0")
+		}
+
+		perm := strings.Join(ver[:i], ".")
+		badCount := strings.Count(perm, "0") + strings.Count(perm, ".")
+
+		if badCount != len(perm) {
+			perms.Add(perm)
+		}
+	}
+
+	return perms.List()
+}

internal/spdxlicense/generate/generate_license_list_test.go

@@ -0,0 +1,146 @@
+package main
+
+import (
+	"strings"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestLicensePermutations(t *testing.T) {
+	var tests = []struct {
+		shortName    string
+		permutations []string
+	}{
+		{
+			"GPL-1-only",
+			[]string{
+				"GPL-1-only",
+				"GPL-1.0-only",
+				"GPL-1.0.0-only",
+			},
+		},
+		{
+			"GPL-2",
+			[]string{
+				"GPL-2",
+				"GPL-2.0",
+				"GPL-2.0.0",
+			},
+		},
+		{
+			"GPL-2.0+",
+			[]string{
+				"GPL-2+",
+				"GPL-2.0+",
+				"GPL-2.0.0+",
+			},
+		},
+		{
+			"GPL-3.0.0-or-later",
+			[]string{
+				"GPL-3-or-later",
+				"GPL-3.0-or-later",
+				"GPL-3.0.0-or-later",
+			},
+		},
+		{
+			"oldap-2.0",
+			[]string{
+				"oldap-2",
+				"oldap-2.0",
+				"oldap-2.0.0",
+			},
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.shortName, func(t *testing.T) {
+			perms := buildLicensePermutations(test.shortName)
+			assert.ElementsMatch(t, test.permutations, perms)
+		})
+	}
+}
+
+func TestVersionPermutations(t *testing.T) {
+	var tests = []struct {
+		version      []string
+		permutations []string
+	}{
+		{
+			[]string{"1", "0"},
+			[]string{"1", "1.0", "1.0.0"},
+		},
+		{
+			[]string{"2"},
+			[]string{"2", "2.0", "2.0.0"},
+		},
+		{
+			[]string{"2", "0"},
+			[]string{"2", "2.0", "2.0.0"},
+		},
+
+		{
+			[]string{"3", "0", "0"},
+			[]string{"3", "3.0", "3.0.0"},
+		},
+		{
+			[]string{"0", "3"},
+			[]string{"0.3", "0.3.0"},
+		},
+		{
+			[]string{"0", "0", "3"},
+			[]string{"0.0.3"},
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(strings.Join(test.version, "."), func(t *testing.T) {
+			got := versionPermutations(test.version)
+			assert.ElementsMatch(t, test.permutations, got)
+		})
+	}
+}
+
+func TestFindLicenseVersion(t *testing.T) {
+	var tests = []struct {
+		license string
+		version []string
+	}{
+		{
+			"GPL-1.0-only",
+			[]string{"1", "0"},
+		},
+		{
+			"GPL-2.0",
+			[]string{"2", "0"},
+		},
+		{
+			"GPL-2.0.0",
+			[]string{"2", "0", "0"},
+		},
+		{
+			"GPL-2",
+			[]string{"2"},
+		},
+		{
+			"bzip2-1",
+			[]string{"1"},
+		},
+		{
+			"php-3.01",
+			[]string{"3", "01"},
+		},
+		{
+			"oldap-2.0",
+			[]string{"2", "0"},
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.license, func(t *testing.T) {
+			got := findLicenseVersion(test.license)
+			assert.Equal(t, test.version, got)
+		})
+	}
+}

internal/spdxlicense/generate_license_list.go

@@ -1,93 +0,0 @@
-//go:build ignore
-// +build ignore
-
-package main
-
-import (
-	"encoding/json"
-	"log"
-	"net/http"
-	"os"
-	"strings"
-	"text/template"
-	"time"
-)
-
-// This program generates license_list.go.
-const (
-	source = "license_list.go"
-	url    = "https://spdx.org/licenses/licenses.json"
-)
-
-var tmp = template.Must(template.New("").Parse(`// Code generated by go generate; DO NOT EDIT.
-// This file was generated by robots at {{ .Timestamp }}
-// using data from {{ .URL }}
-package spdxlicense
-
-const Version = {{ printf "%q" .Version }}
-
-var licenseIDs = map[string]string{
-{{- range $k, $v := .LicenseIDs }}
-	{{ printf "%q" $k }}: {{ printf "%q" $v }},
-{{- end }}
-}
-`))
-
-type LicenseList struct {
-	Version  string `json:"licenseListVersion"`
-	Licenses []struct {
-		ID          string   `json:"licenseId"`
-		Name        string   `json:"name"`
-		Text        string   `json:"licenseText"`
-		Deprecated  bool     `json:"isDeprecatedLicenseId"`
-		OSIApproved bool     `json:"isOsiApproved"`
-		SeeAlso     []string `json:"seeAlso"`
-	} `json:"licenses"`
-}
-
-func main() {
-	resp, err := http.Get(url)
-	if err != nil {
-		log.Fatalf("unable to get licenses list: %+v", err)
-	}
-
-	var result LicenseList
-	if err = json.NewDecoder(resp.Body).Decode(&result); err != nil {
-		log.Fatalf("unable to decode license list: %+v", err)
-	}
-
-	f, err := os.Create(source)
-	if err != nil {
-		log.Fatalf("unable to create %q: %+v", source, err)
-	}
-	defer func() {
-		if err := f.Close(); err != nil {
-			log.Fatalf("unable to close %q: %+v", source, err)
-		}
-	}()
-
-	var licenseIDs = make(map[string]string)
-	for _, l := range result.Licenses {
-		cleanID := strings.ToLower(l.ID)
-		if _, exists := licenseIDs[cleanID]; exists {
-			log.Fatalf("duplicate license ID found: %q", cleanID)
-		}
-		licenseIDs[cleanID] = l.ID
-	}
-
-	err = tmp.Execute(f, struct {
-		Timestamp  time.Time
-		URL        string
-		Version    string
-		LicenseIDs map[string]string
-	}{
-		Timestamp:  time.Now(),
-		URL:        url,
-		Version:    result.Version,
-		LicenseIDs: licenseIDs,
-	})
-
-	if err != nil {
-		log.Fatalf("unable to generate template: %+v", err)
-	}
-}

internal/spdxlicense/license.go

@@ -4,7 +4,15 @@ import (
 	"strings"
 )
 
-//go:generate go run generate_license_list.go
+// https://www.debian.org/doc/packaging-manuals/copyright-format/1.0/#license-short-name
+// License generated in license_list.go uses a regular expression to help resolve cases where
+// x.0.0 and x are supplied as version numbers. For SPDX compatibility, versions with trailing
+// dot-zeroes are considered to be equivalent to versions without (e.g., “2.0.0” is considered equal to “2.0” and “2”).
+// EX: gpl-2+ ---> GPL-2.0+
+// EX: gpl-2.0.0-only ---> GPL-2.0-only
+// See the debian link for more details on the spdx license differences
+
+//go:generate go run generate/generate_license_list.go
 
 func ID(id string) (string, bool) {
 	value, exists := licenseIDs[strings.ToLower(id)]

internal/spdxlicense/license_list_test.go

@@ -10,5 +10,6 @@ func TestLicenceListIDs(t *testing.T) {
 	// do a sanity check on the generated data
 	assert.Equal(t, "0BSD", licenseIDs["0bsd"])
 	assert.Equal(t, "ZPL-2.1", licenseIDs["zpl-2.1"])
+	assert.Equal(t, "GPL-2.0", licenseIDs["gpl-2"])
 	assert.NotEmpty(t, Version)
 }

internal/spdxlicense/license_test.go

@@ -0,0 +1,60 @@
+package spdxlicense
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestIDParse(t *testing.T) {
+	var tests = []struct {
+		shortName string
+		spdx      string
+	}{
+		{
+			"GPL-1-only",
+			"GPL-1.0-only",
+		},
+		{
+			"GPL-2",
+			"GPL-2.0",
+		},
+		{
+			"GPL-2+",
+			"GPL-2.0+",
+		},
+		{
+			"GPL-3.0.0-or-later",
+			"GPL-3.0-or-later",
+		},
+		{
+			"GPL-3-with-autoconf-exception",
+			"GPL-3.0-with-autoconf-exception",
+		},
+		{
+			"CC-by-nc-3-de",
+			"CC-BY-NC-3.0-DE",
+		},
+		// the below few cases are NOT expected, however, seem unavoidable given the current approach
+		{
+			"w3c-20150513.0.0",
+			"W3C-20150513",
+		},
+		{
+			"spencer-86.0.0",
+			"Spencer-86",
+		},
+		{
+			"unicode-dfs-2015.0.0",
+			"Unicode-DFS-2015",
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.shortName, func(t *testing.T) {
+			got, exists := ID(test.shortName)
+			assert.True(t, exists)
+			assert.Equal(t, test.spdx, got)
+		})
+	}
+}