Remove internal string set (#2219)

* remove internal string set

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

* incorporate changes from #2227

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

* beef up the pkg.License.Merg() doc string

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

---------

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

syft/pkg/cataloger/common/cpe/generate.go

@@ -13,7 +13,6 @@ import (
 	"github.com/facebookincubator/nvdtools/wfn"
 	"github.com/scylladb/go-set/strset"
 
-	"github.com/anchore/syft/internal"
 	"github.com/anchore/syft/internal/log"
 	"github.com/anchore/syft/syft/cpe"
 	"github.com/anchore/syft/syft/pkg"
@@ -118,13 +117,13 @@ func Generate(p pkg.Package) []cpe.CPE {
 		return nil
 	}
 
-	keys := internal.NewStringSet()
+	keys := strset.New()
 	cpes := make([]cpe.CPE, 0)
 	for _, product := range products {
 		for _, vendor := range vendors {
 			// prevent duplicate entries...
 			key := fmt.Sprintf("%s|%s|%s", product, vendor, p.Version)
-			if keys.Contains(key) {
+			if keys.Has(key) {
 				continue
 			}
 			keys.Add(key)

syft/pkg/cataloger/deb/parse_copyright.go

@@ -7,6 +7,8 @@ import (
 	"sort"
 	"strings"
 
+	"github.com/scylladb/go-set/strset"
+
 	"github.com/anchore/syft/internal"
 )
 
@@ -18,7 +20,7 @@ var (
 )
 
 func parseLicensesFromCopyright(reader io.Reader) []string {
-	findings := internal.NewStringSet()
+	findings := strset.New()
 	scanner := bufio.NewScanner(reader)
 
 	for scanner.Scan() {
@@ -31,7 +33,7 @@ func parseLicensesFromCopyright(reader io.Reader) []string {
 		}
 	}
 
-	results := findings.ToSlice()
+	results := findings.List()
 
 	sort.Strings(results)
 

syft/pkg/cataloger/golang/licenses.go

@@ -18,6 +18,7 @@ import (
 	"github.com/go-git/go-git/v5"
 	"github.com/go-git/go-git/v5/plumbing"
 	"github.com/go-git/go-git/v5/storage/memory"
+	"github.com/scylladb/go-set/strset"
 
 	"github.com/anchore/syft/internal/licenses"
 	"github.com/anchore/syft/internal/log"
@@ -31,6 +32,7 @@ type goLicenses struct {
 	opts                  GoCatalogerOpts
 	localModCacheResolver file.WritableResolver
 	progress              *monitor.CatalogerTask
+	licenseFileNames      *strset.Set
 }
 
 func newGoLicenses(opts GoCatalogerOpts) goLicenses {
@@ -42,6 +44,7 @@ func newGoLicenses(opts GoCatalogerOpts) goLicenses {
 			RemoveOnCompletion: true,
 			Title:              "Downloading go mod",
 		},
+		licenseFileNames: strset.New(licenses.FileNames()...),
 	}
 }
 
@@ -77,7 +80,7 @@ func modCacheResolver(modCacheDir string) file.WritableResolver {
 }
 
 func (c *goLicenses) getLicenses(resolver file.Resolver, moduleName, moduleVersion string) (licenses []pkg.License, err error) {
-	licenses, err = findLicenses(resolver,
+	licenses, err = c.findLicenses(resolver,
 		fmt.Sprintf(`**/go/pkg/mod/%s@%s/*`, processCaps(moduleName), moduleVersion),
 	)
 	if err != nil || len(licenses) > 0 {
@@ -102,7 +105,7 @@ func (c *goLicenses) getLicensesFromLocal(moduleName, moduleVersion string) ([]p
 
 	// if we're running against a directory on the filesystem, it may not include the
 	// user's homedir / GOPATH, so we defer to using the localModCacheResolver
-	return findLicenses(c.localModCacheResolver, moduleSearchGlob(moduleName, moduleVersion))
+	return c.findLicenses(c.localModCacheResolver, moduleSearchGlob(moduleName, moduleVersion))
 }
 
 func (c *goLicenses) getLicensesFromRemote(moduleName, moduleVersion string) ([]pkg.License, error) {
@@ -139,7 +142,37 @@ func (c *goLicenses) getLicensesFromRemote(moduleName, moduleVersion string) ([]
 		log.Tracef("remote proxy walk failed for: %s", moduleName)
 	}
 
-	return findLicenses(c.localModCacheResolver, moduleSearchGlob(moduleName, moduleVersion))
+	return c.findLicenses(c.localModCacheResolver, moduleSearchGlob(moduleName, moduleVersion))
+}
+
+func (c *goLicenses) findLicenses(resolver file.Resolver, globMatch string) (out []pkg.License, err error) {
+	out = make([]pkg.License, 0)
+	if resolver == nil {
+		return
+	}
+
+	locations, err := resolver.FilesByGlob(globMatch)
+	if err != nil {
+		return nil, err
+	}
+
+	for _, l := range locations {
+		fileName := path.Base(l.RealPath)
+		if c.licenseFileNames.Has(fileName) {
+			contents, err := resolver.FileContentsByLocation(l)
+			if err != nil {
+				return nil, err
+			}
+			parsed, err := licenses.Parse(contents, l)
+			if err != nil {
+				return nil, err
+			}
+
+			out = append(out, parsed...)
+		}
+	}
+
+	return
 }
 
 func moduleDir(moduleName, moduleVersion string) string {
@@ -157,36 +190,6 @@ func requireCollection(licenses []pkg.License) []pkg.License {
 	return licenses
 }
 
-func findLicenses(resolver file.Resolver, globMatch string) (out []pkg.License, err error) {
-	out = make([]pkg.License, 0)
-	if resolver == nil {
-		return
-	}
-
-	locations, err := resolver.FilesByGlob(globMatch)
-	if err != nil {
-		return nil, err
-	}
-
-	for _, l := range locations {
-		fileName := path.Base(l.RealPath)
-		if licenses.FileNameSet.Contains(fileName) {
-			contents, err := resolver.FileContentsByLocation(l)
-			if err != nil {
-				return nil, err
-			}
-			parsed, err := licenses.Parse(contents, l)
-			if err != nil {
-				return nil, err
-			}
-
-			out = append(out, parsed...)
-		}
-	}
-
-	return
-}
-
 var capReplacer = regexp.MustCompile("[A-Z]")
 
 func processCaps(s string) string {

syft/pkg/cataloger/golang/licenses_test.go

@@ -13,7 +13,6 @@ import (
 
 	"github.com/stretchr/testify/require"
 
-	"github.com/anchore/syft/internal"
 	"github.com/anchore/syft/syft/file"
 	"github.com/anchore/syft/syft/internal/fileresolver"
 	"github.com/anchore/syft/syft/license"
@@ -37,7 +36,6 @@ func Test_LocalLicenseSearch(t *testing.T) {
 				SPDXExpression: "Apache-2.0",
 				Type:           license.Concluded,
 				Locations:      file.NewLocationSet(loc1),
-				URLs:           internal.NewStringSet(),
 			},
 		},
 		{
@@ -48,7 +46,6 @@ func Test_LocalLicenseSearch(t *testing.T) {
 				SPDXExpression: "MIT",
 				Type:           license.Concluded,
 				Locations:      file.NewLocationSet(loc2),
-				URLs:           internal.NewStringSet(),
 			},
 		},
 	}
@@ -128,7 +125,6 @@ func Test_RemoteProxyLicenseSearch(t *testing.T) {
 				SPDXExpression: "Apache-2.0",
 				Type:           license.Concluded,
 				Locations:      file.NewLocationSet(loc1),
-				URLs:           internal.NewStringSet(),
 			},
 		},
 		{
@@ -139,7 +135,6 @@ func Test_RemoteProxyLicenseSearch(t *testing.T) {
 				SPDXExpression: "MIT",
 				Type:           license.Concluded,
 				Locations:      file.NewLocationSet(loc2),
-				URLs:           internal.NewStringSet(),
 			},
 		},
 	}

syft/pkg/cataloger/java/archive_parser.go

@@ -332,7 +332,7 @@ func getDigestsFromArchive(archivePath string) ([]file.Digest, error) {
 
 func (j *archiveParser) getLicenseFromFileInArchive() ([]pkg.License, error) {
 	var fileLicenses []pkg.License
-	for _, filename := range licenses.FileNames {
+	for _, filename := range licenses.FileNames() {
 		licenseMatches := j.fileManifest.GlobMatch("/META-INF/" + filename)
 		if len(licenseMatches) == 0 {
 			// Try the root directory if it's not in META-INF

syft/pkg/cataloger/java/archive_parser_test.go

@@ -12,9 +12,9 @@ import (
 	"testing"
 
 	"github.com/gookit/color"
+	"github.com/scylladb/go-set/strset"
 	"github.com/stretchr/testify/require"
 
-	"github.com/anchore/syft/internal"
 	"github.com/anchore/syft/syft/file"
 	"github.com/anchore/syft/syft/license"
 	"github.com/anchore/syft/syft/pkg"
@@ -163,7 +163,6 @@ func TestParseJar(t *testing.T) {
 							Value:          "Apache-2.0",
 							SPDXExpression: "Apache-2.0",
 							Type:           license.Concluded,
-							URLs:           internal.NewStringSet(),
 							Locations:      file.NewLocationSet(file.NewLocation("test-fixtures/java-builds/packages/example-java-app-gradle-0.1.0.jar")),
 						},
 					),
@@ -237,7 +236,6 @@ func TestParseJar(t *testing.T) {
 							Value:          "Apache-2.0",
 							SPDXExpression: "Apache-2.0",
 							Type:           license.Concluded,
-							URLs:           internal.NewStringSet(),
 							Locations:      file.NewLocationSet(file.NewLocation("test-fixtures/java-builds/packages/example-java-app-maven-0.1.0.jar")),
 						},
 					),
@@ -594,7 +592,7 @@ func TestParseNestedJar(t *testing.T) {
 			})
 			require.NoError(t, err)
 
-			expectedNameVersionPairSet := internal.NewStringSet()
+			expectedNameVersionPairSet := strset.New()
 
 			makeKey := func(p *pkg.Package) string {
 				if p == nil {
@@ -607,24 +605,24 @@ func TestParseNestedJar(t *testing.T) {
 				expectedNameVersionPairSet.Add(makeKey(&e))
 			}
 
-			actualNameVersionPairSet := internal.NewStringSet()
+			actualNameVersionPairSet := strset.New()
 			for _, a := range actual {
 				a := a
 				key := makeKey(&a)
 				actualNameVersionPairSet.Add(key)
-				if !expectedNameVersionPairSet.Contains(key) {
+				if !expectedNameVersionPairSet.Has(key) {
 					t.Errorf("extra package: %s", a)
 				}
 			}
 
-			for _, key := range expectedNameVersionPairSet.ToSlice() {
-				if !actualNameVersionPairSet.Contains(key) {
+			for _, key := range expectedNameVersionPairSet.List() {
+				if !actualNameVersionPairSet.Has(key) {
 					t.Errorf("missing package: %s", key)
 				}
 			}
 
-			if len(actual) != len(expectedNameVersionPairSet) {
-				t.Fatalf("unexpected package count: %d!=%d", len(actual), len(expectedNameVersionPairSet))
+			if len(actual) != expectedNameVersionPairSet.Size() {
+				t.Fatalf("unexpected package count: %d!=%d", len(actual), expectedNameVersionPairSet.Size())
 			}
 
 			for _, a := range actual {
@@ -748,7 +746,7 @@ func Test_newPackageFromMavenData(t *testing.T) {
 						Value:          "MIT",
 						SPDXExpression: "MIT",
 						Type:           license.Declared,
-						URLs:           internal.NewStringSet("https://opensource.org/licenses/MIT"),
+						URLs:           []string{"https://opensource.org/licenses/MIT"},
 						Locations:      file.NewLocationSet(file.NewLocation("some-license-path")),
 					},
 				},
@@ -785,7 +783,7 @@ func Test_newPackageFromMavenData(t *testing.T) {
 						Value:          "MIT",
 						SPDXExpression: "MIT",
 						Type:           license.Declared,
-						URLs:           internal.NewStringSet("https://opensource.org/licenses/MIT"),
+						URLs:           []string{"https://opensource.org/licenses/MIT"},
 						Locations:      file.NewLocationSet(file.NewLocation("some-license-path")),
 					},
 				),

syft/pkg/cataloger/java/parse_pom_xml_test.go

@@ -11,7 +11,6 @@ import (
 	"github.com/stretchr/testify/require"
 	"github.com/vifraa/gopom"
 
-	"github.com/anchore/syft/internal"
 	"github.com/anchore/syft/syft/file"
 	"github.com/anchore/syft/syft/license"
 	"github.com/anchore/syft/syft/pkg"
@@ -342,19 +341,18 @@ func Test_parsePomXMLProject(t *testing.T) {
 						Value:          "The Apache Software License, Version 2.0",
 						SPDXExpression: "", // TODO: ideally we would parse this title to get Apache-2.0 (created issue #2210 https://github.com/anchore/syft/issues/2210)
 						Type:           license.Declared,
-						URLs:           internal.NewStringSet("http://www.apache.org/licenses/LICENSE-2.0.txt"),
+						URLs:           []string{"http://www.apache.org/licenses/LICENSE-2.0.txt"},
 						Locations:      file.NewLocationSet(jarLocation),
 					},
 					{
 						Value:          "MIT",
 						SPDXExpression: "MIT",
 						Type:           license.Declared,
-						URLs:           internal.NewStringSet(),
 						Locations:      file.NewLocationSet(jarLocation),
 					},
 					{
 						Type:      license.Declared,
-						URLs:      internal.NewStringSet("https://opensource.org/license/unlicense/"),
+						URLs:      []string{"https://opensource.org/license/unlicense/"},
 						Locations: file.NewLocationSet(jarLocation),
 					},
 				},

syft/pkg/cataloger/javascript/parse_yarn_lock.go

@@ -5,7 +5,8 @@ import (
 	"fmt"
 	"regexp"
 
-	"github.com/anchore/syft/internal"
+	"github.com/scylladb/go-set/strset"
+
 	"github.com/anchore/syft/syft/artifact"
 	"github.com/anchore/syft/syft/file"
 	"github.com/anchore/syft/syft/pkg"
@@ -51,7 +52,7 @@ func parseYarnLock(resolver file.Resolver, _ *generic.Environment, reader file.L
 
 	var pkgs []pkg.Package
 	scanner := bufio.NewScanner(reader)
-	parsedPackages := internal.NewStringSet()
+	parsedPackages := strset.New()
 	currentPackage := noPackage
 	currentVersion := noVersion
 
@@ -60,7 +61,7 @@ func parseYarnLock(resolver file.Resolver, _ *generic.Environment, reader file.L
 
 		if packageName := findPackageName(line); packageName != noPackage {
 			// When we find a new package, check if we have unsaved identifiers
-			if currentPackage != noPackage && currentVersion != noVersion && !parsedPackages.Contains(currentPackage+"@"+currentVersion) {
+			if currentPackage != noPackage && currentVersion != noVersion && !parsedPackages.Has(currentPackage+"@"+currentVersion) {
 				pkgs = append(pkgs, newYarnLockPackage(resolver, reader.Location, currentPackage, currentVersion))
 				parsedPackages.Add(currentPackage + "@" + currentVersion)
 			}
@@ -68,7 +69,7 @@ func parseYarnLock(resolver file.Resolver, _ *generic.Environment, reader file.L
 			currentPackage = packageName
 		} else if version := findPackageVersion(line); version != noVersion {
 			currentVersion = version
-		} else if packageName, version := findPackageAndVersion(line); packageName != noPackage && version != noVersion && !parsedPackages.Contains(packageName+"@"+version) {
+		} else if packageName, version := findPackageAndVersion(line); packageName != noPackage && version != noVersion && !parsedPackages.Has(packageName+"@"+version) {
 			pkgs = append(pkgs, newYarnLockPackage(resolver, reader.Location, packageName, version))
 			parsedPackages.Add(packageName + "@" + version)
 
@@ -79,7 +80,7 @@ func parseYarnLock(resolver file.Resolver, _ *generic.Environment, reader file.L
 	}
 
 	// check if we have valid unsaved data after end-of-file has reached
-	if currentPackage != noPackage && currentVersion != noVersion && !parsedPackages.Contains(currentPackage+"@"+currentVersion) {
+	if currentPackage != noPackage && currentVersion != noVersion && !parsedPackages.Has(currentPackage+"@"+currentVersion) {
 		pkgs = append(pkgs, newYarnLockPackage(resolver, reader.Location, currentPackage, currentVersion))
 		parsedPackages.Add(currentPackage + "@" + currentVersion)
 	}

syft/pkg/cataloger/portage/parse_portage_contents.go

@@ -9,7 +9,8 @@ import (
 	"strconv"
 	"strings"
 
-	"github.com/anchore/syft/internal"
+	"github.com/scylladb/go-set/strset"
+
 	"github.com/anchore/syft/internal/log"
 	"github.com/anchore/syft/syft/artifact"
 	"github.com/anchore/syft/syft/file"
@@ -105,7 +106,7 @@ func addLicenses(resolver file.Resolver, dbLocation file.Location, p *pkg.Packag
 		return
 	}
 
-	findings := internal.NewStringSet()
+	findings := strset.New()
 	scanner := bufio.NewScanner(licenseReader)
 	scanner.Split(bufio.ScanWords)
 	for scanner.Scan() {
@@ -115,7 +116,7 @@ func addLicenses(resolver file.Resolver, dbLocation file.Location, p *pkg.Packag
 		}
 	}
 
-	licenseCandidates := findings.ToSlice()
+	licenseCandidates := findings.List()
 	p.Licenses = pkg.NewLicenseSet(pkg.NewLicensesFromLocation(*location, licenseCandidates...)...)
 	p.Locations.Add(location.WithAnnotation(pkg.EvidenceAnnotationKey, pkg.SupportingEvidenceAnnotation))
 }

syft/pkg/cataloger/ruby/parse_gemfile_lock.go

@@ -4,7 +4,8 @@ import (
 	"bufio"
 	"strings"
 
-	"github.com/anchore/syft/internal"
+	"github.com/scylladb/go-set/strset"
+
 	"github.com/anchore/syft/syft/artifact"
 	"github.com/anchore/syft/syft/file"
 	"github.com/anchore/syft/syft/pkg"
@@ -13,7 +14,7 @@ import (
 
 var _ generic.Parser = parseGemFileLockEntries
 
-var sectionsOfInterest = internal.NewStringSet("GEM", "GIT", "PATH", "PLUGIN SOURCE")
+var sectionsOfInterest = strset.New("GEM", "GIT", "PATH", "PLUGIN SOURCE")
 
 // parseGemFileLockEntries is a parser function for Gemfile.lock contents, returning all Gems discovered.
 func parseGemFileLockEntries(_ file.Resolver, _ *generic.Environment, reader file.LocationReadCloser) ([]pkg.Package, []artifact.Relationship, error) {
@@ -30,7 +31,7 @@ func parseGemFileLockEntries(_ file.Resolver, _ *generic.Environment, reader fil
 			// start of section
 			currentSection = sanitizedLine
 			continue
-		} else if !sectionsOfInterest.Contains(currentSection) {
+		} else if !sectionsOfInterest.Has(currentSection) {
 			// skip this line, we're in the wrong section
 			continue
 		}