more gemspec tests

Signed-off-by: Alex Goodman <alex.goodman@anchore.com>

syft/cataloger/bundler/parse_gemspec.go

@@ -16,53 +16,42 @@ import (
 // integrity check
 var _ common.ParserFn = parseGemFileLockEntries
 
-// for line in gem.splitlines():
+type postProcessor func(string) []string
-// line = line.strip()
-// line = re.sub(r"\.freeze", "", line)
-
-// # look for the unicode \u{} format and try to convert to something python can use
-// patt = re.match(r".*\.homepage *= *(.*) *", line)
-// if patt:
-// 	sourcepkg = json.loads(patt.group(1))
-
-// patt = re.match(r".*\.licenses *= *(.*) *", line)
-// if patt:
-// 	lstr = re.sub(r"^\[|\]$", "", patt.group(1)).split(',')
-// 	for thestr in lstr:
-// 		thestr = re.sub(' *" *', "", thestr)
-// 		lics.append(thestr)
-
-// patt = re.match(r".*\.authors *= *(.*) *", line)
-// if patt:
-// 	lstr = re.sub(r"^\[|\]$", "", patt.group(1)).split(',')
-// 	for thestr in lstr:
-// 		thestr = re.sub(' *" *', "", thestr)
-// 		origins.append(thestr)
-
-// patt = re.match(r".*\.files *= *(.*) *", line)
-// if patt:
-// 	lstr = re.sub(r"^\[|\]$", "", patt.group(1)).split(',')
-// 	for thestr in lstr:
-// 		thestr = re.sub(' *" *', "", thestr)
-// 		rfiles.append(thestr)
-
-type listProcessor func(string) []string
 
 var patterns = map[string]*regexp.Regexp{
 	// match example:       name = "railties".freeze   --->   railties
 	"name": regexp.MustCompile(`.*\.name\s*=\s*["']{1}(?P<name>.*)["']{1} *`),
+
 	// match example:       version = "1.0.4".freeze   --->   1.0.4
 	"version": regexp.MustCompile(`.*\.version\s*=\s*["']{1}(?P<version>.*)["']{1} *`),
-	// match example:       homepage = "https://github.com/anchore/syft".freeze   --->   https://github.com/anchore/syft
+
+	// match example:
+	// homepage = "https://github.com/anchore/syft".freeze   --->   https://github.com/anchore/syft
 	"homepage": regexp.MustCompile(`.*\.homepage\s*=\s*["']{1}(?P<homepage>.*)["']{1} *`),
-	// TODO: add more fields
+
+	// match example:       files = ["exe/bundle".freeze, "exe/bundler".freeze]    --->    "exe/bundle".freeze, "exe/bundler".freeze
+	"files": regexp.MustCompile(`.*\.files\s*=\s*\[(?P<files>.*)\] *`),
+
+	// match example:       authors = ["Andr\u00E9 Arko".freeze, "Samuel Giddins".freeze, "Colby Swandale".freeze,
+	//								   "Hiroshi Shibata".freeze, "David Rodr\u00EDguez".freeze, "Grey Baker".freeze...]
+	"authors": regexp.MustCompile(`.*\.authors\s*=\s*\[(?P<authors>.*)\] *`),
+
+	// match example:	    licenses = ["MIT".freeze]   ----> "MIT".freeze
+	"licenses": regexp.MustCompile(`.*\.licenses\s*=\s*\[(?P<licenses>.*)\] *`),
 }
 
-// TODO: use post processors for lists
+var postProcessors = map[string]postProcessor{
-var postProcessors = map[string]listProcessor{
+	"files":    processList,
-	//"files": func(s string) []string {
+	"authors":  processList,
-	//
+	"licenses": processList,
-	//},
+}
+
+func processList(s string) []string {
+	var results []string
+	for _, item := range strings.Split(s, ",") {
+		results = append(results, strings.Trim(item, "\" "))
+	}
+	return results
 }
 
 func parseGemSpecEntries(_ string, reader io.Reader) ([]pkg.Package, error) {
@@ -75,12 +64,16 @@ func parseGemSpecEntries(_ string, reader io.Reader) ([]pkg.Package, error) {
 
 		// TODO: sanitize unicode? (see engine code)
 		sanitizedLine := strings.TrimSpace(line)
+		sanitizedLine = strings.ReplaceAll(sanitizedLine, ".freeze", "")
 
 		if sanitizedLine == "" {
 			continue
 		}
 
 		for field, pattern := range patterns {
+			if strings.Contains(sanitizedLine, "licenses") {
+				println("Found it.")
+			}
 			matchMap := matchCaptureGroups(pattern, sanitizedLine)
 			if value := matchMap[field]; value != "" {
 				if postProcessor := postProcessors[field]; postProcessor != nil {
@@ -103,6 +96,7 @@ func parseGemSpecEntries(_ string, reader io.Reader) ([]pkg.Package, error) {
 		pkgs = append(pkgs, pkg.Package{
 			Name:     metadata.Name,
 			Version:  metadata.Version,
+			Licenses: metadata.Licenses,
 			Language: pkg.Ruby,
 			Type:     pkg.GemPkg,
 			Metadata: metadata,

syft/cataloger/bundler/parse_gemspec_test.go

@@ -5,11 +5,23 @@ import (
 	"testing"
 
 	"github.com/anchore/syft/syft/pkg"
+	"github.com/go-test/deep"
 )
 
 func TestParseGemspec(t *testing.T) {
-	var expectedGems = map[string]string{
+	var expectedPkg = pkg.Package{
-		"bundler": "2.1.4",
+		Name:     "bundler",
+		Version:  "2.1.4",
+		Type:     pkg.GemPkg,
+		Licenses: []string{"MIT"},
+		Language: pkg.Ruby,
+		Metadata: pkg.GemMetadata{
+			Name:    "bundler",
+			Version: "2.1.4",
+			Files:   []string{"exe/bundle", "exe/bundler"},
+			Authors:  []string{"André Arko", "Samuel Giddins", "Colby Swandale", "Hiroshi Shibata", "David Rodréguez", "Grey Baker", "Stephanie Morillo", "Chris Morris", "James Wen", "Tim Moore", "André Medeiros", "Jessica Lynn Suttles", "Terence Lee", "Carl Lerche", "Yehuda Katz"},
+			Licenses: []string{"MIT"},
+		},
 	}
 
 	fixture, err := os.Open("test-fixtures/bundler.gemspec")
@@ -22,29 +34,14 @@ func TestParseGemspec(t *testing.T) {
 		t.Fatalf("failed to parse gemspec: %+v", err)
 	}
 
-	if len(actual) != len(expectedGems) {
+	if len(actual) != 1 {
 		for _, a := range actual {
 			t.Log("   ", a)
 		}
-		t.Fatalf("unexpected package count: %d!=%d", len(actual), len(expectedGems))
+		t.Fatalf("unexpected package count: %d!=1", len(actual))
 	}
 
-	for _, a := range actual {
+	for _, d := range deep.Equal(actual[0], expectedPkg) {
-		expectedVersion, ok := expectedGems[a.Name]
+		t.Errorf("diff: %+v", d)
-		if !ok {
-			t.Errorf("unexpected package found: %s", a.Name)
-		}
-
-		if expectedVersion != a.Version {
-			t.Errorf("unexpected package version (pkg=%s): %s", a.Name, a.Version)
-		}
-
-		if a.Language != pkg.Ruby {
-			t.Errorf("bad language (pkg=%+v): %+v", a.Name, a.Language)
-		}
-
-		if a.Type != pkg.GemPkg {
-			t.Errorf("bad package type (pkg=%+v): %+v", a.Name, a.Type)
-		}
 	}
 }

syft/cataloger/bundler/test-fixtures/bundler.gemspec

@@ -7,8 +7,9 @@ Gem::Specification.new do |s|
   s.version = "2.1.4"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 2.5.2".freeze) if s.respond_to? :required_rubygems_version=
+  s.metadata = { "bug_tracker_uri" => "https://github.com/bundler/bundler/issues", "changelog_uri" => "https://github.com/bundler/bundler/blob/master/CHANGELOG.md", "homepage_uri" => "https://bundler.io/", "source_code_uri" => "https://github.com/bundler/bundler/" } if s.respond_to? :metadata=
   s.require_paths = ["lib".freeze]
-    s.authors = ["Andr\u00E9 Arko".freeze, "Samuel Giddins".freeze, "Colby Swandale".freeze, "Hiroshi Shibata".freeze, "David Rodr\u00EDguez".freeze, "Grey Baker".f
+  s.authors = ["Andr\u00E9 Arko".freeze, "Samuel Giddins".freeze, "Colby Swandale".freeze, "Hiroshi Shibata".freeze, "David Rodr\u00EDguez".freeze, "Grey Baker".freeze, "Stephanie Morillo".freeze, "Chris Morris".freeze, "James Wen".freeze, "Tim Moore".freeze, "Andr\u00E9 Medeiros".freeze, "Jessica Lynn Suttles".freeze, "Terence Lee".freeze, "Carl Lerche".freeze, "Yehuda Katz".freeze]
   s.bindir = "exe".freeze
   s.date = "2020-01-05"
   s.description = "Bundler manages an application's dependencies through its entire life, across many machines, systematically and repeatably".freeze

syft/pkg/gem_metadata.go

@@ -3,5 +3,7 @@ package pkg
 type GemMetadata struct {
 	Name    string   `mapstructure:"name" json:"name"`
 	Version string   `mapstructure:"version" json:"version"`
-	// TODO: add more fields from the gemspec
+	Files   []string `mapstructure:"files" json:"files"`
+	Authors  []string `mapstructure:"authors" json:"authors"`
+	Licenses []string `mapstructure:"licenses" json:"licenses"`
 }