adding ruby gemspec support.

Signed-off-by: Toure Dunnon <toure.dunnon@anchore.com>

syft/cataloger/bundler/cataloger.go

@@ -18,7 +18,8 @@ type Cataloger struct {
 // New returns a new Bundler cataloger object.
 func New() *Cataloger {
 	globParsers := map[string]common.ParserFn{
-		"**/Gemfile.lock": parseGemfileLockEntries,
+		"**/Gemfile.lock": parseGemfileLockEntries, // valid in a dir context
+		//"**/specification/*.gemspec": parseGemSpecEntries,     // valid in an image context (against installed gems)
 	}
 
 	return &Cataloger{

syft/cataloger/bundler/parse_gemfile_lock_test.go

@@ -7,61 +7,62 @@ import (
 	"github.com/anchore/syft/syft/pkg"
 )
 
-var expected = map[string]string{
-	"actionmailer":         "4.1.1",
-	"actionpack":           "4.1.1",
-	"actionview":           "4.1.1",
-	"activemodel":          "4.1.1",
-	"activerecord":         "4.1.1",
-	"activesupport":        "4.1.1",
-	"arel":                 "5.0.1.20140414130214",
-	"bootstrap-sass":       "3.1.1.1",
-	"builder":              "3.2.2",
-	"coffee-rails":         "4.0.1",
-	"coffee-script":        "2.2.0",
-	"coffee-script-source": "1.7.0",
-	"erubis":               "2.7.0",
-	"execjs":               "2.0.2",
-	"hike":                 "1.2.3",
-	"i18n":                 "0.6.9",
-	"jbuilder":             "2.0.7",
-	"jquery-rails":         "3.1.0",
-	"json":                 "1.8.1",
-	"kgio":                 "2.9.2",
-	"libv8":                "3.16.14.3",
-	"mail":                 "2.5.4",
-	"mime-types":           "1.25.1",
-	"minitest":             "5.3.4",
-	"multi_json":           "1.10.1",
-	"mysql2":               "0.3.16",
-	"polyglot":             "0.3.4",
-	"rack":                 "1.5.2",
-	"rack-test":            "0.6.2",
-	"rails":                "4.1.1",
-	"railties":             "4.1.1",
-	"raindrops":            "0.13.0",
-	"rake":                 "10.3.2",
-	"rdoc":                 "4.1.1",
-	"ref":                  "1.0.5",
-	"sass":                 "3.2.19",
-	"sass-rails":           "4.0.3",
-	"sdoc":                 "0.4.0",
-	"spring":               "1.1.3",
-	"sprockets":            "2.11.0",
-	"sprockets-rails":      "2.1.3",
-	"sqlite3":              "1.3.9",
-	"therubyracer":         "0.12.1",
-	"thor":                 "0.19.1",
-	"thread_safe":          "0.3.3",
-	"tilt":                 "1.4.1",
-	"treetop":              "1.4.15",
-	"turbolinks":           "2.2.2",
-	"tzinfo":               "1.2.0",
-	"uglifier":             "2.5.0",
-	"unicorn":              "4.8.3",
-}
-
 func TestParseGemfileLockEntries(t *testing.T) {
+
+	var expectedGems = map[string]string{
+		"actionmailer":         "4.1.1",
+		"actionpack":           "4.1.1",
+		"actionview":           "4.1.1",
+		"activemodel":          "4.1.1",
+		"activerecord":         "4.1.1",
+		"activesupport":        "4.1.1",
+		"arel":                 "5.0.1.20140414130214",
+		"bootstrap-sass":       "3.1.1.1",
+		"builder":              "3.2.2",
+		"coffee-rails":         "4.0.1",
+		"coffee-script":        "2.2.0",
+		"coffee-script-source": "1.7.0",
+		"erubis":               "2.7.0",
+		"execjs":               "2.0.2",
+		"hike":                 "1.2.3",
+		"i18n":                 "0.6.9",
+		"jbuilder":             "2.0.7",
+		"jquery-rails":         "3.1.0",
+		"json":                 "1.8.1",
+		"kgio":                 "2.9.2",
+		"libv8":                "3.16.14.3",
+		"mail":                 "2.5.4",
+		"mime-types":           "1.25.1",
+		"minitest":             "5.3.4",
+		"multi_json":           "1.10.1",
+		"mysql2":               "0.3.16",
+		"polyglot":             "0.3.4",
+		"rack":                 "1.5.2",
+		"rack-test":            "0.6.2",
+		"rails":                "4.1.1",
+		"railties":             "4.1.1",
+		"raindrops":            "0.13.0",
+		"rake":                 "10.3.2",
+		"rdoc":                 "4.1.1",
+		"ref":                  "1.0.5",
+		"sass":                 "3.2.19",
+		"sass-rails":           "4.0.3",
+		"sdoc":                 "0.4.0",
+		"spring":               "1.1.3",
+		"sprockets":            "2.11.0",
+		"sprockets-rails":      "2.1.3",
+		"sqlite3":              "1.3.9",
+		"therubyracer":         "0.12.1",
+		"thor":                 "0.19.1",
+		"thread_safe":          "0.3.3",
+		"tilt":                 "1.4.1",
+		"treetop":              "1.4.15",
+		"turbolinks":           "2.2.2",
+		"tzinfo":               "1.2.0",
+		"uglifier":             "2.5.0",
+		"unicorn":              "4.8.3",
+	}
+
 	fixture, err := os.Open("test-fixtures/Gemfile.lock")
 	if err != nil {
 		t.Fatalf("failed to open fixture: %+v", err)
@@ -72,15 +73,15 @@ func TestParseGemfileLockEntries(t *testing.T) {
 		t.Fatalf("failed to parse gemfile lock: %+v", err)
 	}
 
-	if len(actual) != len(expected) {
+	if len(actual) != len(expectedGems) {
 		for _, a := range actual {
 			t.Log("   ", a)
 		}
-		t.Fatalf("unexpected package count: %d!=%d", len(actual), len(expected))
+		t.Fatalf("unexpected package count: %d!=%d", len(actual), len(expectedGems))
 	}
 
 	for _, a := range actual {
-		expectedVersion, ok := expected[a.Name]
+		expectedVersion, ok := expectedGems[a.Name]
 		if !ok {
 			t.Errorf("unexpected package found: %s", a.Name)
 		}

syft/cataloger/bundler/parse_gemspec.go

@@ -0,0 +1,125 @@
+package bundler
+
+import (
+	"bufio"
+	"fmt"
+	"io"
+	"regexp"
+	"strings"
+
+	"github.com/mitchellh/mapstructure"
+
+	"github.com/anchore/syft/syft/cataloger/common"
+	"github.com/anchore/syft/syft/pkg"
+)
+
+// integrity check
+var _ common.ParserFn = parseGemfileLockEntries
+
+// for line in gem.splitlines():
+// line = line.strip()
+// line = re.sub(r"\.freeze", "", line)
+
+// # look for the unicode \u{} format and try to convert to something python can use
+// patt = re.match(r".*\.homepage *= *(.*) *", line)
+// if patt:
+// 	sourcepkg = json.loads(patt.group(1))
+
+// patt = re.match(r".*\.licenses *= *(.*) *", line)
+// if patt:
+// 	lstr = re.sub(r"^\[|\]$", "", patt.group(1)).split(',')
+// 	for thestr in lstr:
+// 		thestr = re.sub(' *" *', "", thestr)
+// 		lics.append(thestr)
+
+// patt = re.match(r".*\.authors *= *(.*) *", line)
+// if patt:
+// 	lstr = re.sub(r"^\[|\]$", "", patt.group(1)).split(',')
+// 	for thestr in lstr:
+// 		thestr = re.sub(' *" *', "", thestr)
+// 		origins.append(thestr)
+
+// patt = re.match(r".*\.files *= *(.*) *", line)
+// if patt:
+// 	lstr = re.sub(r"^\[|\]$", "", patt.group(1)).split(',')
+// 	for thestr in lstr:
+// 		thestr = re.sub(' *" *', "", thestr)
+// 		rfiles.append(thestr)
+
+type listProcessor func(string) []string
+
+var patterns = map[string]*regexp.Regexp{
+	// match example:       name = "railties".freeze   --->   railties
+	"name": regexp.MustCompile(`.*\.name\s*=\s*["']{1}(?P<name>.*)["']{1} *`),
+	// match example:       version = "1.0.4".freeze   --->   1.0.4
+	"version": regexp.MustCompile(`.*\.version\s*=\s*["']{1}(?P<version>.*)["']{1} *`),
+	// match example:       homepage = "https://github.com/anchore/syft".freeze   --->   https://github.com/anchore/syft
+	"homepage": regexp.MustCompile(`.*\.homepage\s*=\s*["']{1}(?P<homepage>.*)["']{1} *`),
+	// TODO: add more fields
+}
+
+// TODO: use post processors for lists
+var postProcessors = map[string]listProcessor{
+	//"files": func(s string) []string {
+	//
+	//},
+}
+
+func parseGemspecEntries(_ string, reader io.Reader) ([]pkg.Package, error) {
+	var pkgs []pkg.Package
+	var fields = make(map[string]interface{})
+	scanner := bufio.NewScanner(reader)
+
+	for scanner.Scan() {
+		line := scanner.Text()
+
+		// TODO: sanitize unicode? (see engine code)
+		sanitizedLine := strings.TrimSpace(line)
+
+		if sanitizedLine == "" {
+			continue
+		}
+
+		for field, pattern := range patterns {
+			matchMap := matchCaptureGroups(pattern, sanitizedLine)
+			if value := matchMap[field]; value != "" {
+				if postProcessor := postProcessors[field]; postProcessor != nil {
+					fields[field] = postProcessor(value)
+				} else {
+					fields[field] = value
+				}
+				// TODO: know that a line could actually match on multiple patterns, this is unlikely though
+				break
+			}
+		}
+	}
+
+	if fields["name"] != "" && fields["version"] != "" {
+		var metadata pkg.GemMetadata
+		if err := mapstructure.Decode(fields, &metadata); err != nil {
+			return nil, fmt.Errorf("unable to decode gem metadata: %w", err)
+		}
+
+		pkgs = append(pkgs, pkg.Package{
+			Name:     metadata.Name,
+			Version:  metadata.Version,
+			Language: pkg.Ruby,
+			Type:     pkg.BundlerPkg,
+			Metadata: metadata,
+		})
+	}
+
+	return pkgs, nil
+}
+
+// matchCaptureGroups takes a regular expression and string and returns all of the named capture group results in a map.
+func matchCaptureGroups(regEx *regexp.Regexp, str string) map[string]string {
+	match := regEx.FindStringSubmatch(str)
+	results := make(map[string]string)
+	for i, name := range regEx.SubexpNames() {
+		if i > 0 && i <= len(match) {
+			results[name] = match[i]
+		}
+	}
+	return results
+}

syft/cataloger/bundler/parse_gemspec_test.go

@@ -0,0 +1,50 @@
+package bundler
+
+import (
+	"os"
+	"testing"
+
+	"github.com/anchore/syft/syft/pkg"
+)
+
+func TestParseGemspec(t *testing.T) {
+	var expectedGems = map[string]string{
+		"bundler": "2.1.4",
+	}
+
+	fixture, err := os.Open("test-fixtures/bundler.gemspec")
+	if err != nil {
+		t.Fatalf("failed to open fixture: %+v", err)
+	}
+
+	actual, err := parseGemspecEntries(fixture.Name(), fixture)
+	if err != nil {
+		t.Fatalf("failed to parse gemspec: %+v", err)
+	}
+
+	if len(actual) != len(expectedGems) {
+		for _, a := range actual {
+			t.Log("   ", a)
+		}
+		t.Fatalf("unexpected package count: %d!=%d", len(actual), len(expectedGems))
+	}
+
+	for _, a := range actual {
+		expectedVersion, ok := expectedGems[a.Name]
+		if !ok {
+			t.Errorf("unexpected package found: %s", a.Name)
+		}
+
+		if expectedVersion != a.Version {
+			t.Errorf("unexpected package version (pkg=%s): %s", a.Name, a.Version)
+		}
+
+		if a.Language != pkg.Ruby {
+			t.Errorf("bad language (pkg=%+v): %+v", a.Name, a.Language)
+		}
+
+		if a.Type != pkg.BundlerPkg {
+			t.Errorf("bad package type (pkg=%+v): %+v", a.Name, a.Type)
+		}
+	}
+}

syft/cataloger/bundler/test-fixtures/bundler.gemspec

@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+# -*- encoding: utf-8 -*-                                                                                                                                         
+# stub: bundler 2.1.4 ruby lib                                                                                                                                    
+                                                                                                                                                                  
+Gem::Specification.new do |s|                                                                                                                                     
+    s.name = "bundler".freeze                                                                                                                                       
+    s.version = "2.1.4"                                                                                                                                             
+                                                                                                                                                                    
+    s.required_rubygems_version = Gem::Requirement.new(">= 2.5.2".freeze) if s.respond_to? :required_rubygems_version=                                              
+    s.require_paths = ["lib".freeze]                                                                                                                                
+    s.authors = ["Andr\u00E9 Arko".freeze, "Samuel Giddins".freeze, "Colby Swandale".freeze, "Hiroshi Shibata".freeze, "David Rodr\u00EDguez".freeze, "Grey Baker".f
+    s.bindir = "exe".freeze                                                                                                                                         
+    s.date = "2020-01-05"                                                                                                                                           
+    s.description = "Bundler manages an application's dependencies through its entire life, across many machines, systematically and repeatably".freeze             
+    s.email = ["team@bundler.io".freeze]                                                                                                                            
+    s.executables = ["bundle".freeze, "bundler".freeze]                                                                                                             
+    s.files = ["exe/bundle".freeze, "exe/bundler".freeze]                                                                                                           
+    s.homepage = "https://bundler.io".freeze                                                                                                                        
+    s.licenses = ["MIT".freeze]                                                                                                                                     
+    s.required_ruby_version = Gem::Requirement.new(">= 2.3.0".freeze)                                                                                               
+    s.rubygems_version = "3.1.2".freeze                                                                                                                             
+    s.summary = "The best way to manage your application's dependencies".freeze                                                                                     
+                                                                                                                                                                    
+    s.installed_by_version = "3.1.2" if s.respond_to? :installed_by_version                                                                                         
+  end 

syft/lib.go

@@ -62,6 +62,16 @@ func IdentifyDistro(s scope.Scope) distro.Distro {
 // Catalog the given scope, which may represent a container image or filesystem. Returns the discovered set of packages.
 func CatalogFromScope(s scope.Scope) (*pkg.Catalog, error) {
 	log.Info("building the catalog")
+
+	// conditionally have two sets of catalogers
+	//var catalogers []cataloger.Cataloger
+	//// if image
+	//// use one set of catalogers
+	//catalogers = ...
+	//
+	//// if dir
+	//// use another set of catalogers
+
 	return cataloger.Catalog(s.Resolver, cataloger.All()...)
 }
 

syft/pkg/gem_metadata.go

@@ -0,0 +1,7 @@
+package pkg
+
+type GemMetadata struct {
+	Name    string `mapstructure:"name" json:"name"`
+	Version string `mapstructure:"version" json:"version"`
+	// TODO: add more fields from the gemspec
+}