migrate syft/cataloger to syft/pkg/cataloger

Signed-off-by: Alex Goodman <alex.goodman@anchore.com>

syft/pkg/cataloger/ruby/catalogers.go

@@ -0,0 +1,26 @@
+/*
+Package bundler provides a concrete Cataloger implementation for Ruby Gemfile.lock bundler files.
+*/
+package ruby
+
+import (
+	"github.com/anchore/syft/syft/pkg/cataloger/common"
+)
+
+// NewGemFileLockCataloger returns a new Bundler cataloger object tailored for parsing index-oriented files (e.g. Gemfile.lock).
+func NewGemFileLockCataloger() *common.GenericCataloger {
+	globParsers := map[string]common.ParserFn{
+		"**/Gemfile.lock": parseGemFileLockEntries,
+	}
+
+	return common.NewGenericCataloger(nil, globParsers, "ruby-gemfile-cataloger")
+}
+
+// NewGemSpecCataloger returns a new Bundler cataloger object tailored for detecting installations of gems (e.g. Gemspec).
+func NewGemSpecCataloger() *common.GenericCataloger {
+	globParsers := map[string]common.ParserFn{
+		"**/specifications/**/*.gemspec": parseGemSpecEntries,
+	}
+
+	return common.NewGenericCataloger(nil, globParsers, "ruby-gemspec-cataloger")
+}

syft/pkg/cataloger/ruby/parse_gemfile_lock.go

@@ -0,0 +1,62 @@
+package ruby
+
+import (
+	"bufio"
+	"io"
+	"strings"
+
+	"github.com/anchore/syft/internal"
+	"github.com/anchore/syft/syft/pkg"
+	"github.com/anchore/syft/syft/pkg/cataloger/common"
+)
+
+// integrity check
+var _ common.ParserFn = parseGemFileLockEntries
+
+var sectionsOfInterest = internal.NewStringSetFromSlice([]string{"GEM"})
+
+// parseGemFileLockEntries is a parser function for Gemfile.lock contents, returning all Gems discovered.
+func parseGemFileLockEntries(_ string, reader io.Reader) ([]pkg.Package, error) {
+	pkgs := make([]pkg.Package, 0)
+	scanner := bufio.NewScanner(reader)
+
+	var currentSection string
+
+	for scanner.Scan() {
+		line := scanner.Text()
+		sanitizedLine := strings.TrimSpace(line)
+
+		if len(line) > 1 && line[0] != ' ' {
+			// start of section
+			currentSection = sanitizedLine
+			continue
+		} else if !sectionsOfInterest.Contains(currentSection) {
+			// skip this line, we're in the wrong section
+			continue
+		}
+
+		if isDependencyLine(line) {
+			candidate := strings.Fields(sanitizedLine)
+			if len(candidate) != 2 {
+				continue
+			}
+			pkgs = append(pkgs, pkg.Package{
+				Name:     candidate[0],
+				Version:  strings.Trim(candidate[1], "()"),
+				Language: pkg.Ruby,
+				Type:     pkg.GemPkg,
+			})
+		}
+	}
+	if err := scanner.Err(); err != nil {
+		return nil, err
+	}
+	return pkgs, nil
+}
+
+func isDependencyLine(line string) bool {
+	if len(line) < 5 {
+		return false
+	}
+	return strings.Count(line[:5], " ") == 4
+}

syft/pkg/cataloger/ruby/parse_gemfile_lock_test.go

@@ -0,0 +1,101 @@
+package ruby
+
+import (
+	"os"
+	"testing"
+
+	"github.com/anchore/syft/syft/pkg"
+)
+
+func TestParseGemfileLockEntries(t *testing.T) {
+
+	var expectedGems = map[string]string{
+		"actionmailer":         "4.1.1",
+		"actionpack":           "4.1.1",
+		"actionview":           "4.1.1",
+		"activemodel":          "4.1.1",
+		"activerecord":         "4.1.1",
+		"activesupport":        "4.1.1",
+		"arel":                 "5.0.1.20140414130214",
+		"bootstrap-sass":       "3.1.1.1",
+		"builder":              "3.2.2",
+		"coffee-rails":         "4.0.1",
+		"coffee-script":        "2.2.0",
+		"coffee-script-source": "1.7.0",
+		"erubis":               "2.7.0",
+		"execjs":               "2.0.2",
+		"hike":                 "1.2.3",
+		"i18n":                 "0.6.9",
+		"jbuilder":             "2.0.7",
+		"jquery-rails":         "3.1.0",
+		"json":                 "1.8.1",
+		"kgio":                 "2.9.2",
+		"libv8":                "3.16.14.3",
+		"mail":                 "2.5.4",
+		"mime-types":           "1.25.1",
+		"minitest":             "5.3.4",
+		"multi_json":           "1.10.1",
+		"mysql2":               "0.3.16",
+		"polyglot":             "0.3.4",
+		"rack":                 "1.5.2",
+		"rack-test":            "0.6.2",
+		"rails":                "4.1.1",
+		"railties":             "4.1.1",
+		"raindrops":            "0.13.0",
+		"rake":                 "10.3.2",
+		"rdoc":                 "4.1.1",
+		"ref":                  "1.0.5",
+		"sass":                 "3.2.19",
+		"sass-rails":           "4.0.3",
+		"sdoc":                 "0.4.0",
+		"spring":               "1.1.3",
+		"sprockets":            "2.11.0",
+		"sprockets-rails":      "2.1.3",
+		"sqlite3":              "1.3.9",
+		"therubyracer":         "0.12.1",
+		"thor":                 "0.19.1",
+		"thread_safe":          "0.3.3",
+		"tilt":                 "1.4.1",
+		"treetop":              "1.4.15",
+		"turbolinks":           "2.2.2",
+		"tzinfo":               "1.2.0",
+		"uglifier":             "2.5.0",
+		"unicorn":              "4.8.3",
+	}
+
+	fixture, err := os.Open("test-fixtures/Gemfile.lock")
+	if err != nil {
+		t.Fatalf("failed to open fixture: %+v", err)
+	}
+
+	actual, err := parseGemFileLockEntries(fixture.Name(), fixture)
+	if err != nil {
+		t.Fatalf("failed to parse gemfile lock: %+v", err)
+	}
+
+	if len(actual) != len(expectedGems) {
+		for _, a := range actual {
+			t.Log("   ", a)
+		}
+		t.Fatalf("unexpected package count: %d!=%d", len(actual), len(expectedGems))
+	}
+
+	for _, a := range actual {
+		expectedVersion, ok := expectedGems[a.Name]
+		if !ok {
+			t.Errorf("unexpected package found: %s", a.Name)
+		}
+
+		if expectedVersion != a.Version {
+			t.Errorf("unexpected package version (pkg=%s): %s", a.Name, a.Version)
+		}
+
+		if a.Language != pkg.Ruby {
+			t.Errorf("bad language (pkg=%+v): %+v", a.Name, a.Language)
+		}
+
+		if a.Type != pkg.GemPkg {
+			t.Errorf("bad package type (pkg=%+v): %+v", a.Name, a.Type)
+		}
+	}
+}

syft/pkg/cataloger/ruby/parse_gemspec.go

@@ -0,0 +1,126 @@
+package ruby
+
+import (
+	"bufio"
+	"encoding/json"
+	"fmt"
+	"io"
+	"regexp"
+	"strings"
+
+	"github.com/anchore/syft/internal"
+
+	"github.com/mitchellh/mapstructure"
+
+	"github.com/anchore/syft/syft/pkg"
+	"github.com/anchore/syft/syft/pkg/cataloger/common"
+)
+
+// integrity check
+var _ common.ParserFn = parseGemFileLockEntries
+
+type postProcessor func(string) []string
+
+// match example:      Al\u003Ex   --->   003E
+var unicodePattern = regexp.MustCompile(`\\u(?P<unicode>[0-9A-F]{4})`)
+
+var patterns = map[string]*regexp.Regexp{
+	// match example:       name = "railties".freeze   --->   railties
+	"name": regexp.MustCompile(`.*\.name\s*=\s*["']{1}(?P<name>.*)["']{1} *`),
+
+	// match example:       version = "1.0.4".freeze   --->   1.0.4
+	"version": regexp.MustCompile(`.*\.version\s*=\s*["']{1}(?P<version>.*)["']{1} *`),
+
+	// match example:
+	// homepage = "https://github.com/anchore/syft".freeze   --->   https://github.com/anchore/syft
+	"homepage": regexp.MustCompile(`.*\.homepage\s*=\s*["']{1}(?P<homepage>.*)["']{1} *`),
+
+	// match example:       files = ["exe/bundle".freeze, "exe/bundler".freeze]    --->    "exe/bundle".freeze, "exe/bundler".freeze
+	"files": regexp.MustCompile(`.*\.files\s*=\s*\[(?P<files>.*)\] *`),
+
+	// match example:       authors = ["Andr\u00E9 Arko".freeze, "Samuel Giddins".freeze, "Colby Swandale".freeze,
+	//								   "Hiroshi Shibata".freeze, "David Rodr\u00EDguez".freeze, "Grey Baker".freeze...]
+	"authors": regexp.MustCompile(`.*\.authors\s*=\s*\[(?P<authors>.*)\] *`),
+
+	// match example:	    licenses = ["MIT".freeze]   ----> "MIT".freeze
+	"licenses": regexp.MustCompile(`.*\.licenses\s*=\s*\[(?P<licenses>.*)\] *`),
+}
+
+var postProcessors = map[string]postProcessor{
+	"files":    processList,
+	"authors":  processList,
+	"licenses": processList,
+}
+
+func processList(s string) []string {
+	// nolint:prealloc
+	var results []string
+	for _, item := range strings.Split(s, ",") {
+		results = append(results, strings.Trim(item, "\" "))
+	}
+	return results
+}
+
+func parseGemSpecEntries(_ string, reader io.Reader) ([]pkg.Package, error) {
+	var pkgs []pkg.Package
+	var fields = make(map[string]interface{})
+	scanner := bufio.NewScanner(reader)
+
+	for scanner.Scan() {
+		line := scanner.Text()
+
+		sanitizedLine := strings.TrimSpace(line)
+		sanitizedLine = strings.ReplaceAll(sanitizedLine, ".freeze", "")
+		sanitizedLine = renderUtf8(sanitizedLine)
+
+		if sanitizedLine == "" {
+			continue
+		}
+
+		for field, pattern := range patterns {
+			matchMap := internal.MatchCaptureGroups(pattern, sanitizedLine)
+			if value := matchMap[field]; value != "" {
+				if postProcessor := postProcessors[field]; postProcessor != nil {
+					fields[field] = postProcessor(value)
+				} else {
+					fields[field] = value
+				}
+				// TODO: know that a line could actually match on multiple patterns, this is unlikely though
+				break
+			}
+		}
+	}
+
+	if fields["name"] != "" && fields["version"] != "" {
+		var metadata pkg.GemMetadata
+		if err := mapstructure.Decode(fields, &metadata); err != nil {
+			return nil, fmt.Errorf("unable to decode gem metadata: %w", err)
+		}
+
+		pkgs = append(pkgs, pkg.Package{
+			Name:         metadata.Name,
+			Version:      metadata.Version,
+			Licenses:     metadata.Licenses,
+			Language:     pkg.Ruby,
+			Type:         pkg.GemPkg,
+			MetadataType: pkg.GemMetadataType,
+			Metadata:     metadata,
+		})
+	}
+
+	return pkgs, nil
+}
+
+// renderUtf8 takes any string escaped string sub-sections from the ruby string and replaces those sections with the UTF8 runes.
+func renderUtf8(s string) string {
+	fullReplacement := unicodePattern.ReplaceAllStringFunc(s, func(unicodeSection string) string {
+		var replacement string
+		// note: the json parser already has support for interpreting hex-representations of unicode escaped strings as unicode runes.
+		// we can do this ourselves with strconv.Atoi, or leverage the existing json package.
+		if err := json.Unmarshal([]byte(`"`+unicodeSection+`"`), &replacement); err != nil {
+			return unicodeSection
+		}
+		return replacement
+	})
+	return fullReplacement
+}

syft/pkg/cataloger/ruby/parse_gemspec_test.go

@@ -0,0 +1,49 @@
+package ruby
+
+import (
+	"os"
+	"testing"
+
+	"github.com/anchore/syft/syft/pkg"
+	"github.com/go-test/deep"
+)
+
+func TestParseGemspec(t *testing.T) {
+	var expectedPkg = pkg.Package{
+		Name:         "bundler",
+		Version:      "2.1.4",
+		Type:         pkg.GemPkg,
+		Licenses:     []string{"MIT"},
+		Language:     pkg.Ruby,
+		MetadataType: pkg.GemMetadataType,
+		Metadata: pkg.GemMetadata{
+			Name:     "bundler",
+			Version:  "2.1.4",
+			Files:    []string{"exe/bundle", "exe/bundler"},
+			Authors:  []string{"André Arko", "Samuel Giddins", "Colby Swandale", "Hiroshi Shibata", "David Rodríguez", "Grey Baker", "Stephanie Morillo", "Chris Morris", "James Wen", "Tim Moore", "André Medeiros", "Jessica Lynn Suttles", "Terence Lee", "Carl Lerche", "Yehuda Katz"},
+			Licenses: []string{"MIT"},
+			Homepage: "https://bundler.io",
+		},
+	}
+
+	fixture, err := os.Open("test-fixtures/bundler.gemspec")
+	if err != nil {
+		t.Fatalf("failed to open fixture: %+v", err)
+	}
+
+	actual, err := parseGemSpecEntries(fixture.Name(), fixture)
+	if err != nil {
+		t.Fatalf("failed to parse gemspec: %+v", err)
+	}
+
+	if len(actual) != 1 {
+		for _, a := range actual {
+			t.Log("   ", a)
+		}
+		t.Fatalf("unexpected package count: %d!=1", len(actual))
+	}
+
+	for _, d := range deep.Equal(actual[0], expectedPkg) {
+		t.Errorf("diff: %+v", d)
+	}
+}

syft/pkg/cataloger/ruby/test-fixtures/Gemfile.lock

@@ -0,0 +1,149 @@
+GEM
+  remote: https://rubygems.org/
+  specs:
+    actionmailer (4.1.1)
+      actionpack (= 4.1.1)
+      actionview (= 4.1.1)
+      mail (~> 2.5.4)
+    actionpack (4.1.1)
+      actionview (= 4.1.1)
+      activesupport (= 4.1.1)
+      rack (~> 1.5.2)
+      rack-test (~> 0.6.2)
+    actionview (4.1.1)
+      activesupport (= 4.1.1)
+      builder (~> 3.1)
+      erubis (~> 2.7.0)
+    activemodel (4.1.1)
+      activesupport (= 4.1.1)
+      builder (~> 3.1)
+    activerecord (4.1.1)
+      activemodel (= 4.1.1)
+      activesupport (= 4.1.1)
+      arel (~> 5.0.0)
+    activesupport (4.1.1)
+      i18n (~> 0.6, >= 0.6.9)
+      json (~> 1.7, >= 1.7.7)
+      minitest (~> 5.1)
+      thread_safe (~> 0.1)
+      tzinfo (~> 1.1)
+    arel (5.0.1.20140414130214)
+    bootstrap-sass (3.1.1.1)
+      sass (~> 3.2)
+    builder (3.2.2)
+    coffee-rails (4.0.1)
+      coffee-script (>= 2.2.0)
+      railties (>= 4.0.0, < 5.0)
+    coffee-script (2.2.0)
+      coffee-script-source
+      execjs
+    coffee-script-source (1.7.0)
+    erubis (2.7.0)
+    execjs (2.0.2)
+    hike (1.2.3)
+    i18n (0.6.9)
+    jbuilder (2.0.7)
+      activesupport (>= 3.0.0, < 5)
+      multi_json (~> 1.2)
+    jquery-rails (3.1.0)
+      railties (>= 3.0, < 5.0)
+      thor (>= 0.14, < 2.0)
+    json (1.8.1)
+    kgio (2.9.2)
+    libv8 (3.16.14.3)
+    mail (2.5.4)
+      mime-types (~> 1.16)
+      treetop (~> 1.4.8)
+    mime-types (1.25.1)
+    minitest (5.3.4)
+    multi_json (1.10.1)
+    mysql2 (0.3.16)
+    polyglot (0.3.4)
+    rack (1.5.2)
+    rack-test (0.6.2)
+      rack (>= 1.0)
+    rails (4.1.1)
+      actionmailer (= 4.1.1)
+      actionpack (= 4.1.1)
+      actionview (= 4.1.1)
+      activemodel (= 4.1.1)
+      activerecord (= 4.1.1)
+      activesupport (= 4.1.1)
+      bundler (>= 1.3.0, < 2.0)
+      railties (= 4.1.1)
+      sprockets-rails (~> 2.0)
+    railties (4.1.1)
+      actionpack (= 4.1.1)
+      activesupport (= 4.1.1)
+      rake (>= 0.8.7)
+      thor (>= 0.18.1, < 2.0)
+    raindrops (0.13.0)
+    rake (10.3.2)
+    rdoc (4.1.1)
+      json (~> 1.4)
+    ref (1.0.5)
+    sass (3.2.19)
+    sass-rails (4.0.3)
+      railties (>= 4.0.0, < 5.0)
+      sass (~> 3.2.0)
+      sprockets (~> 2.8, <= 2.11.0)
+      sprockets-rails (~> 2.0)
+    sdoc (0.4.0)
+      json (~> 1.8)
+      rdoc (~> 4.0, < 5.0)
+    spring (1.1.3)
+    sprockets (2.11.0)
+      hike (~> 1.2)
+      multi_json (~> 1.0)
+      rack (~> 1.0)
+      tilt (~> 1.1, != 1.3.0)
+    sprockets-rails (2.1.3)
+      actionpack (>= 3.0)
+      activesupport (>= 3.0)
+      sprockets (~> 2.8)
+    sqlite3 (1.3.9)
+    therubyracer (0.12.1)
+      libv8 (~> 3.16.14.0)
+      ref
+    thor (0.19.1)
+    thread_safe (0.3.3)
+    tilt (1.4.1)
+    treetop (1.4.15)
+      polyglot
+      polyglot (>= 0.3.1)
+    turbolinks (2.2.2)
+      coffee-rails
+    tzinfo (1.2.0)
+      thread_safe (~> 0.1)
+    uglifier (2.5.0)
+      execjs (>= 0.3.0)
+      json (>= 1.8.0)
+    unicorn (4.8.3)
+      kgio (~> 2.6)
+      rack
+      raindrops (~> 0.7)
+
+BAD-SECTION
+    bad-entry (5.5.5)
+      bad-kgio (~> 2.6)
+      bad-rack
+      bad-raindrops (~> 0.7)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  bootstrap-sass
+  coffee-rails (~> 4.0.0)
+  jbuilder (~> 2.0)
+  jquery-rails
+  mysql2 (~> 0.3.16)
+  rails (= 4.1.1)
+  sass-rails (~> 4.0.3)
+  sdoc (~> 0.4.0)
+  spring
+  sqlite3
+  therubyracer
+  turbolinks
+  uglifier (>= 1.3.0)
+  unicorn

syft/pkg/cataloger/ruby/test-fixtures/bundler.gemspec

@@ -0,0 +1,26 @@
+# frozen_string_literal: true
+# -*- encoding: utf-8 -*-
+# stub: bundler 2.1.4 ruby lib
+
+Gem::Specification.new do |s|
+  s.name = "bundler".freeze
+  s.version = "2.1.4"
+
+  s.required_rubygems_version = Gem::Requirement.new(">= 2.5.2".freeze) if s.respond_to? :required_rubygems_version=
+  s.metadata = { "bug_tracker_uri" => "https://github.com/bundler/bundler/issues", "changelog_uri" => "https://github.com/bundler/bundler/blob/master/CHANGELOG.md", "homepage_uri" => "https://bundler.io/", "source_code_uri" => "https://github.com/bundler/bundler/" } if s.respond_to? :metadata=
+  s.require_paths = ["lib".freeze]
+  s.authors = ["Andr\u00E9 Arko".freeze, "Samuel Giddins".freeze, "Colby Swandale".freeze, "Hiroshi Shibata".freeze, "David Rodr\u00EDguez".freeze, "Grey Baker".freeze, "Stephanie Morillo".freeze, "Chris Morris".freeze, "James Wen".freeze, "Tim Moore".freeze, "Andr\u00E9 Medeiros".freeze, "Jessica Lynn Suttles".freeze, "Terence Lee".freeze, "Carl Lerche".freeze, "Yehuda Katz".freeze]
+  s.bindir = "exe".freeze
+  s.date = "2020-01-05"
+  s.description = "Bundler manages an application's dependencies through its entire life, across many machines, systematically and repeatably".freeze
+  s.email = ["team@bundler.io".freeze]
+  s.executables = ["bundle".freeze, "bundler".freeze]
+  s.files = ["exe/bundle".freeze, "exe/bundler".freeze]
+  s.homepage = "https://bundler.io".freeze
+  s.licenses = ["MIT".freeze]
+  s.required_ruby_version = Gem::Requirement.new(">= 2.3.0".freeze)
+  s.rubygems_version = "3.1.2".freeze
+  s.summary = "The best way to manage your application's dependencies".freeze
+
+  s.installed_by_version = "3.1.2" if s.respond_to? :installed_by_version
+end