diff --git a/syft/cataloger/bundler/cataloger.go b/syft/cataloger/bundler/cataloger.go index 265702659..aabc4ca42 100644 --- a/syft/cataloger/bundler/cataloger.go +++ b/syft/cataloger/bundler/cataloger.go @@ -18,7 +18,8 @@ type Cataloger struct { // New returns a new Bundler cataloger object. func New() *Cataloger { globParsers := map[string]common.ParserFn{ - "**/Gemfile.lock": parseGemfileLockEntries, + "**/Gemfile.lock": parseGemfileLockEntries, // valid in a dir context + //"**/specification/*.gemspec": parseGemSpecEntries, // valid in an image context (against installed gems) } return &Cataloger{ diff --git a/syft/cataloger/bundler/parse_gemfile_lock_test.go b/syft/cataloger/bundler/parse_gemfile_lock_test.go index cc17ad0af..428a80c01 100644 --- a/syft/cataloger/bundler/parse_gemfile_lock_test.go +++ b/syft/cataloger/bundler/parse_gemfile_lock_test.go @@ -7,61 +7,62 @@ import ( "github.com/anchore/syft/syft/pkg" ) -var expected = map[string]string{ - "actionmailer": "4.1.1", - "actionpack": "4.1.1", - "actionview": "4.1.1", - "activemodel": "4.1.1", - "activerecord": "4.1.1", - "activesupport": "4.1.1", - "arel": "5.0.1.20140414130214", - "bootstrap-sass": "3.1.1.1", - "builder": "3.2.2", - "coffee-rails": "4.0.1", - "coffee-script": "2.2.0", - "coffee-script-source": "1.7.0", - "erubis": "2.7.0", - "execjs": "2.0.2", - "hike": "1.2.3", - "i18n": "0.6.9", - "jbuilder": "2.0.7", - "jquery-rails": "3.1.0", - "json": "1.8.1", - "kgio": "2.9.2", - "libv8": "3.16.14.3", - "mail": "2.5.4", - "mime-types": "1.25.1", - "minitest": "5.3.4", - "multi_json": "1.10.1", - "mysql2": "0.3.16", - "polyglot": "0.3.4", - "rack": "1.5.2", - "rack-test": "0.6.2", - "rails": "4.1.1", - "railties": "4.1.1", - "raindrops": "0.13.0", - "rake": "10.3.2", - "rdoc": "4.1.1", - "ref": "1.0.5", - "sass": "3.2.19", - "sass-rails": "4.0.3", - "sdoc": "0.4.0", - "spring": "1.1.3", - "sprockets": "2.11.0", - "sprockets-rails": "2.1.3", - "sqlite3": "1.3.9", - "therubyracer": "0.12.1", - "thor": "0.19.1", - "thread_safe": "0.3.3", - "tilt": "1.4.1", - "treetop": "1.4.15", - "turbolinks": "2.2.2", - "tzinfo": "1.2.0", - "uglifier": "2.5.0", - "unicorn": "4.8.3", -} - func TestParseGemfileLockEntries(t *testing.T) { + + var expectedGems = map[string]string{ + "actionmailer": "4.1.1", + "actionpack": "4.1.1", + "actionview": "4.1.1", + "activemodel": "4.1.1", + "activerecord": "4.1.1", + "activesupport": "4.1.1", + "arel": "5.0.1.20140414130214", + "bootstrap-sass": "3.1.1.1", + "builder": "3.2.2", + "coffee-rails": "4.0.1", + "coffee-script": "2.2.0", + "coffee-script-source": "1.7.0", + "erubis": "2.7.0", + "execjs": "2.0.2", + "hike": "1.2.3", + "i18n": "0.6.9", + "jbuilder": "2.0.7", + "jquery-rails": "3.1.0", + "json": "1.8.1", + "kgio": "2.9.2", + "libv8": "3.16.14.3", + "mail": "2.5.4", + "mime-types": "1.25.1", + "minitest": "5.3.4", + "multi_json": "1.10.1", + "mysql2": "0.3.16", + "polyglot": "0.3.4", + "rack": "1.5.2", + "rack-test": "0.6.2", + "rails": "4.1.1", + "railties": "4.1.1", + "raindrops": "0.13.0", + "rake": "10.3.2", + "rdoc": "4.1.1", + "ref": "1.0.5", + "sass": "3.2.19", + "sass-rails": "4.0.3", + "sdoc": "0.4.0", + "spring": "1.1.3", + "sprockets": "2.11.0", + "sprockets-rails": "2.1.3", + "sqlite3": "1.3.9", + "therubyracer": "0.12.1", + "thor": "0.19.1", + "thread_safe": "0.3.3", + "tilt": "1.4.1", + "treetop": "1.4.15", + "turbolinks": "2.2.2", + "tzinfo": "1.2.0", + "uglifier": "2.5.0", + "unicorn": "4.8.3", + } + fixture, err := os.Open("test-fixtures/Gemfile.lock") if err != nil { t.Fatalf("failed to open fixture: %+v", err) @@ -72,15 +73,15 @@ func TestParseGemfileLockEntries(t *testing.T) { t.Fatalf("failed to parse gemfile lock: %+v", err) } - if len(actual) != len(expected) { + if len(actual) != len(expectedGems) { for _, a := range actual { t.Log(" ", a) } - t.Fatalf("unexpected package count: %d!=%d", len(actual), len(expected)) + t.Fatalf("unexpected package count: %d!=%d", len(actual), len(expectedGems)) } for _, a := range actual { - expectedVersion, ok := expected[a.Name] + expectedVersion, ok := expectedGems[a.Name] if !ok { t.Errorf("unexpected package found: %s", a.Name) } diff --git a/syft/cataloger/bundler/parse_gemspec.go b/syft/cataloger/bundler/parse_gemspec.go new file mode 100644 index 000000000..91cfa8c01 --- /dev/null +++ b/syft/cataloger/bundler/parse_gemspec.go @@ -0,0 +1,125 @@ +package bundler + +import ( + "bufio" + "fmt" + "io" + "regexp" + "strings" + + "github.com/mitchellh/mapstructure" + + "github.com/anchore/syft/syft/cataloger/common" + "github.com/anchore/syft/syft/pkg" +) + +// integrity check +var _ common.ParserFn = parseGemfileLockEntries + +// for line in gem.splitlines(): +// line = line.strip() +// line = re.sub(r"\.freeze", "", line) + +// # look for the unicode \u{} format and try to convert to something python can use +// patt = re.match(r".*\.homepage *= *(.*) *", line) +// if patt: +// sourcepkg = json.loads(patt.group(1)) + +// patt = re.match(r".*\.licenses *= *(.*) *", line) +// if patt: +// lstr = re.sub(r"^\[|\]$", "", patt.group(1)).split(',') +// for thestr in lstr: +// thestr = re.sub(' *" *', "", thestr) +// lics.append(thestr) + +// patt = re.match(r".*\.authors *= *(.*) *", line) +// if patt: +// lstr = re.sub(r"^\[|\]$", "", patt.group(1)).split(',') +// for thestr in lstr: +// thestr = re.sub(' *" *', "", thestr) +// origins.append(thestr) + +// patt = re.match(r".*\.files *= *(.*) *", line) +// if patt: +// lstr = re.sub(r"^\[|\]$", "", patt.group(1)).split(',') +// for thestr in lstr: +// thestr = re.sub(' *" *', "", thestr) +// rfiles.append(thestr) + +type listProcessor func(string) []string + +var patterns = map[string]*regexp.Regexp{ + // match example: name = "railties".freeze ---> railties + "name": regexp.MustCompile(`.*\.name\s*=\s*["']{1}(?P.*)["']{1} *`), + // match example: version = "1.0.4".freeze ---> 1.0.4 + "version": regexp.MustCompile(`.*\.version\s*=\s*["']{1}(?P.*)["']{1} *`), + // match example: homepage = "https://github.com/anchore/syft".freeze ---> https://github.com/anchore/syft + "homepage": regexp.MustCompile(`.*\.homepage\s*=\s*["']{1}(?P.*)["']{1} *`), + // TODO: add more fields +} + +// TODO: use post processors for lists +var postProcessors = map[string]listProcessor{ + //"files": func(s string) []string { + // + //}, +} + +func parseGemspecEntries(_ string, reader io.Reader) ([]pkg.Package, error) { + var pkgs []pkg.Package + var fields = make(map[string]interface{}) + scanner := bufio.NewScanner(reader) + + for scanner.Scan() { + line := scanner.Text() + + // TODO: sanitize unicode? (see engine code) + sanitizedLine := strings.TrimSpace(line) + + if sanitizedLine == "" { + continue + } + + for field, pattern := range patterns { + matchMap := matchCaptureGroups(pattern, sanitizedLine) + if value := matchMap[field]; value != "" { + if postProcessor := postProcessors[field]; postProcessor != nil { + fields[field] = postProcessor(value) + } else { + fields[field] = value + } + // TODO: know that a line could actually match on multiple patterns, this is unlikely though + break + } + } + } + + if fields["name"] != "" && fields["version"] != "" { + var metadata pkg.GemMetadata + if err := mapstructure.Decode(fields, &metadata); err != nil { + return nil, fmt.Errorf("unable to decode gem metadata: %w", err) + } + + pkgs = append(pkgs, pkg.Package{ + Name: metadata.Name, + Version: metadata.Version, + Language: pkg.Ruby, + Type: pkg.BundlerPkg, + Metadata: metadata, + }) + } + + return pkgs, nil +} + +// matchCaptureGroups takes a regular expression and string and returns all of the named capture group results in a map. +func matchCaptureGroups(regEx *regexp.Regexp, str string) map[string]string { + match := regEx.FindStringSubmatch(str) + results := make(map[string]string) + for i, name := range regEx.SubexpNames() { + if i > 0 && i <= len(match) { + results[name] = match[i] + } + } + return results +} diff --git a/syft/cataloger/bundler/parse_gemspec_test.go b/syft/cataloger/bundler/parse_gemspec_test.go new file mode 100644 index 000000000..f05bb982b --- /dev/null +++ b/syft/cataloger/bundler/parse_gemspec_test.go @@ -0,0 +1,50 @@ +package bundler + +import ( + "os" + "testing" + + "github.com/anchore/syft/syft/pkg" +) + +func TestParseGemspec(t *testing.T) { + var expectedGems = map[string]string{ + "bundler": "2.1.4", + } + + fixture, err := os.Open("test-fixtures/bundler.gemspec") + if err != nil { + t.Fatalf("failed to open fixture: %+v", err) + } + + actual, err := parseGemspecEntries(fixture.Name(), fixture) + if err != nil { + t.Fatalf("failed to parse gemspec: %+v", err) + } + + if len(actual) != len(expectedGems) { + for _, a := range actual { + t.Log(" ", a) + } + t.Fatalf("unexpected package count: %d!=%d", len(actual), len(expectedGems)) + } + + for _, a := range actual { + expectedVersion, ok := expectedGems[a.Name] + if !ok { + t.Errorf("unexpected package found: %s", a.Name) + } + + if expectedVersion != a.Version { + t.Errorf("unexpected package version (pkg=%s): %s", a.Name, a.Version) + } + + if a.Language != pkg.Ruby { + t.Errorf("bad language (pkg=%+v): %+v", a.Name, a.Language) + } + + if a.Type != pkg.BundlerPkg { + t.Errorf("bad package type (pkg=%+v): %+v", a.Name, a.Type) + } + } +} diff --git a/syft/cataloger/bundler/test-fixtures/bundler.gemspec b/syft/cataloger/bundler/test-fixtures/bundler.gemspec new file mode 100644 index 000000000..450b81096 --- /dev/null +++ b/syft/cataloger/bundler/test-fixtures/bundler.gemspec @@ -0,0 +1,25 @@ +# frozen_string_literal: true +# -*- encoding: utf-8 -*- +# stub: bundler 2.1.4 ruby lib + +Gem::Specification.new do |s| + s.name = "bundler".freeze + s.version = "2.1.4" + + s.required_rubygems_version = Gem::Requirement.new(">= 2.5.2".freeze) if s.respond_to? :required_rubygems_version= + s.require_paths = ["lib".freeze] + s.authors = ["Andr\u00E9 Arko".freeze, "Samuel Giddins".freeze, "Colby Swandale".freeze, "Hiroshi Shibata".freeze, "David Rodr\u00EDguez".freeze, "Grey Baker".f + s.bindir = "exe".freeze + s.date = "2020-01-05" + s.description = "Bundler manages an application's dependencies through its entire life, across many machines, systematically and repeatably".freeze + s.email = ["team@bundler.io".freeze] + s.executables = ["bundle".freeze, "bundler".freeze] + s.files = ["exe/bundle".freeze, "exe/bundler".freeze] + s.homepage = "https://bundler.io".freeze + s.licenses = ["MIT".freeze] + s.required_ruby_version = Gem::Requirement.new(">= 2.3.0".freeze) + s.rubygems_version = "3.1.2".freeze + s.summary = "The best way to manage your application's dependencies".freeze + + s.installed_by_version = "3.1.2" if s.respond_to? :installed_by_version + end \ No newline at end of file diff --git a/syft/lib.go b/syft/lib.go index 7d7a7da88..e766247fb 100644 --- a/syft/lib.go +++ b/syft/lib.go @@ -62,6 +62,16 @@ func IdentifyDistro(s scope.Scope) distro.Distro { // Catalog the given scope, which may represent a container image or filesystem. Returns the discovered set of packages. func CatalogFromScope(s scope.Scope) (*pkg.Catalog, error) { log.Info("building the catalog") + + // conditionally have two sets of catalogers + //var catalogers []cataloger.Cataloger + //// if image + //// use one set of catalogers + //catalogers = ... + // + //// if dir + //// use another set of catalogers + return cataloger.Catalog(s.Resolver, cataloger.All()...) } diff --git a/syft/pkg/gem_metadata.go b/syft/pkg/gem_metadata.go new file mode 100644 index 000000000..164f6b007 --- /dev/null +++ b/syft/pkg/gem_metadata.go @@ -0,0 +1,7 @@ +package pkg + +type GemMetadata struct { + Name string `mapstructure:"name" json:"name"` + Version string `mapstructure:"version" json:"version"` + // TODO: add more fields from the gemspec +}