From 1f0f6fa3e5fc6e48e448ed8e3d7fdf504e94c1f8 Mon Sep 17 00:00:00 2001 From: Alex Goodman Date: Tue, 6 Oct 2020 14:02:04 -0400 Subject: [PATCH] more gemspec tests Signed-off-by: Alex Goodman --- syft/cataloger/bundler/parse_gemspec.go | 70 +++++++++---------- syft/cataloger/bundler/parse_gemspec_test.go | 39 +++++------ .../bundler/test-fixtures/bundler.gemspec | 49 ++++++------- syft/pkg/gem_metadata.go | 8 ++- 4 files changed, 80 insertions(+), 86 deletions(-) diff --git a/syft/cataloger/bundler/parse_gemspec.go b/syft/cataloger/bundler/parse_gemspec.go index 90ab8766e..00a3f5c70 100644 --- a/syft/cataloger/bundler/parse_gemspec.go +++ b/syft/cataloger/bundler/parse_gemspec.go @@ -16,53 +16,42 @@ import ( // integrity check var _ common.ParserFn = parseGemFileLockEntries -// for line in gem.splitlines(): -// line = line.strip() -// line = re.sub(r"\.freeze", "", line) - -// # look for the unicode \u{} format and try to convert to something python can use -// patt = re.match(r".*\.homepage *= *(.*) *", line) -// if patt: -// sourcepkg = json.loads(patt.group(1)) - -// patt = re.match(r".*\.licenses *= *(.*) *", line) -// if patt: -// lstr = re.sub(r"^\[|\]$", "", patt.group(1)).split(',') -// for thestr in lstr: -// thestr = re.sub(' *" *', "", thestr) -// lics.append(thestr) - -// patt = re.match(r".*\.authors *= *(.*) *", line) -// if patt: -// lstr = re.sub(r"^\[|\]$", "", patt.group(1)).split(',') -// for thestr in lstr: -// thestr = re.sub(' *" *', "", thestr) -// origins.append(thestr) - -// patt = re.match(r".*\.files *= *(.*) *", line) -// if patt: -// lstr = re.sub(r"^\[|\]$", "", patt.group(1)).split(',') -// for thestr in lstr: -// thestr = re.sub(' *" *', "", thestr) -// rfiles.append(thestr) - -type listProcessor func(string) []string +type postProcessor func(string) []string var patterns = map[string]*regexp.Regexp{ // match example: name = "railties".freeze ---> railties "name": regexp.MustCompile(`.*\.name\s*=\s*["']{1}(?P.*)["']{1} *`), + // match example: version = "1.0.4".freeze ---> 1.0.4 "version": regexp.MustCompile(`.*\.version\s*=\s*["']{1}(?P.*)["']{1} *`), - // match example: homepage = "https://github.com/anchore/syft".freeze ---> https://github.com/anchore/syft + + // match example: + // homepage = "https://github.com/anchore/syft".freeze ---> https://github.com/anchore/syft "homepage": regexp.MustCompile(`.*\.homepage\s*=\s*["']{1}(?P.*)["']{1} *`), - // TODO: add more fields + + // match example: files = ["exe/bundle".freeze, "exe/bundler".freeze] ---> "exe/bundle".freeze, "exe/bundler".freeze + "files": regexp.MustCompile(`.*\.files\s*=\s*\[(?P.*)\] *`), + + // match example: authors = ["Andr\u00E9 Arko".freeze, "Samuel Giddins".freeze, "Colby Swandale".freeze, + // "Hiroshi Shibata".freeze, "David Rodr\u00EDguez".freeze, "Grey Baker".freeze...] + "authors": regexp.MustCompile(`.*\.authors\s*=\s*\[(?P.*)\] *`), + + // match example: licenses = ["MIT".freeze] ----> "MIT".freeze + "licenses": regexp.MustCompile(`.*\.licenses\s*=\s*\[(?P.*)\] *`), } -// TODO: use post processors for lists -var postProcessors = map[string]listProcessor{ - //"files": func(s string) []string { - // - //}, +var postProcessors = map[string]postProcessor{ + "files": processList, + "authors": processList, + "licenses": processList, +} + +func processList(s string) []string { + var results []string + for _, item := range strings.Split(s, ",") { + results = append(results, strings.Trim(item, "\" ")) + } + return results } func parseGemSpecEntries(_ string, reader io.Reader) ([]pkg.Package, error) { @@ -75,12 +64,16 @@ func parseGemSpecEntries(_ string, reader io.Reader) ([]pkg.Package, error) { // TODO: sanitize unicode? (see engine code) sanitizedLine := strings.TrimSpace(line) + sanitizedLine = strings.ReplaceAll(sanitizedLine, ".freeze", "") if sanitizedLine == "" { continue } for field, pattern := range patterns { + if strings.Contains(sanitizedLine, "licenses") { + println("Found it.") + } matchMap := matchCaptureGroups(pattern, sanitizedLine) if value := matchMap[field]; value != "" { if postProcessor := postProcessors[field]; postProcessor != nil { @@ -103,6 +96,7 @@ func parseGemSpecEntries(_ string, reader io.Reader) ([]pkg.Package, error) { pkgs = append(pkgs, pkg.Package{ Name: metadata.Name, Version: metadata.Version, + Licenses: metadata.Licenses, Language: pkg.Ruby, Type: pkg.GemPkg, Metadata: metadata, diff --git a/syft/cataloger/bundler/parse_gemspec_test.go b/syft/cataloger/bundler/parse_gemspec_test.go index 8889899d2..11d1276f1 100644 --- a/syft/cataloger/bundler/parse_gemspec_test.go +++ b/syft/cataloger/bundler/parse_gemspec_test.go @@ -5,11 +5,23 @@ import ( "testing" "github.com/anchore/syft/syft/pkg" + "github.com/go-test/deep" ) func TestParseGemspec(t *testing.T) { - var expectedGems = map[string]string{ - "bundler": "2.1.4", + var expectedPkg = pkg.Package{ + Name: "bundler", + Version: "2.1.4", + Type: pkg.GemPkg, + Licenses: []string{"MIT"}, + Language: pkg.Ruby, + Metadata: pkg.GemMetadata{ + Name: "bundler", + Version: "2.1.4", + Files: []string{"exe/bundle", "exe/bundler"}, + Authors: []string{"André Arko", "Samuel Giddins", "Colby Swandale", "Hiroshi Shibata", "David Rodréguez", "Grey Baker", "Stephanie Morillo", "Chris Morris", "James Wen", "Tim Moore", "André Medeiros", "Jessica Lynn Suttles", "Terence Lee", "Carl Lerche", "Yehuda Katz"}, + Licenses: []string{"MIT"}, + }, } fixture, err := os.Open("test-fixtures/bundler.gemspec") @@ -22,29 +34,14 @@ func TestParseGemspec(t *testing.T) { t.Fatalf("failed to parse gemspec: %+v", err) } - if len(actual) != len(expectedGems) { + if len(actual) != 1 { for _, a := range actual { t.Log(" ", a) } - t.Fatalf("unexpected package count: %d!=%d", len(actual), len(expectedGems)) + t.Fatalf("unexpected package count: %d!=1", len(actual)) } - for _, a := range actual { - expectedVersion, ok := expectedGems[a.Name] - if !ok { - t.Errorf("unexpected package found: %s", a.Name) - } - - if expectedVersion != a.Version { - t.Errorf("unexpected package version (pkg=%s): %s", a.Name, a.Version) - } - - if a.Language != pkg.Ruby { - t.Errorf("bad language (pkg=%+v): %+v", a.Name, a.Language) - } - - if a.Type != pkg.GemPkg { - t.Errorf("bad package type (pkg=%+v): %+v", a.Name, a.Type) - } + for _, d := range deep.Equal(actual[0], expectedPkg) { + t.Errorf("diff: %+v", d) } } diff --git a/syft/cataloger/bundler/test-fixtures/bundler.gemspec b/syft/cataloger/bundler/test-fixtures/bundler.gemspec index 450b81096..a877840b3 100644 --- a/syft/cataloger/bundler/test-fixtures/bundler.gemspec +++ b/syft/cataloger/bundler/test-fixtures/bundler.gemspec @@ -1,25 +1,26 @@ # frozen_string_literal: true -# -*- encoding: utf-8 -*- -# stub: bundler 2.1.4 ruby lib - -Gem::Specification.new do |s| - s.name = "bundler".freeze - s.version = "2.1.4" - - s.required_rubygems_version = Gem::Requirement.new(">= 2.5.2".freeze) if s.respond_to? :required_rubygems_version= - s.require_paths = ["lib".freeze] - s.authors = ["Andr\u00E9 Arko".freeze, "Samuel Giddins".freeze, "Colby Swandale".freeze, "Hiroshi Shibata".freeze, "David Rodr\u00EDguez".freeze, "Grey Baker".f - s.bindir = "exe".freeze - s.date = "2020-01-05" - s.description = "Bundler manages an application's dependencies through its entire life, across many machines, systematically and repeatably".freeze - s.email = ["team@bundler.io".freeze] - s.executables = ["bundle".freeze, "bundler".freeze] - s.files = ["exe/bundle".freeze, "exe/bundler".freeze] - s.homepage = "https://bundler.io".freeze - s.licenses = ["MIT".freeze] - s.required_ruby_version = Gem::Requirement.new(">= 2.3.0".freeze) - s.rubygems_version = "3.1.2".freeze - s.summary = "The best way to manage your application's dependencies".freeze - - s.installed_by_version = "3.1.2" if s.respond_to? :installed_by_version - end \ No newline at end of file +# -*- encoding: utf-8 -*- +# stub: bundler 2.1.4 ruby lib + +Gem::Specification.new do |s| + s.name = "bundler".freeze + s.version = "2.1.4" + + s.required_rubygems_version = Gem::Requirement.new(">= 2.5.2".freeze) if s.respond_to? :required_rubygems_version= + s.metadata = { "bug_tracker_uri" => "https://github.com/bundler/bundler/issues", "changelog_uri" => "https://github.com/bundler/bundler/blob/master/CHANGELOG.md", "homepage_uri" => "https://bundler.io/", "source_code_uri" => "https://github.com/bundler/bundler/" } if s.respond_to? :metadata= + s.require_paths = ["lib".freeze] + s.authors = ["Andr\u00E9 Arko".freeze, "Samuel Giddins".freeze, "Colby Swandale".freeze, "Hiroshi Shibata".freeze, "David Rodr\u00EDguez".freeze, "Grey Baker".freeze, "Stephanie Morillo".freeze, "Chris Morris".freeze, "James Wen".freeze, "Tim Moore".freeze, "Andr\u00E9 Medeiros".freeze, "Jessica Lynn Suttles".freeze, "Terence Lee".freeze, "Carl Lerche".freeze, "Yehuda Katz".freeze] + s.bindir = "exe".freeze + s.date = "2020-01-05" + s.description = "Bundler manages an application's dependencies through its entire life, across many machines, systematically and repeatably".freeze + s.email = ["team@bundler.io".freeze] + s.executables = ["bundle".freeze, "bundler".freeze] + s.files = ["exe/bundle".freeze, "exe/bundler".freeze] + s.homepage = "https://bundler.io".freeze + s.licenses = ["MIT".freeze] + s.required_ruby_version = Gem::Requirement.new(">= 2.3.0".freeze) + s.rubygems_version = "3.1.2".freeze + s.summary = "The best way to manage your application's dependencies".freeze + + s.installed_by_version = "3.1.2" if s.respond_to? :installed_by_version +end \ No newline at end of file diff --git a/syft/pkg/gem_metadata.go b/syft/pkg/gem_metadata.go index 164f6b007..aa2edc310 100644 --- a/syft/pkg/gem_metadata.go +++ b/syft/pkg/gem_metadata.go @@ -1,7 +1,9 @@ package pkg type GemMetadata struct { - Name string `mapstructure:"name" json:"name"` - Version string `mapstructure:"version" json:"version"` - // TODO: add more fields from the gemspec + Name string `mapstructure:"name" json:"name"` + Version string `mapstructure:"version" json:"version"` + Files []string `mapstructure:"files" json:"files"` + Authors []string `mapstructure:"authors" json:"authors"` + Licenses []string `mapstructure:"licenses" json:"licenses"` }