port ruby cataloger to new generic cataloger pattern (#1322)

Signed-off-by: Alex Goodman <alex.goodman@anchore.com>

Signed-off-by: Alex Goodman <alex.goodman@anchore.com>
This commit is contained in:
Alex Goodman 2022-11-04 11:33:55 -04:00 committed by GitHub
parent 3048382bbd
commit 41464bbd7f
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
8 changed files with 182 additions and 177 deletions

View File

@ -4,23 +4,17 @@ Package ruby bundler provides a concrete Cataloger implementation for Ruby Gemfi
package ruby
import (
"github.com/anchore/syft/syft/pkg/cataloger/common"
"github.com/anchore/syft/syft/pkg/cataloger/generic"
)
// NewGemFileLockCataloger returns a new Bundler cataloger object tailored for parsing index-oriented files (e.g. Gemfile.lock).
func NewGemFileLockCataloger() *common.GenericCataloger {
globParsers := map[string]common.ParserFn{
"**/Gemfile.lock": parseGemFileLockEntries,
}
return common.NewGenericCataloger(nil, globParsers, "ruby-gemfile-cataloger")
func NewGemFileLockCataloger() *generic.Cataloger {
return generic.NewCataloger("ruby-gemfile-cataloger").
WithParserByGlobs(parseGemFileLockEntries, "**/Gemfile.lock")
}
// NewGemSpecCataloger returns a new Bundler cataloger object tailored for detecting installations of gems (e.g. Gemspec).
func NewGemSpecCataloger() *common.GenericCataloger {
globParsers := map[string]common.ParserFn{
"**/specifications/**/*.gemspec": parseGemSpecEntries,
}
return common.NewGenericCataloger(nil, globParsers, "ruby-gemspec-cataloger")
func NewGemSpecCataloger() *generic.Cataloger {
return generic.NewCataloger("ruby-gemspec-cataloger").
WithParserByGlobs(parseGemSpecEntries, "**/specifications/**/*.gemspec")
}

View File

@ -0,0 +1,53 @@
package ruby
import (
"github.com/anchore/packageurl-go"
"github.com/anchore/syft/syft/pkg"
"github.com/anchore/syft/syft/source"
)
func newGemfileLockPackage(name, version string, locations ...source.Location) pkg.Package {
p := pkg.Package{
Name: name,
Version: version,
PURL: packageURL(name, version),
Locations: source.NewLocationSet(locations...),
Language: pkg.Ruby,
Type: pkg.GemPkg,
}
p.SetID()
return p
}
func newGemspecPackage(m pkg.GemMetadata, locations ...source.Location) pkg.Package {
p := pkg.Package{
Name: m.Name,
Version: m.Version,
Locations: source.NewLocationSet(locations...),
PURL: packageURL(m.Name, m.Version),
Licenses: m.Licenses,
Language: pkg.Ruby,
Type: pkg.GemPkg,
MetadataType: pkg.GemMetadataType,
Metadata: m,
}
p.SetID()
return p
}
func packageURL(name, version string) string {
var qualifiers packageurl.Qualifiers
return packageurl.NewPackageURL(
packageurl.TypeGem,
"",
name,
version,
qualifiers,
"",
).ToString()
}

View File

@ -0,0 +1,31 @@
package ruby
import (
"fmt"
"testing"
"github.com/google/go-cmp/cmp"
)
func Test_packageURL(t *testing.T) {
tests := []struct {
name string
version string
expected string
}{
{
name: "p",
version: "v",
expected: "pkg:gem/p@v",
},
}
for _, test := range tests {
t.Run(fmt.Sprintf("%s@%s", test.name, test.version), func(t *testing.T) {
actual := packageURL(test.name, test.version)
if diff := cmp.Diff(test.expected, actual); diff != "" {
t.Errorf("unexpected packageURL (-want +got):\n%s", diff)
}
})
}
}

View File

@ -2,23 +2,22 @@ package ruby
import (
"bufio"
"io"
"strings"
"github.com/anchore/syft/internal"
"github.com/anchore/syft/syft/artifact"
"github.com/anchore/syft/syft/pkg"
"github.com/anchore/syft/syft/pkg/cataloger/common"
"github.com/anchore/syft/syft/pkg/cataloger/generic"
"github.com/anchore/syft/syft/source"
)
// integrity check
var _ common.ParserFn = parseGemFileLockEntries
var _ generic.Parser = parseGemFileLockEntries
var sectionsOfInterest = internal.NewStringSet("GEM")
// parseGemFileLockEntries is a parser function for Gemfile.lock contents, returning all Gems discovered.
func parseGemFileLockEntries(_ string, reader io.Reader) ([]*pkg.Package, []artifact.Relationship, error) {
pkgs := make([]*pkg.Package, 0)
func parseGemFileLockEntries(_ source.FileResolver, _ *generic.Environment, reader source.LocationReadCloser) ([]pkg.Package, []artifact.Relationship, error) {
var pkgs []pkg.Package
scanner := bufio.NewScanner(reader)
var currentSection string
@ -41,12 +40,13 @@ func parseGemFileLockEntries(_ string, reader io.Reader) ([]*pkg.Package, []arti
if len(candidate) != 2 {
continue
}
pkgs = append(pkgs, &pkg.Package{
Name: candidate[0],
Version: strings.Trim(candidate[1], "()"),
Language: pkg.Ruby,
Type: pkg.GemPkg,
})
pkgs = append(pkgs,
newGemfileLockPackage(
candidate[0],
strings.Trim(candidate[1], "()"),
reader.Location,
),
)
}
}
if err := scanner.Err(); err != nil {

View File

@ -1,102 +1,69 @@
package ruby
import (
"os"
"testing"
"github.com/anchore/syft/syft/pkg"
"github.com/anchore/syft/syft/pkg/cataloger/internal/pkgtest"
"github.com/anchore/syft/syft/source"
)
func TestParseGemfileLockEntries(t *testing.T) {
var expectedGems = map[string]string{
"actionmailer": "4.1.1",
"actionpack": "4.1.1",
"actionview": "4.1.1",
"activemodel": "4.1.1",
"activerecord": "4.1.1",
"activesupport": "4.1.1",
"arel": "5.0.1.20140414130214",
"bootstrap-sass": "3.1.1.1",
"builder": "3.2.2",
"coffee-rails": "4.0.1",
"coffee-script": "2.2.0",
"coffee-script-source": "1.7.0",
"erubis": "2.7.0",
"execjs": "2.0.2",
"hike": "1.2.3",
"i18n": "0.6.9",
"jbuilder": "2.0.7",
"jquery-rails": "3.1.0",
"json": "1.8.1",
"kgio": "2.9.2",
"libv8": "3.16.14.3",
"mail": "2.5.4",
"mime-types": "1.25.1",
"minitest": "5.3.4",
"multi_json": "1.10.1",
"mysql2": "0.3.16",
"polyglot": "0.3.4",
"rack": "1.5.2",
"rack-test": "0.6.2",
"rails": "4.1.1",
"railties": "4.1.1",
"raindrops": "0.13.0",
"rake": "10.3.2",
"rdoc": "4.1.1",
"ref": "1.0.5",
"sass": "3.2.19",
"sass-rails": "4.0.3",
"sdoc": "0.4.0",
"spring": "1.1.3",
"sprockets": "2.11.0",
"sprockets-rails": "2.1.3",
"sqlite3": "1.3.9",
"therubyracer": "0.12.1",
"thor": "0.19.1",
"thread_safe": "0.3.3",
"tilt": "1.4.1",
"treetop": "1.4.15",
"turbolinks": "2.2.2",
"tzinfo": "1.2.0",
"uglifier": "2.5.0",
"unicorn": "4.8.3",
fixture := "test-fixtures/Gemfile.lock"
locations := source.NewLocationSet(source.NewLocation(fixture))
var expectedPkgs = []pkg.Package{
{Name: "actionmailer", Version: "4.1.1", PURL: "pkg:gem/actionmailer@4.1.1", Locations: locations, Language: pkg.Ruby, Type: pkg.GemPkg},
{Name: "actionpack", Version: "4.1.1", PURL: "pkg:gem/actionpack@4.1.1", Locations: locations, Language: pkg.Ruby, Type: pkg.GemPkg},
{Name: "actionview", Version: "4.1.1", PURL: "pkg:gem/actionview@4.1.1", Locations: locations, Language: pkg.Ruby, Type: pkg.GemPkg},
{Name: "activemodel", Version: "4.1.1", PURL: "pkg:gem/activemodel@4.1.1", Locations: locations, Language: pkg.Ruby, Type: pkg.GemPkg},
{Name: "activerecord", Version: "4.1.1", PURL: "pkg:gem/activerecord@4.1.1", Locations: locations, Language: pkg.Ruby, Type: pkg.GemPkg},
{Name: "activesupport", Version: "4.1.1", PURL: "pkg:gem/activesupport@4.1.1", Locations: locations, Language: pkg.Ruby, Type: pkg.GemPkg},
{Name: "arel", Version: "5.0.1.20140414130214", PURL: "pkg:gem/arel@5.0.1.20140414130214", Locations: locations, Language: pkg.Ruby, Type: pkg.GemPkg},
{Name: "bootstrap-sass", Version: "3.1.1.1", PURL: "pkg:gem/bootstrap-sass@3.1.1.1", Locations: locations, Language: pkg.Ruby, Type: pkg.GemPkg},
{Name: "builder", Version: "3.2.2", PURL: "pkg:gem/builder@3.2.2", Locations: locations, Language: pkg.Ruby, Type: pkg.GemPkg},
{Name: "coffee-rails", Version: "4.0.1", PURL: "pkg:gem/coffee-rails@4.0.1", Locations: locations, Language: pkg.Ruby, Type: pkg.GemPkg},
{Name: "coffee-script", Version: "2.2.0", PURL: "pkg:gem/coffee-script@2.2.0", Locations: locations, Language: pkg.Ruby, Type: pkg.GemPkg},
{Name: "coffee-script-source", Version: "1.7.0", PURL: "pkg:gem/coffee-script-source@1.7.0", Locations: locations, Language: pkg.Ruby, Type: pkg.GemPkg},
{Name: "erubis", Version: "2.7.0", PURL: "pkg:gem/erubis@2.7.0", Locations: locations, Language: pkg.Ruby, Type: pkg.GemPkg},
{Name: "execjs", Version: "2.0.2", PURL: "pkg:gem/execjs@2.0.2", Locations: locations, Language: pkg.Ruby, Type: pkg.GemPkg},
{Name: "hike", Version: "1.2.3", PURL: "pkg:gem/hike@1.2.3", Locations: locations, Language: pkg.Ruby, Type: pkg.GemPkg},
{Name: "i18n", Version: "0.6.9", PURL: "pkg:gem/i18n@0.6.9", Locations: locations, Language: pkg.Ruby, Type: pkg.GemPkg},
{Name: "jbuilder", Version: "2.0.7", PURL: "pkg:gem/jbuilder@2.0.7", Locations: locations, Language: pkg.Ruby, Type: pkg.GemPkg},
{Name: "jquery-rails", Version: "3.1.0", PURL: "pkg:gem/jquery-rails@3.1.0", Locations: locations, Language: pkg.Ruby, Type: pkg.GemPkg},
{Name: "json", Version: "1.8.1", PURL: "pkg:gem/json@1.8.1", Locations: locations, Language: pkg.Ruby, Type: pkg.GemPkg},
{Name: "kgio", Version: "2.9.2", PURL: "pkg:gem/kgio@2.9.2", Locations: locations, Language: pkg.Ruby, Type: pkg.GemPkg},
{Name: "libv8", Version: "3.16.14.3", PURL: "pkg:gem/libv8@3.16.14.3", Locations: locations, Language: pkg.Ruby, Type: pkg.GemPkg},
{Name: "mail", Version: "2.5.4", PURL: "pkg:gem/mail@2.5.4", Locations: locations, Language: pkg.Ruby, Type: pkg.GemPkg},
{Name: "mime-types", Version: "1.25.1", PURL: "pkg:gem/mime-types@1.25.1", Locations: locations, Language: pkg.Ruby, Type: pkg.GemPkg},
{Name: "minitest", Version: "5.3.4", PURL: "pkg:gem/minitest@5.3.4", Locations: locations, Language: pkg.Ruby, Type: pkg.GemPkg},
{Name: "multi_json", Version: "1.10.1", PURL: "pkg:gem/multi_json@1.10.1", Locations: locations, Language: pkg.Ruby, Type: pkg.GemPkg},
{Name: "mysql2", Version: "0.3.16", PURL: "pkg:gem/mysql2@0.3.16", Locations: locations, Language: pkg.Ruby, Type: pkg.GemPkg},
{Name: "polyglot", Version: "0.3.4", PURL: "pkg:gem/polyglot@0.3.4", Locations: locations, Language: pkg.Ruby, Type: pkg.GemPkg},
{Name: "rack", Version: "1.5.2", PURL: "pkg:gem/rack@1.5.2", Locations: locations, Language: pkg.Ruby, Type: pkg.GemPkg},
{Name: "rack-test", Version: "0.6.2", PURL: "pkg:gem/rack-test@0.6.2", Locations: locations, Language: pkg.Ruby, Type: pkg.GemPkg},
{Name: "rails", Version: "4.1.1", PURL: "pkg:gem/rails@4.1.1", Locations: locations, Language: pkg.Ruby, Type: pkg.GemPkg},
{Name: "railties", Version: "4.1.1", PURL: "pkg:gem/railties@4.1.1", Locations: locations, Language: pkg.Ruby, Type: pkg.GemPkg},
{Name: "raindrops", Version: "0.13.0", PURL: "pkg:gem/raindrops@0.13.0", Locations: locations, Language: pkg.Ruby, Type: pkg.GemPkg},
{Name: "rake", Version: "10.3.2", PURL: "pkg:gem/rake@10.3.2", Locations: locations, Language: pkg.Ruby, Type: pkg.GemPkg},
{Name: "rdoc", Version: "4.1.1", PURL: "pkg:gem/rdoc@4.1.1", Locations: locations, Language: pkg.Ruby, Type: pkg.GemPkg},
{Name: "ref", Version: "1.0.5", PURL: "pkg:gem/ref@1.0.5", Locations: locations, Language: pkg.Ruby, Type: pkg.GemPkg},
{Name: "sass", Version: "3.2.19", PURL: "pkg:gem/sass@3.2.19", Locations: locations, Language: pkg.Ruby, Type: pkg.GemPkg},
{Name: "sass-rails", Version: "4.0.3", PURL: "pkg:gem/sass-rails@4.0.3", Locations: locations, Language: pkg.Ruby, Type: pkg.GemPkg},
{Name: "sdoc", Version: "0.4.0", PURL: "pkg:gem/sdoc@0.4.0", Locations: locations, Language: pkg.Ruby, Type: pkg.GemPkg},
{Name: "spring", Version: "1.1.3", PURL: "pkg:gem/spring@1.1.3", Locations: locations, Language: pkg.Ruby, Type: pkg.GemPkg},
{Name: "sprockets", Version: "2.11.0", PURL: "pkg:gem/sprockets@2.11.0", Locations: locations, Language: pkg.Ruby, Type: pkg.GemPkg},
{Name: "sprockets-rails", Version: "2.1.3", PURL: "pkg:gem/sprockets-rails@2.1.3", Locations: locations, Language: pkg.Ruby, Type: pkg.GemPkg},
{Name: "sqlite3", Version: "1.3.9", PURL: "pkg:gem/sqlite3@1.3.9", Locations: locations, Language: pkg.Ruby, Type: pkg.GemPkg},
{Name: "therubyracer", Version: "0.12.1", PURL: "pkg:gem/therubyracer@0.12.1", Locations: locations, Language: pkg.Ruby, Type: pkg.GemPkg},
{Name: "thor", Version: "0.19.1", PURL: "pkg:gem/thor@0.19.1", Locations: locations, Language: pkg.Ruby, Type: pkg.GemPkg},
{Name: "thread_safe", Version: "0.3.3", PURL: "pkg:gem/thread_safe@0.3.3", Locations: locations, Language: pkg.Ruby, Type: pkg.GemPkg},
{Name: "tilt", Version: "1.4.1", PURL: "pkg:gem/tilt@1.4.1", Locations: locations, Language: pkg.Ruby, Type: pkg.GemPkg},
{Name: "treetop", Version: "1.4.15", PURL: "pkg:gem/treetop@1.4.15", Locations: locations, Language: pkg.Ruby, Type: pkg.GemPkg},
{Name: "turbolinks", Version: "2.2.2", PURL: "pkg:gem/turbolinks@2.2.2", Locations: locations, Language: pkg.Ruby, Type: pkg.GemPkg},
{Name: "tzinfo", Version: "1.2.0", PURL: "pkg:gem/tzinfo@1.2.0", Locations: locations, Language: pkg.Ruby, Type: pkg.GemPkg},
{Name: "uglifier", Version: "2.5.0", PURL: "pkg:gem/uglifier@2.5.0", Locations: locations, Language: pkg.Ruby, Type: pkg.GemPkg},
{Name: "unicorn", Version: "4.8.3", PURL: "pkg:gem/unicorn@4.8.3", Locations: locations, Language: pkg.Ruby, Type: pkg.GemPkg},
}
fixture, err := os.Open("test-fixtures/Gemfile.lock")
if err != nil {
t.Fatalf("failed to open fixture: %+v", err)
}
// TODO: no relationships are under test yet
actual, _, err := parseGemFileLockEntries(fixture.Name(), fixture)
if err != nil {
t.Fatalf("failed to parse gemfile lock: %+v", err)
}
if len(actual) != len(expectedGems) {
for _, a := range actual {
t.Log(" ", a)
}
t.Fatalf("unexpected package count: %d!=%d", len(actual), len(expectedGems))
}
for _, a := range actual {
expectedVersion, ok := expectedGems[a.Name]
if !ok {
t.Errorf("unexpected package found: %s", a.Name)
}
if expectedVersion != a.Version {
t.Errorf("unexpected package version (pkg=%s): %s", a.Name, a.Version)
}
if a.Language != pkg.Ruby {
t.Errorf("bad language (pkg=%+v): %+v", a.Name, a.Language)
}
if a.Type != pkg.GemPkg {
t.Errorf("bad package type (pkg=%+v): %+v", a.Name, a.Type)
}
}
pkgtest.TestFileParser(t, fixture, parseGemFileLockEntries, expectedPkgs, nil)
}

View File

@ -4,7 +4,6 @@ import (
"bufio"
"encoding/json"
"fmt"
"io"
"regexp"
"strings"
@ -13,11 +12,11 @@ import (
"github.com/anchore/syft/internal"
"github.com/anchore/syft/syft/artifact"
"github.com/anchore/syft/syft/pkg"
"github.com/anchore/syft/syft/pkg/cataloger/common"
"github.com/anchore/syft/syft/pkg/cataloger/generic"
"github.com/anchore/syft/syft/source"
)
// integrity check
var _ common.ParserFn = parseGemFileLockEntries
var _ generic.Parser = parseGemFileLockEntries
type postProcessor func(string) []string
@ -36,14 +35,14 @@ var patterns = map[string]*regexp.Regexp{
"homepage": regexp.MustCompile(`.*\.homepage\s*=\s*["']{1}(?P<homepage>.*)["']{1} *`),
// match example: files = ["exe/bundle".freeze, "exe/bundler".freeze] ---> "exe/bundle".freeze, "exe/bundler".freeze
"files": regexp.MustCompile(`.*\.files\s*=\s*\[(?P<files>.*)\] *`),
"files": regexp.MustCompile(`.*\.files\s*=\s*\[(?P<files>.*)] *`),
// match example: authors = ["Andr\u00E9 Arko".freeze, "Samuel Giddins".freeze, "Colby Swandale".freeze,
// "Hiroshi Shibata".freeze, "David Rodr\u00EDguez".freeze, "Grey Baker".freeze...]
"authors": regexp.MustCompile(`.*\.authors\s*=\s*\[(?P<authors>.*)\] *`),
"authors": regexp.MustCompile(`.*\.authors\s*=\s*\[(?P<authors>.*)] *`),
// match example: licenses = ["MIT".freeze] ----> "MIT".freeze
"licenses": regexp.MustCompile(`.*\.licenses\s*=\s*\[(?P<licenses>.*)\] *`),
"licenses": regexp.MustCompile(`.*\.licenses\s*=\s*\[(?P<licenses>.*)] *`),
}
var postProcessors = map[string]postProcessor{
@ -60,8 +59,8 @@ func processList(s string) []string {
return results
}
func parseGemSpecEntries(_ string, reader io.Reader) ([]*pkg.Package, []artifact.Relationship, error) {
var pkgs []*pkg.Package
func parseGemSpecEntries(_ source.FileResolver, _ *generic.Environment, reader source.LocationReadCloser) ([]pkg.Package, []artifact.Relationship, error) {
var pkgs []pkg.Package
var fields = make(map[string]interface{})
scanner := bufio.NewScanner(reader)
@ -79,8 +78,8 @@ func parseGemSpecEntries(_ string, reader io.Reader) ([]*pkg.Package, []artifact
for field, pattern := range patterns {
matchMap := internal.MatchNamedCaptureGroups(pattern, sanitizedLine)
if value := matchMap[field]; value != "" {
if postProcessor := postProcessors[field]; postProcessor != nil {
fields[field] = postProcessor(value)
if pp := postProcessors[field]; pp != nil {
fields[field] = pp(value)
} else {
fields[field] = value
}
@ -96,15 +95,7 @@ func parseGemSpecEntries(_ string, reader io.Reader) ([]*pkg.Package, []artifact
return nil, nil, fmt.Errorf("unable to decode gem metadata: %w", err)
}
pkgs = append(pkgs, &pkg.Package{
Name: metadata.Name,
Version: metadata.Version,
Licenses: metadata.Licenses,
Language: pkg.Ruby,
Type: pkg.GemPkg,
MetadataType: pkg.GemMetadataType,
Metadata: metadata,
})
pkgs = append(pkgs, newGemspecPackage(metadata, reader.Location))
}
return pkgs, nil, nil

View File

@ -1,18 +1,23 @@
package ruby
import (
"os"
"testing"
"github.com/go-test/deep"
"github.com/anchore/syft/syft/pkg"
"github.com/anchore/syft/syft/pkg/cataloger/internal/pkgtest"
"github.com/anchore/syft/syft/source"
)
func TestParseGemspec(t *testing.T) {
fixture := "test-fixtures/bundler.gemspec"
locations := source.NewLocationSet(source.NewLocation(fixture))
var expectedPkg = pkg.Package{
Name: "bundler",
Version: "2.1.4",
PURL: "pkg:gem/bundler@2.1.4",
Locations: locations,
Type: pkg.GemPkg,
Licenses: []string{"MIT"},
Language: pkg.Ruby,
@ -27,25 +32,5 @@ func TestParseGemspec(t *testing.T) {
},
}
fixture, err := os.Open("test-fixtures/bundler.gemspec")
if err != nil {
t.Fatalf("failed to open fixture: %+v", err)
}
// TODO: no relationships are under test yet
actual, _, err := parseGemSpecEntries(fixture.Name(), fixture)
if err != nil {
t.Fatalf("failed to parse gemspec: %+v", err)
}
if len(actual) != 1 {
for _, a := range actual {
t.Log(" ", a)
}
t.Fatalf("unexpected package count: %d!=1", len(actual))
}
for _, d := range deep.Equal(actual[0], &expectedPkg) {
t.Errorf("diff: %+v", d)
}
pkgtest.TestFileParser(t, fixture, parseGemSpecEntries, []pkg.Package{expectedPkg}, nil)
}

View File

@ -17,24 +17,6 @@ func TestPackageURL(t *testing.T) {
distro *linux.Release
expected string
}{
{
name: "gem",
pkg: Package{
Name: "name",
Version: "v0.1.0",
Type: GemPkg,
},
expected: "pkg:gem/name@v0.1.0",
},
{
name: "npm",
pkg: Package{
Name: "name",
Version: "v0.1.0",
Type: NpmPkg,
},
expected: "pkg:npm/name@v0.1.0",
},
{
name: "cargo",
pkg: Package{
@ -109,6 +91,8 @@ func TestPackageURL(t *testing.T) {
expectedTypes.Remove(string(PhpComposerPkg))
expectedTypes.Remove(string(PythonPkg))
expectedTypes.Remove(string(RpmPkg))
expectedTypes.Remove(string(GemPkg))
expectedTypes.Remove(string(NpmPkg))
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {