oss/syft
2
0
Fork 0
mirror of https://github.com/anchore/syft.git synced 2026-11-03 16:06:36 +01:00
Code Issues Packages Projects Releases Wiki Activity

add gem analyzer

Browse Source
This commit is contained in:
Alex Goodman
2020-06-04 14:42:59 -04:00
parent 033c91a233
commit f33720eaa1
18 changed files with 815 additions and 109 deletions
Download Patch File Download Diff File Expand all files Collapse all files
imgbom/analyzer/bundler/analyzer.go
+90
View File
@@ -0,0 +1,90 @@
package bundler
import (
"io"
"strings"
"github.com/anchore/imgbom/imgbom/pkg"
"github.com/anchore/imgbom/internal/log"
"github.com/anchore/stereoscope/pkg/file"
"github.com/anchore/stereoscope/pkg/tree"
)
var parserDispatch = map[string]parserFn{
"*/Gemfile.lock": ParseGemfileLockEntries,
}
type parserFn func(io.Reader) ([]pkg.Package, error)
type Analyzer struct {
selectedFiles []file.Reference
parsers map[file.Reference]parserFn
}
func NewAnalyzer() *Analyzer {
return &Analyzer{
selectedFiles: make([]file.Reference, 0),
parsers: make(map[file.Reference]parserFn),
}
}
func (a *Analyzer) Name() string {
return "bundler-analyzer"
}
func (a *Analyzer) register(files []file.Reference, parser parserFn) {
a.selectedFiles = append(a.selectedFiles, files...)
for _, f := range files {
a.parsers[f] = parser
}
}
func (a *Analyzer) clear() {
a.selectedFiles = make([]file.Reference, 0)
a.parsers = make(map[file.Reference]parserFn)
}
func (a *Analyzer) SelectFiles(trees []tree.FileTreeReader) []file.Reference {
for _, tree := range trees {
for globPattern, parser := range parserDispatch {
fileMatches, err := tree.FilesByGlob(globPattern)
if err != nil {
log.Errorf("'%s' failed to find files by glob: %s", a.Name(), globPattern)
}
if fileMatches != nil {
a.register(fileMatches, parser)
}
}
}
return a.selectedFiles
}
func (a *Analyzer) Analyze(contents map[file.Reference]string) ([]pkg.Package, error) {
defer a.clear()
packages := make([]pkg.Package, 0)
for reference, parser := range a.parsers {
content, ok := contents[reference]
if !ok {
log.Errorf("analyzer '%s' file content missing: %+v", a.Name(), reference)
continue
}
entries, err := parser(strings.NewReader(content))
if err != nil {
log.Errorf("analyzer failed to parse entries (reference=%+v): %w", reference, err)
continue
}
for _, entry := range entries {
entry.FoundBy = a.Name()
entry.Source = []file.Reference{reference}
packages = append(packages, entry)
}
}
return packages, nil
}
imgbom/analyzer/bundler/parse_gemfile_lock.go
+57
View File
@@ -0,0 +1,57 @@
package bundler
import (
"bufio"
"io"
"strings"
"github.com/anchore/imgbom/imgbom/pkg"
"github.com/anchore/imgbom/internal"
)
var sectionsOfInterest = internal.NewStringSetFromSlice([]string{"GEM"})
func ParseGemfileLockEntries(reader io.Reader) ([]pkg.Package, error) {
pkgs := make([]pkg.Package, 0)
scanner := bufio.NewScanner(reader)
var currentSection string
for scanner.Scan() {
line := scanner.Text()
sanitizedLine := strings.TrimSpace(line)
if len(line) > 1 && line[0] != ' ' {
// start of section
currentSection = sanitizedLine
continue
} else if !sectionsOfInterest.Contains(currentSection) {
// skip this line, we're in the wrong section
continue
}
if isDependencyLine(line) {
candidate := strings.Fields(sanitizedLine)
if len(candidate) != 2 {
continue
}
pkgs = append(pkgs, pkg.Package{
Name: candidate[0],
Version: strings.Trim(candidate[1], "()"),
Language: pkg.Ruby,
Type: pkg.BundlerPkg,
})
}
}
if err := scanner.Err(); err != nil {
return nil, err
}
return pkgs, nil
}
func isDependencyLine(line string) bool {
if len(line) < 5 {
return false
}
return strings.Count(line[:5], " ") == 4
}
imgbom/analyzer/bundler/parse_gemfile_lock_test.go
+100
View File
@@ -0,0 +1,100 @@
package bundler
import (
"os"
"testing"
"github.com/anchore/imgbom/imgbom/pkg"
)
var expected = map[string]string{
"actionmailer": "4.1.1",
"actionpack": "4.1.1",
"actionview": "4.1.1",
"activemodel": "4.1.1",
"activerecord": "4.1.1",
"activesupport": "4.1.1",
"arel": "5.0.1.20140414130214",
"bootstrap-sass": "3.1.1.1",
"builder": "3.2.2",
"coffee-rails": "4.0.1",
"coffee-script": "2.2.0",
"coffee-script-source": "1.7.0",
"erubis": "2.7.0",
"execjs": "2.0.2",
"hike": "1.2.3",
"i18n": "0.6.9",
"jbuilder": "2.0.7",
"jquery-rails": "3.1.0",
"json": "1.8.1",
"kgio": "2.9.2",
"libv8": "3.16.14.3",
"mail": "2.5.4",
"mime-types": "1.25.1",
"minitest": "5.3.4",
"multi_json": "1.10.1",
"mysql2": "0.3.16",
"polyglot": "0.3.4",
"rack": "1.5.2",
"rack-test": "0.6.2",
"rails": "4.1.1",
"railties": "4.1.1",
"raindrops": "0.13.0",
"rake": "10.3.2",
"rdoc": "4.1.1",
"ref": "1.0.5",
"sass": "3.2.19",
"sass-rails": "4.0.3",
"sdoc": "0.4.0",
"spring": "1.1.3",
"sprockets": "2.11.0",
"sprockets-rails": "2.1.3",
"sqlite3": "1.3.9",
"therubyracer": "0.12.1",
"thor": "0.19.1",
"thread_safe": "0.3.3",
"tilt": "1.4.1",
"treetop": "1.4.15",
"turbolinks": "2.2.2",
"tzinfo": "1.2.0",
"uglifier": "2.5.0",
"unicorn": "4.8.3",
}
func TestParseGemfileLockEntries(t *testing.T) {
fixture, err := os.Open("test-fixtures/Gemfile.lock")
if err != nil {
t.Fatalf("failed to open fixture: %+v", err)
}
actual, err := ParseGemfileLockEntries(fixture)
if err != nil {
t.Fatalf("failed to parse gemfile lock: %+v", err)
}
if len(actual) != len(expected) {
for _, a := range actual {
t.Log(" ", a)
}
t.Fatalf("unexpected package count: %d!=%d", len(actual), len(expected))
}
for _, a := range actual {
expectedVersion, ok := expected[a.Name]
if !ok {
t.Errorf("unexpected package found: %s", a.Name)
}
if expectedVersion != a.Version {
t.Errorf("unexpected package version (pkg=%s): %s", a.Name, a.Version)
}
if a.Language != pkg.Ruby {
t.Errorf("bad language (pkg=%+v): %+v", a.Name, a.Language)
}
if a.Type != pkg.BundlerPkg {
t.Errorf("bad package type (pkg=%+v): %+v", a.Name, a.Type)
}
}
}
imgbom/analyzer/bundler/test-fixtures/Gemfile.lock
+149
View File
@@ -0,0 +1,149 @@
GEM
remote: https://rubygems.org/
specs:
actionmailer (4.1.1)
actionpack (= 4.1.1)
actionview (= 4.1.1)
mail (~> 2.5.4)
actionpack (4.1.1)
actionview (= 4.1.1)
activesupport (= 4.1.1)
rack (~> 1.5.2)
rack-test (~> 0.6.2)
actionview (4.1.1)
activesupport (= 4.1.1)
builder (~> 3.1)
erubis (~> 2.7.0)
activemodel (4.1.1)
activesupport (= 4.1.1)
builder (~> 3.1)
activerecord (4.1.1)
activemodel (= 4.1.1)
activesupport (= 4.1.1)
arel (~> 5.0.0)
activesupport (4.1.1)
i18n (~> 0.6, >= 0.6.9)
json (~> 1.7, >= 1.7.7)
minitest (~> 5.1)
thread_safe (~> 0.1)
tzinfo (~> 1.1)
arel (5.0.1.20140414130214)
bootstrap-sass (3.1.1.1)
sass (~> 3.2)
builder (3.2.2)
coffee-rails (4.0.1)
coffee-script (>= 2.2.0)
railties (>= 4.0.0, < 5.0)
coffee-script (2.2.0)
coffee-script-source
execjs
coffee-script-source (1.7.0)
erubis (2.7.0)
execjs (2.0.2)
hike (1.2.3)
i18n (0.6.9)
jbuilder (2.0.7)
activesupport (>= 3.0.0, < 5)
multi_json (~> 1.2)
jquery-rails (3.1.0)
railties (>= 3.0, < 5.0)
thor (>= 0.14, < 2.0)
json (1.8.1)
kgio (2.9.2)
libv8 (3.16.14.3)
mail (2.5.4)
mime-types (~> 1.16)
treetop (~> 1.4.8)
mime-types (1.25.1)
minitest (5.3.4)
multi_json (1.10.1)
mysql2 (0.3.16)
polyglot (0.3.4)
rack (1.5.2)
rack-test (0.6.2)
rack (>= 1.0)
rails (4.1.1)
actionmailer (= 4.1.1)
actionpack (= 4.1.1)
actionview (= 4.1.1)
activemodel (= 4.1.1)
activerecord (= 4.1.1)
activesupport (= 4.1.1)
bundler (>= 1.3.0, < 2.0)
railties (= 4.1.1)
sprockets-rails (~> 2.0)
railties (4.1.1)
actionpack (= 4.1.1)
activesupport (= 4.1.1)
rake (>= 0.8.7)
thor (>= 0.18.1, < 2.0)
raindrops (0.13.0)
rake (10.3.2)
rdoc (4.1.1)
json (~> 1.4)
ref (1.0.5)
sass (3.2.19)
sass-rails (4.0.3)
railties (>= 4.0.0, < 5.0)
sass (~> 3.2.0)
sprockets (~> 2.8, <= 2.11.0)
sprockets-rails (~> 2.0)
sdoc (0.4.0)
json (~> 1.8)
rdoc (~> 4.0, < 5.0)
spring (1.1.3)
sprockets (2.11.0)
hike (~> 1.2)
multi_json (~> 1.0)
rack (~> 1.0)
tilt (~> 1.1, != 1.3.0)
sprockets-rails (2.1.3)
actionpack (>= 3.0)
activesupport (>= 3.0)
sprockets (~> 2.8)
sqlite3 (1.3.9)
therubyracer (0.12.1)
libv8 (~> 3.16.14.0)
ref
thor (0.19.1)
thread_safe (0.3.3)
tilt (1.4.1)
treetop (1.4.15)
polyglot
polyglot (>= 0.3.1)
turbolinks (2.2.2)
coffee-rails
tzinfo (1.2.0)
thread_safe (~> 0.1)
uglifier (2.5.0)
execjs (>= 0.3.0)
json (>= 1.8.0)
unicorn (4.8.3)
kgio (~> 2.6)
rack
raindrops (~> 0.7)
BAD-SECTION
bad-entry (5.5.5)
bad-kgio (~> 2.6)
bad-rack
bad-raindrops (~> 0.7)
PLATFORMS
ruby
DEPENDENCIES
bootstrap-sass
coffee-rails (~> 4.0.0)
jbuilder (~> 2.0)
jquery-rails
mysql2 (~> 0.3.16)
rails (= 4.1.1)
sass-rails (~> 4.0.3)
sdoc (~> 0.4.0)
spring
sqlite3
therubyracer
turbolinks
uglifier (>= 1.3.0)
unicorn
imgbom/analyzer/controller.go
+11 -4
View File
@@ -1,6 +1,7 @@
package analyzer
import (
"github.com/anchore/imgbom/imgbom/analyzer/bundler"
"github.com/anchore/imgbom/imgbom/analyzer/dpkg"
"github.com/anchore/imgbom/imgbom/pkg"
"github.com/anchore/imgbom/imgbom/scope"
@@ -12,10 +13,7 @@ import (
var controllerInstance controller
func init() {
controllerInstance = controller{
analyzers: make([]Analyzer, 0),
}
controllerInstance.add(dpkg.NewAnalyzer())
controllerInstance = newController()
}
func Analyze(s scope.Scope) (*pkg.Catalog, error) {
@@ -26,6 +24,15 @@ type controller struct {
analyzers []Analyzer
}
func newController() controller {
ctrlr := controller{
analyzers: make([]Analyzer, 0),
}
ctrlr.add(dpkg.NewAnalyzer())
ctrlr.add(bundler.NewAnalyzer())
return ctrlr
}
func (c *controller) add(a Analyzer) {
log.Debugf("adding analyzer: %s", a.Name())
c.analyzers = append(c.analyzers, a)
imgbom/analyzer/dpkg/analyzer.go
+33 -14
View File
@@ -1,6 +1,7 @@
package dpkg
import (
"io"
"strings"
"github.com/anchore/imgbom/imgbom/pkg"
@@ -9,51 +10,69 @@ import (
"github.com/anchore/stereoscope/pkg/tree"
)
var parserDispatch = map[string]parserFn{
"/var/lib/dpkg/status": ParseDpkgStatusEntries,
}
type parserFn func(io.Reader) ([]pkg.DpkgMetadata, error)
type Analyzer struct {
selectedFiles []file.Reference
parsers map[file.Reference]parserFn
}
func NewAnalyzer() *Analyzer {
return &Analyzer{}
return &Analyzer{
selectedFiles: make([]file.Reference, 0),
parsers: make(map[file.Reference]parserFn),
}
}
func (a *Analyzer) Name() string {
return "dpkg-analyzer"
}
func (a *Analyzer) register(f file.Reference, parser parserFn) {
a.selectedFiles = append(a.selectedFiles, f)
a.parsers[f] = parser
}
func (a *Analyzer) clear() {
a.selectedFiles = make([]file.Reference, 0)
a.parsers = make(map[file.Reference]parserFn)
}
func (a *Analyzer) SelectFiles(trees []tree.FileTreeReader) []file.Reference {
files := make([]file.Reference, 0)
for _, tree := range trees {
// TODO: extract into function/slice/etc
file := tree.File("/var/lib/dpkg/status")
if file != nil {
files = append(files, *file)
for exactPath, parser := range parserDispatch {
match := tree.File(file.Path(exactPath))
if match != nil {
a.register(*match, parser)
}
}
}
a.selectedFiles = files
return files
return a.selectedFiles
}
func (a *Analyzer) Analyze(contents map[file.Reference]string) ([]pkg.Package, error) {
defer a.clear()
packages := make([]pkg.Package, 0)
for _, reference := range a.selectedFiles {
content, ok := contents[reference]
if !ok {
// TODO: test case
log.Errorf("analyzer '%s' file content missing: %+v", a.Name(), reference)
continue
}
entries, err := ParseEntries(strings.NewReader(content))
entries, err := ParseDpkgStatusEntries(strings.NewReader(content))
if err != nil {
// TODO: test case
log.Errorf("analyzer failed to parse entries (reference=%+v): %w", reference, err)
continue
}
for _, entry := range entries {
packages = append(packages, pkg.Package{
Name: entry.Package,
imgbom/analyzer/dpkg/parser.go → imgbom/analyzer/dpkg/parse_dpkg_status.go
+3 -3
View File
@@ -12,12 +12,12 @@ import (
var errEndOfPackages = fmt.Errorf("no more packages to read")
func ParseEntries(reader io.Reader) ([]pkg.DpkgMetadata, error) {
func ParseDpkgStatusEntries(reader io.Reader) ([]pkg.DpkgMetadata, error) {
buffedReader := bufio.NewReader(reader)
var entries = make([]pkg.DpkgMetadata, 0)
for {
entry, err := parseEntry(buffedReader)
entry, err := parseDpkgStatusEntry(buffedReader)
if err != nil {
if err == errEndOfPackages {
break
@@ -30,7 +30,7 @@ func ParseEntries(reader io.Reader) ([]pkg.DpkgMetadata, error) {
return entries, nil
}
func parseEntry(reader *bufio.Reader) (entry pkg.DpkgMetadata, err error) {
func parseDpkgStatusEntry(reader *bufio.Reader) (entry pkg.DpkgMetadata, err error) {
dpkgFields := make(map[string]string)
var key string
imgbom/analyzer/dpkg/parser_test.go → imgbom/analyzer/dpkg/parse_dpkg_status_test.go
+2 -2
View File
@@ -46,7 +46,7 @@ func TestSinglePackage(t *testing.T) {
reader := bufio.NewReader(file)
entry, err := parseEntry(reader)
entry, err := parseDpkgStatusEntry(reader)
if err != nil {
t.Fatal("Unable to read file contents: ", err)
}
@@ -90,7 +90,7 @@ func TestMultiplePackages(t *testing.T) {
}
}()
entries, err := ParseEntries(file)
entries, err := ParseDpkgStatusEntries(file)
if err != nil {
t.Fatal("Unable to read file contents: ", err)
}
imgbom/pkg/catalog.go
+4
View File
@@ -26,6 +26,10 @@ func NewCatalog() *Catalog {
}
}
func (c *Catalog) PackageCount() int {
return len(c.byID)
}
func (c *Catalog) Package(id ID) *Package {
return c.byID[id]
}
imgbom/pkg/language.go
+33
View File
@@ -0,0 +1,33 @@
package pkg
const (
UnknownLanguage Language = iota
Java
JavaScript
Python
Ruby
)
type Language uint
var languageStr = []string{
"UnknownLanguage",
"java",
"javascript",
"python",
"ruby",
}
var AllLanguages = []Language{
Java,
JavaScript,
Python,
Ruby,
}
func (t Language) String() string {
if int(t) >= len(languageStr) {
return languageStr[0]
}
return languageStr[t]
}
imgbom/pkg/package.go
+1
View File
@@ -16,6 +16,7 @@ type Package struct {
FoundBy string
Source []file.Reference
Licenses []string
Language Language // TODO: should this support multiple languages as a slice?
Type Type
Metadata interface{}
}
imgbom/pkg/type.go
+10 -8
View File
@@ -3,13 +3,10 @@ package pkg
const (
UnknownPkg Type = iota
ApkPkg
BundlerPkg
DebPkg
JavaPkg
JavaScriptPkg
PacmanPkg
PythonPkg
RpmPkg
RubyPkg
)
type Type uint
@@ -17,13 +14,18 @@ type Type uint
var typeStr = []string{
"UnknownPackage",
"apk",
"bundler",
"deb",
"java",
"javascript",
"pacman",
"python",
"rpm",
"ruby",
}
var AllPkgs = []Type{
ApkPkg,
BundlerPkg,
DebPkg,
PacmanPkg,
RpmPkg,
}
func (t Type) String() string {