generalize common analyzer elements

go.mod

@@ -21,6 +21,7 @@ require (
 	go.uber.org/zap v1.15.0
 	golang.org/x/net v0.0.0-20200602114024-627f9648deb9 // indirect
 	golang.org/x/sys v0.0.0-20200602225109-6fdc65e7d980 // indirect
+	google.golang.org/appengine v1.6.6
 	google.golang.org/protobuf v1.24.0 // indirect
 	gopkg.in/ini.v1 v1.57.0 // indirect
 	gopkg.in/yaml.v2 v2.3.0

go.sum

@@ -459,6 +459,7 @@ github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
 github.com/lib/pq v1.1.1/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
 github.com/lib/pq v1.2.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
 github.com/logrusorgru/aurora v0.0.0-20181002194514-a7b3b318ed4e/go.mod h1:7rIyQOR62GCctdiQpZ/zOJlFyk6y+94wXzv6RNZgaR4=
+github.com/logrusorgru/aurora v0.0.0-20200102142835-e9ef32dff381 h1:bqDmpDG49ZRnB5PcgP0RXtQvnMSgIF14M7CBd2shtXs=
 github.com/logrusorgru/aurora v0.0.0-20200102142835-e9ef32dff381/go.mod h1:7rIyQOR62GCctdiQpZ/zOJlFyk6y+94wXzv6RNZgaR4=
 github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
 github.com/magiconair/properties v1.8.1 h1:ZC2Vc7/ZFkGmsVC9KvOjumD+G5lXy2RtTKyzRKO2BQ4=
@@ -595,6 +596,7 @@ github.com/securego/gosec v0.0.0-20200103095621-79fbf3af8d83/go.mod h1:vvbZ2Ae7A
 github.com/securego/gosec v0.0.0-20200401082031-e946c8c39989/go.mod h1:i9l/TNj+yDFh9SZXUTvspXTjbFXgZGP/UvhU1S65A4A=
 github.com/securego/gosec/v2 v2.3.0/go.mod h1:UzeVyUXbxukhLeHKV3VVqo7HdoQR9MrRfFmZYotn8ME=
 github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo=
+github.com/sergi/go-diff v1.1.0 h1:we8PVUC3FE2uYfodKH/nBHMSetSfHDR6scGdBi+erh0=
 github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM=
 github.com/shirou/gopsutil v0.0.0-20190901111213-e4ec7b275ada/go.mod h1:WWnYX4lzhCH5h/3YBfyVA3VbLYjlMZZAQcW9ojMexNc=
 github.com/shirou/w32 v0.0.0-20160930032740-bb4de0191aa4/go.mod h1:qsXQc7+bwAM3Q1u/4XEfrquwF8Lw7D7y5cD8CuHnfIc=
@@ -969,6 +971,7 @@ google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7
 google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
 google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0=
 google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
+google.golang.org/appengine v1.6.6 h1:lMO5rYAqUxkmaj76jAkRUvt5JZgFymx/+Q5Mzfivuhc=
 google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
 google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
 google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=

imgbom/analyzer/bundler/analyzer.go

@@ -1,30 +1,23 @@
 package bundler
 
 import (
-	"io"
+	"github.com/anchore/imgbom/imgbom/analyzer/common"
-	"strings"
-
 	"github.com/anchore/imgbom/imgbom/pkg"
-	"github.com/anchore/imgbom/internal/log"
 	"github.com/anchore/stereoscope/pkg/file"
 	"github.com/anchore/stereoscope/pkg/tree"
 )
 
-var parserDispatch = map[string]parserFn{
-	"*/Gemfile.lock": ParseGemfileLockEntries,
-}
-
-type parserFn func(io.Reader) ([]pkg.Package, error)
-
 type Analyzer struct {
-	selectedFiles []file.Reference
+	analyzer common.GenericAnalyzer
-	parsers       map[file.Reference]parserFn
 }
 
 func NewAnalyzer() *Analyzer {
+	globParserDispatch := map[string]common.ParserFn{
+		"*/Gemfile.lock": ParseGemfileLockEntries,
+	}
+
 	return &Analyzer{
-		selectedFiles: make([]file.Reference, 0),
+		analyzer: common.NewGenericAnalyzer(nil, globParserDispatch),
-		parsers:       make(map[file.Reference]parserFn),
 	}
 }
 
@@ -32,59 +25,10 @@ func (a *Analyzer) Name() string {
 	return "bundler-analyzer"
 }
 
-func (a *Analyzer) register(files []file.Reference, parser parserFn) {
-	a.selectedFiles = append(a.selectedFiles, files...)
-	for _, f := range files {
-		a.parsers[f] = parser
-	}
-}
-
-func (a *Analyzer) clear() {
-	a.selectedFiles = make([]file.Reference, 0)
-	a.parsers = make(map[file.Reference]parserFn)
-}
-
 func (a *Analyzer) SelectFiles(trees []tree.FileTreeReader) []file.Reference {
-	for _, tree := range trees {
+	return a.analyzer.SelectFiles(trees)
-		for globPattern, parser := range parserDispatch {
-			fileMatches, err := tree.FilesByGlob(globPattern)
-			if err != nil {
-				log.Errorf("'%s' failed to find files by glob: %s", a.Name(), globPattern)
-			}
-			if fileMatches != nil {
-				a.register(fileMatches, parser)
-			}
-		}
-	}
-
-	return a.selectedFiles
 }
 
 func (a *Analyzer) Analyze(contents map[file.Reference]string) ([]pkg.Package, error) {
-	defer a.clear()
+	return a.analyzer.Analyze(contents, a.Name())
-
-	packages := make([]pkg.Package, 0)
-
-	for reference, parser := range a.parsers {
-		content, ok := contents[reference]
-		if !ok {
-			log.Errorf("analyzer '%s' file content missing: %+v", a.Name(), reference)
-			continue
-		}
-
-		entries, err := parser(strings.NewReader(content))
-		if err != nil {
-			log.Errorf("analyzer failed to parse entries (reference=%+v): %w", reference, err)
-			continue
-		}
-
-		for _, entry := range entries {
-			entry.FoundBy = a.Name()
-			entry.Source = []file.Reference{reference}
-
-			packages = append(packages, entry)
-		}
-	}
-
-	return packages, nil
 }

imgbom/analyzer/common/generic_analyzer.go

@@ -0,0 +1,92 @@
+package common
+
+import (
+	"strings"
+
+	"github.com/anchore/imgbom/imgbom/pkg"
+	"github.com/anchore/imgbom/internal/log"
+	"github.com/anchore/stereoscope/pkg/file"
+	"github.com/anchore/stereoscope/pkg/tree"
+)
+
+type GenericAnalyzer struct {
+	globParserDispatch map[string]ParserFn
+	pathParserDispatch map[string]ParserFn
+	selectedFiles      []file.Reference
+	parsers            map[file.Reference]ParserFn
+}
+
+func NewGenericAnalyzer(pathParserDispatch map[string]ParserFn, globParserDispatch map[string]ParserFn) GenericAnalyzer {
+	return GenericAnalyzer{
+		globParserDispatch: globParserDispatch,
+		pathParserDispatch: pathParserDispatch,
+		selectedFiles:      make([]file.Reference, 0),
+		parsers:            make(map[file.Reference]ParserFn),
+	}
+}
+
+func (a *GenericAnalyzer) register(files []file.Reference, parser ParserFn) {
+	a.selectedFiles = append(a.selectedFiles, files...)
+	for _, f := range files {
+		a.parsers[f] = parser
+	}
+}
+
+func (a *GenericAnalyzer) clear() {
+	a.selectedFiles = make([]file.Reference, 0)
+	a.parsers = make(map[file.Reference]ParserFn)
+}
+
+func (a *GenericAnalyzer) SelectFiles(trees []tree.FileTreeReader) []file.Reference {
+	for _, tree := range trees {
+		// select by exact path
+		for path, parser := range a.globParserDispatch {
+			f := tree.File(file.Path(path))
+			if f != nil {
+				a.register([]file.Reference{*f}, parser)
+			}
+		}
+
+		// select by pattern
+		for globPattern, parser := range a.globParserDispatch {
+			fileMatches, err := tree.FilesByGlob(globPattern)
+			if err != nil {
+				log.Errorf("failed to find files by glob: %s", globPattern)
+			}
+			if fileMatches != nil {
+				a.register(fileMatches, parser)
+			}
+		}
+	}
+
+	return a.selectedFiles
+}
+
+func (a *GenericAnalyzer) Analyze(contents map[file.Reference]string, upstreamMatcher string) ([]pkg.Package, error) {
+	defer a.clear()
+
+	packages := make([]pkg.Package, 0)
+
+	for reference, parser := range a.parsers {
+		content, ok := contents[reference]
+		if !ok {
+			log.Errorf("analyzer '%s' missing file content: %+v", upstreamMatcher, reference)
+			continue
+		}
+
+		entries, err := parser(strings.NewReader(content))
+		if err != nil {
+			log.Errorf("analyzer '%s' failed to parse entries (reference=%+v): %w", upstreamMatcher, reference, err)
+			continue
+		}
+
+		for _, entry := range entries {
+			entry.FoundBy = upstreamMatcher
+			entry.Source = []file.Reference{reference}
+
+			packages = append(packages, entry)
+		}
+	}
+
+	return packages, nil
+}

imgbom/analyzer/common/parser.go

@@ -0,0 +1,9 @@
+package common
+
+import (
+	"io"
+
+	"github.com/anchore/imgbom/imgbom/pkg"
+)
+
+type ParserFn func(io.Reader) ([]pkg.Package, error)

imgbom/analyzer/dpkg/analyzer.go

@@ -1,30 +1,23 @@
 package dpkg
 
 import (
-	"io"
+	"github.com/anchore/imgbom/imgbom/analyzer/common"
-	"strings"
-
 	"github.com/anchore/imgbom/imgbom/pkg"
-	"github.com/anchore/imgbom/internal/log"
 	"github.com/anchore/stereoscope/pkg/file"
 	"github.com/anchore/stereoscope/pkg/tree"
 )
 
-var parserDispatch = map[string]parserFn{
-	"/var/lib/dpkg/status": ParseDpkgStatusEntries,
-}
-
-type parserFn func(io.Reader) ([]pkg.DpkgMetadata, error)
-
 type Analyzer struct {
-	selectedFiles []file.Reference
+	analyzer common.GenericAnalyzer
-	parsers       map[file.Reference]parserFn
 }
 
 func NewAnalyzer() *Analyzer {
+	pathParserDispatch := map[string]common.ParserFn{
+		"/var/lib/dpkg/status": ParseDpkgStatus,
+	}
+
 	return &Analyzer{
-		selectedFiles: make([]file.Reference, 0),
+		analyzer: common.NewGenericAnalyzer(pathParserDispatch, nil),
-		parsers:       make(map[file.Reference]parserFn),
 	}
 }
 
@@ -32,58 +25,10 @@ func (a *Analyzer) Name() string {
 	return "dpkg-analyzer"
 }
 
-func (a *Analyzer) register(f file.Reference, parser parserFn) {
-	a.selectedFiles = append(a.selectedFiles, f)
-	a.parsers[f] = parser
-}
-
-func (a *Analyzer) clear() {
-	a.selectedFiles = make([]file.Reference, 0)
-	a.parsers = make(map[file.Reference]parserFn)
-}
-
 func (a *Analyzer) SelectFiles(trees []tree.FileTreeReader) []file.Reference {
-	for _, tree := range trees {
+	return a.analyzer.SelectFiles(trees)
-		for exactPath, parser := range parserDispatch {
-			match := tree.File(file.Path(exactPath))
-			if match != nil {
-				a.register(*match, parser)
-			}
-		}
-	}
-
-	return a.selectedFiles
 }
 
 func (a *Analyzer) Analyze(contents map[file.Reference]string) ([]pkg.Package, error) {
-	defer a.clear()
+	return a.analyzer.Analyze(contents, a.Name())
-
-	packages := make([]pkg.Package, 0)
-
-	for _, reference := range a.selectedFiles {
-		content, ok := contents[reference]
-		if !ok {
-			log.Errorf("analyzer '%s' file content missing: %+v", a.Name(), reference)
-			continue
-		}
-
-		entries, err := ParseDpkgStatusEntries(strings.NewReader(content))
-		if err != nil {
-			log.Errorf("analyzer failed to parse entries (reference=%+v): %w", reference, err)
-			continue
-		}
-
-		for _, entry := range entries {
-			packages = append(packages, pkg.Package{
-				Name:     entry.Package,
-				Version:  entry.Version,
-				Type:     pkg.DebPkg,
-				FoundBy:  a.Name(),
-				Source:   []file.Reference{reference},
-				Metadata: entry,
-			})
-		}
-	}
-
-	return packages, nil
 }

imgbom/analyzer/dpkg/parse_dpkg_status.go

@@ -12,9 +12,9 @@ import (
 
 var errEndOfPackages = fmt.Errorf("no more packages to read")
 
-func ParseDpkgStatusEntries(reader io.Reader) ([]pkg.DpkgMetadata, error) {
+func ParseDpkgStatus(reader io.Reader) ([]pkg.Package, error) {
 	buffedReader := bufio.NewReader(reader)
-	var entries = make([]pkg.DpkgMetadata, 0)
+	var packages = make([]pkg.Package, 0)
 
 	for {
 		entry, err := parseDpkgStatusEntry(buffedReader)
@@ -24,10 +24,15 @@ func ParseDpkgStatusEntries(reader io.Reader) ([]pkg.DpkgMetadata, error) {
 			}
 			return nil, err
 		}
-		entries = append(entries, entry)
+		packages = append(packages, pkg.Package{
+			Name:     entry.Package,
+			Version:  entry.Version,
+			Type:     pkg.DebPkg,
+			Metadata: entry,
+		})
 	}
 
-	return entries, nil
+	return packages, nil
 }
 
 func parseDpkgStatusEntry(reader *bufio.Reader) (entry pkg.DpkgMetadata, err error) {

imgbom/analyzer/dpkg/parse_dpkg_status_test.go

@@ -90,17 +90,17 @@ func TestMultiplePackages(t *testing.T) {
 				}
 			}()
 
-			entries, err := ParseDpkgStatusEntries(file)
+			pkgs, err := ParseDpkgStatus(file)
 			if err != nil {
 				t.Fatal("Unable to read file contents: ", err)
 			}
 
-			if len(entries) != 2 {
+			if len(pkgs) != 2 {
-				t.Fatalf("unexpected number of entries: %d", len(entries))
+				t.Fatalf("unexpected number of entries: %d", len(pkgs))
 			}
 
-			for idx, entry := range entries {
+			for idx, entry := range pkgs {
-				compareEntries(t, entry, test.expected[idx])
+				compareEntries(t, entry.Metadata.(pkg.DpkgMetadata), test.expected[idx])
 			}
 
 		})