diff --git a/go.mod b/go.mod index 1aa65886a..7e3db971e 100644 --- a/go.mod +++ b/go.mod @@ -21,6 +21,7 @@ require ( go.uber.org/zap v1.15.0 golang.org/x/net v0.0.0-20200602114024-627f9648deb9 // indirect golang.org/x/sys v0.0.0-20200602225109-6fdc65e7d980 // indirect + google.golang.org/appengine v1.6.6 google.golang.org/protobuf v1.24.0 // indirect gopkg.in/ini.v1 v1.57.0 // indirect gopkg.in/yaml.v2 v2.3.0 diff --git a/go.sum b/go.sum index 00f730a15..ea61f80a4 100644 --- a/go.sum +++ b/go.sum @@ -459,6 +459,7 @@ github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/lib/pq v1.1.1/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/lib/pq v1.2.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/logrusorgru/aurora v0.0.0-20181002194514-a7b3b318ed4e/go.mod h1:7rIyQOR62GCctdiQpZ/zOJlFyk6y+94wXzv6RNZgaR4= +github.com/logrusorgru/aurora v0.0.0-20200102142835-e9ef32dff381 h1:bqDmpDG49ZRnB5PcgP0RXtQvnMSgIF14M7CBd2shtXs= github.com/logrusorgru/aurora v0.0.0-20200102142835-e9ef32dff381/go.mod h1:7rIyQOR62GCctdiQpZ/zOJlFyk6y+94wXzv6RNZgaR4= github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= github.com/magiconair/properties v1.8.1 h1:ZC2Vc7/ZFkGmsVC9KvOjumD+G5lXy2RtTKyzRKO2BQ4= @@ -595,6 +596,7 @@ github.com/securego/gosec v0.0.0-20200103095621-79fbf3af8d83/go.mod h1:vvbZ2Ae7A github.com/securego/gosec v0.0.0-20200401082031-e946c8c39989/go.mod h1:i9l/TNj+yDFh9SZXUTvspXTjbFXgZGP/UvhU1S65A4A= github.com/securego/gosec/v2 v2.3.0/go.mod h1:UzeVyUXbxukhLeHKV3VVqo7HdoQR9MrRfFmZYotn8ME= github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= +github.com/sergi/go-diff v1.1.0 h1:we8PVUC3FE2uYfodKH/nBHMSetSfHDR6scGdBi+erh0= github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM= github.com/shirou/gopsutil v0.0.0-20190901111213-e4ec7b275ada/go.mod h1:WWnYX4lzhCH5h/3YBfyVA3VbLYjlMZZAQcW9ojMexNc= github.com/shirou/w32 v0.0.0-20160930032740-bb4de0191aa4/go.mod h1:qsXQc7+bwAM3Q1u/4XEfrquwF8Lw7D7y5cD8CuHnfIc= @@ -969,6 +971,7 @@ google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7 google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/appengine v1.6.6 h1:lMO5rYAqUxkmaj76jAkRUvt5JZgFymx/+Q5Mzfivuhc= google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= diff --git a/imgbom/analyzer/bundler/analyzer.go b/imgbom/analyzer/bundler/analyzer.go index 190be74e6..7c4dceec1 100644 --- a/imgbom/analyzer/bundler/analyzer.go +++ b/imgbom/analyzer/bundler/analyzer.go @@ -1,30 +1,23 @@ package bundler import ( - "io" - "strings" - + "github.com/anchore/imgbom/imgbom/analyzer/common" "github.com/anchore/imgbom/imgbom/pkg" - "github.com/anchore/imgbom/internal/log" "github.com/anchore/stereoscope/pkg/file" "github.com/anchore/stereoscope/pkg/tree" ) -var parserDispatch = map[string]parserFn{ - "*/Gemfile.lock": ParseGemfileLockEntries, -} - -type parserFn func(io.Reader) ([]pkg.Package, error) - type Analyzer struct { - selectedFiles []file.Reference - parsers map[file.Reference]parserFn + analyzer common.GenericAnalyzer } func NewAnalyzer() *Analyzer { + globParserDispatch := map[string]common.ParserFn{ + "*/Gemfile.lock": ParseGemfileLockEntries, + } + return &Analyzer{ - selectedFiles: make([]file.Reference, 0), - parsers: make(map[file.Reference]parserFn), + analyzer: common.NewGenericAnalyzer(nil, globParserDispatch), } } @@ -32,59 +25,10 @@ func (a *Analyzer) Name() string { return "bundler-analyzer" } -func (a *Analyzer) register(files []file.Reference, parser parserFn) { - a.selectedFiles = append(a.selectedFiles, files...) - for _, f := range files { - a.parsers[f] = parser - } -} - -func (a *Analyzer) clear() { - a.selectedFiles = make([]file.Reference, 0) - a.parsers = make(map[file.Reference]parserFn) -} - func (a *Analyzer) SelectFiles(trees []tree.FileTreeReader) []file.Reference { - for _, tree := range trees { - for globPattern, parser := range parserDispatch { - fileMatches, err := tree.FilesByGlob(globPattern) - if err != nil { - log.Errorf("'%s' failed to find files by glob: %s", a.Name(), globPattern) - } - if fileMatches != nil { - a.register(fileMatches, parser) - } - } - } - - return a.selectedFiles + return a.analyzer.SelectFiles(trees) } func (a *Analyzer) Analyze(contents map[file.Reference]string) ([]pkg.Package, error) { - defer a.clear() - - packages := make([]pkg.Package, 0) - - for reference, parser := range a.parsers { - content, ok := contents[reference] - if !ok { - log.Errorf("analyzer '%s' file content missing: %+v", a.Name(), reference) - continue - } - - entries, err := parser(strings.NewReader(content)) - if err != nil { - log.Errorf("analyzer failed to parse entries (reference=%+v): %w", reference, err) - continue - } - - for _, entry := range entries { - entry.FoundBy = a.Name() - entry.Source = []file.Reference{reference} - - packages = append(packages, entry) - } - } - - return packages, nil + return a.analyzer.Analyze(contents, a.Name()) } diff --git a/imgbom/analyzer/common/generic_analyzer.go b/imgbom/analyzer/common/generic_analyzer.go new file mode 100644 index 000000000..b9a61a385 --- /dev/null +++ b/imgbom/analyzer/common/generic_analyzer.go @@ -0,0 +1,92 @@ +package common + +import ( + "strings" + + "github.com/anchore/imgbom/imgbom/pkg" + "github.com/anchore/imgbom/internal/log" + "github.com/anchore/stereoscope/pkg/file" + "github.com/anchore/stereoscope/pkg/tree" +) + +type GenericAnalyzer struct { + globParserDispatch map[string]ParserFn + pathParserDispatch map[string]ParserFn + selectedFiles []file.Reference + parsers map[file.Reference]ParserFn +} + +func NewGenericAnalyzer(pathParserDispatch map[string]ParserFn, globParserDispatch map[string]ParserFn) GenericAnalyzer { + return GenericAnalyzer{ + globParserDispatch: globParserDispatch, + pathParserDispatch: pathParserDispatch, + selectedFiles: make([]file.Reference, 0), + parsers: make(map[file.Reference]ParserFn), + } +} + +func (a *GenericAnalyzer) register(files []file.Reference, parser ParserFn) { + a.selectedFiles = append(a.selectedFiles, files...) + for _, f := range files { + a.parsers[f] = parser + } +} + +func (a *GenericAnalyzer) clear() { + a.selectedFiles = make([]file.Reference, 0) + a.parsers = make(map[file.Reference]ParserFn) +} + +func (a *GenericAnalyzer) SelectFiles(trees []tree.FileTreeReader) []file.Reference { + for _, tree := range trees { + // select by exact path + for path, parser := range a.globParserDispatch { + f := tree.File(file.Path(path)) + if f != nil { + a.register([]file.Reference{*f}, parser) + } + } + + // select by pattern + for globPattern, parser := range a.globParserDispatch { + fileMatches, err := tree.FilesByGlob(globPattern) + if err != nil { + log.Errorf("failed to find files by glob: %s", globPattern) + } + if fileMatches != nil { + a.register(fileMatches, parser) + } + } + } + + return a.selectedFiles +} + +func (a *GenericAnalyzer) Analyze(contents map[file.Reference]string, upstreamMatcher string) ([]pkg.Package, error) { + defer a.clear() + + packages := make([]pkg.Package, 0) + + for reference, parser := range a.parsers { + content, ok := contents[reference] + if !ok { + log.Errorf("analyzer '%s' missing file content: %+v", upstreamMatcher, reference) + continue + } + + entries, err := parser(strings.NewReader(content)) + if err != nil { + log.Errorf("analyzer '%s' failed to parse entries (reference=%+v): %w", upstreamMatcher, reference, err) + continue + } + + for _, entry := range entries { + entry.FoundBy = upstreamMatcher + entry.Source = []file.Reference{reference} + + packages = append(packages, entry) + } + } + + return packages, nil +} diff --git a/imgbom/analyzer/common/parser.go b/imgbom/analyzer/common/parser.go new file mode 100644 index 000000000..11232dfdb --- /dev/null +++ b/imgbom/analyzer/common/parser.go @@ -0,0 +1,9 @@ +package common + +import ( + "io" + + "github.com/anchore/imgbom/imgbom/pkg" +) + +type ParserFn func(io.Reader) ([]pkg.Package, error) diff --git a/imgbom/analyzer/dpkg/analyzer.go b/imgbom/analyzer/dpkg/analyzer.go index e5a0d6dc1..7af4cc810 100644 --- a/imgbom/analyzer/dpkg/analyzer.go +++ b/imgbom/analyzer/dpkg/analyzer.go @@ -1,30 +1,23 @@ package dpkg import ( - "io" - "strings" - + "github.com/anchore/imgbom/imgbom/analyzer/common" "github.com/anchore/imgbom/imgbom/pkg" - "github.com/anchore/imgbom/internal/log" "github.com/anchore/stereoscope/pkg/file" "github.com/anchore/stereoscope/pkg/tree" ) -var parserDispatch = map[string]parserFn{ - "/var/lib/dpkg/status": ParseDpkgStatusEntries, -} - -type parserFn func(io.Reader) ([]pkg.DpkgMetadata, error) - type Analyzer struct { - selectedFiles []file.Reference - parsers map[file.Reference]parserFn + analyzer common.GenericAnalyzer } func NewAnalyzer() *Analyzer { + pathParserDispatch := map[string]common.ParserFn{ + "/var/lib/dpkg/status": ParseDpkgStatus, + } + return &Analyzer{ - selectedFiles: make([]file.Reference, 0), - parsers: make(map[file.Reference]parserFn), + analyzer: common.NewGenericAnalyzer(pathParserDispatch, nil), } } @@ -32,58 +25,10 @@ func (a *Analyzer) Name() string { return "dpkg-analyzer" } -func (a *Analyzer) register(f file.Reference, parser parserFn) { - a.selectedFiles = append(a.selectedFiles, f) - a.parsers[f] = parser -} - -func (a *Analyzer) clear() { - a.selectedFiles = make([]file.Reference, 0) - a.parsers = make(map[file.Reference]parserFn) -} - func (a *Analyzer) SelectFiles(trees []tree.FileTreeReader) []file.Reference { - for _, tree := range trees { - for exactPath, parser := range parserDispatch { - match := tree.File(file.Path(exactPath)) - if match != nil { - a.register(*match, parser) - } - } - } - - return a.selectedFiles + return a.analyzer.SelectFiles(trees) } func (a *Analyzer) Analyze(contents map[file.Reference]string) ([]pkg.Package, error) { - defer a.clear() - - packages := make([]pkg.Package, 0) - - for _, reference := range a.selectedFiles { - content, ok := contents[reference] - if !ok { - log.Errorf("analyzer '%s' file content missing: %+v", a.Name(), reference) - continue - } - - entries, err := ParseDpkgStatusEntries(strings.NewReader(content)) - if err != nil { - log.Errorf("analyzer failed to parse entries (reference=%+v): %w", reference, err) - continue - } - - for _, entry := range entries { - packages = append(packages, pkg.Package{ - Name: entry.Package, - Version: entry.Version, - Type: pkg.DebPkg, - FoundBy: a.Name(), - Source: []file.Reference{reference}, - Metadata: entry, - }) - } - } - - return packages, nil + return a.analyzer.Analyze(contents, a.Name()) } diff --git a/imgbom/analyzer/dpkg/parse_dpkg_status.go b/imgbom/analyzer/dpkg/parse_dpkg_status.go index ecc4246ad..cee6e5b33 100644 --- a/imgbom/analyzer/dpkg/parse_dpkg_status.go +++ b/imgbom/analyzer/dpkg/parse_dpkg_status.go @@ -12,9 +12,9 @@ import ( var errEndOfPackages = fmt.Errorf("no more packages to read") -func ParseDpkgStatusEntries(reader io.Reader) ([]pkg.DpkgMetadata, error) { +func ParseDpkgStatus(reader io.Reader) ([]pkg.Package, error) { buffedReader := bufio.NewReader(reader) - var entries = make([]pkg.DpkgMetadata, 0) + var packages = make([]pkg.Package, 0) for { entry, err := parseDpkgStatusEntry(buffedReader) @@ -24,10 +24,15 @@ func ParseDpkgStatusEntries(reader io.Reader) ([]pkg.DpkgMetadata, error) { } return nil, err } - entries = append(entries, entry) + packages = append(packages, pkg.Package{ + Name: entry.Package, + Version: entry.Version, + Type: pkg.DebPkg, + Metadata: entry, + }) } - return entries, nil + return packages, nil } func parseDpkgStatusEntry(reader *bufio.Reader) (entry pkg.DpkgMetadata, err error) { diff --git a/imgbom/analyzer/dpkg/parse_dpkg_status_test.go b/imgbom/analyzer/dpkg/parse_dpkg_status_test.go index ccd353757..339ac980b 100644 --- a/imgbom/analyzer/dpkg/parse_dpkg_status_test.go +++ b/imgbom/analyzer/dpkg/parse_dpkg_status_test.go @@ -90,17 +90,17 @@ func TestMultiplePackages(t *testing.T) { } }() - entries, err := ParseDpkgStatusEntries(file) + pkgs, err := ParseDpkgStatus(file) if err != nil { t.Fatal("Unable to read file contents: ", err) } - if len(entries) != 2 { - t.Fatalf("unexpected number of entries: %d", len(entries)) + if len(pkgs) != 2 { + t.Fatalf("unexpected number of entries: %d", len(pkgs)) } - for idx, entry := range entries { - compareEntries(t, entry, test.expected[idx]) + for idx, entry := range pkgs { + compareEntries(t, entry.Metadata.(pkg.DpkgMetadata), test.expected[idx]) } })