Upgrade generic cataloger (#1281)

* add second generation of generic cataloger

Signed-off-by: Alex Goodman <alex.goodman@anchore.com>

* upgrade aplm cataloger to use generic.Cataloger

Signed-off-by: Alex Goodman <alex.goodman@anchore.com>

* remove pacakge found-by attribute from the definition of a package ID

Signed-off-by: Alex Goodman <alex.goodman@anchore.com>

Signed-off-by: Alex Goodman <alex.goodman@anchore.com>

syft/pkg/cataloger/generic/cataloger.go

@@ -0,0 +1,151 @@
+package generic
+
+import (
+	"github.com/anchore/syft/internal"
+	"github.com/anchore/syft/internal/log"
+	"github.com/anchore/syft/syft/artifact"
+	"github.com/anchore/syft/syft/linux"
+	"github.com/anchore/syft/syft/pkg"
+	"github.com/anchore/syft/syft/source"
+)
+
+type processor func(resolver source.FileResolver, env Environment) []request
+
+type request struct {
+	source.Location
+	Parser
+}
+
+// Cataloger implements the Catalog interface and is responsible for dispatching the proper parser function for
+// a given path or glob pattern. This is intended to be reusable across many package cataloger types.
+type Cataloger struct {
+	processor         []processor
+	upstreamCataloger string
+}
+
+func (c *Cataloger) WithParserByGlobs(parser Parser, globs ...string) *Cataloger {
+	c.processor = append(c.processor,
+		func(resolver source.FileResolver, env Environment) []request {
+			var requests []request
+			for _, g := range globs {
+				// TODO: add more trace logging here
+				matches, err := resolver.FilesByGlob(g)
+				if err != nil {
+					log.Warnf("unable to process glob=%q: %+v", g, err)
+					continue
+				}
+				requests = append(requests, makeRequests(parser, matches)...)
+			}
+			return requests
+		},
+	)
+	return c
+}
+
+func (c *Cataloger) WithParserByMimeTypes(parser Parser, types ...string) *Cataloger {
+	c.processor = append(c.processor,
+		func(resolver source.FileResolver, env Environment) []request {
+			var requests []request
+			for _, t := range types {
+				// TODO: add more trace logging here
+				matches, err := resolver.FilesByMIMEType(t)
+				if err != nil {
+					log.Warnf("unable to process mimetype=%q: %+v", t, err)
+					continue
+				}
+				requests = append(requests, makeRequests(parser, matches)...)
+			}
+			return requests
+		},
+	)
+	return c
+}
+
+func (c *Cataloger) WithParserByPath(parser Parser, paths ...string) *Cataloger {
+	c.processor = append(c.processor,
+		func(resolver source.FileResolver, env Environment) []request {
+			var requests []request
+			for _, g := range paths {
+				// TODO: add more trace logging here
+				matches, err := resolver.FilesByPath(g)
+				if err != nil {
+					log.Warnf("unable to process path=%q: %+v", g, err)
+					continue
+				}
+				requests = append(requests, makeRequests(parser, matches)...)
+			}
+			return requests
+		},
+	)
+	return c
+}
+
+func makeRequests(parser Parser, locations []source.Location) []request {
+	var requests []request
+	for _, l := range locations {
+		requests = append(requests, request{
+			Location: l,
+			Parser:   parser,
+		})
+	}
+	return requests
+}
+
+// NewCataloger if provided path-to-parser-function and glob-to-parser-function lookups creates a Cataloger
+func NewCataloger(upstreamCataloger string) *Cataloger {
+	return &Cataloger{
+		upstreamCataloger: upstreamCataloger,
+	}
+}
+
+// Name returns a string that uniquely describes the upstream cataloger that this Generic Cataloger represents.
+func (c *Cataloger) Name() string {
+	return c.upstreamCataloger
+}
+
+// Catalog is given an object to resolve file references and content, this function returns any discovered Packages after analyzing the catalog source.
+func (c *Cataloger) Catalog(resolver source.FileResolver) ([]pkg.Package, []artifact.Relationship, error) {
+	var packages []pkg.Package
+	var relationships []artifact.Relationship
+
+	logger := log.Nested("cataloger", c.upstreamCataloger)
+
+	env := Environment{
+		// TODO: consider passing into the cataloger, this would affect the cataloger interface (and all implementations). This can be deferred until later.
+		LinuxRelease: linux.IdentifyRelease(resolver),
+	}
+
+	for _, req := range c.selectFiles(resolver) {
+		location, parser := req.Location, req.Parser
+
+		contentReader, err := resolver.FileContentsByLocation(location)
+		if err != nil {
+			logger.WithFields("location", location.RealPath, "error", err).Warn("unable to fetch contents")
+			continue
+		}
+
+		discoveredPackages, discoveredRelationships, err := parser(resolver, &env, source.NewLocationReadCloser(location, contentReader))
+		internal.CloseAndLogError(contentReader, location.VirtualPath)
+		if err != nil {
+			logger.WithFields("location", location.RealPath, "error", err).Warnf("cataloger failed")
+			continue
+		}
+
+		for _, p := range discoveredPackages {
+			p.FoundBy = c.upstreamCataloger
+			packages = append(packages, p)
+		}
+
+		relationships = append(relationships, discoveredRelationships...)
+	}
+	return packages, relationships, nil
+}
+
+// selectFiles takes a set of file trees and resolves and file references of interest for future cataloging
+func (c *Cataloger) selectFiles(resolver source.FileResolver) []request {
+	var requests []request
+	for _, proc := range c.processor {
+		requests = append(requests, proc(resolver, Environment{})...)
+	}
+	return requests
+}

syft/pkg/cataloger/generic/cataloger_test.go

@@ -0,0 +1,87 @@
+package generic
+
+import (
+	"fmt"
+	"io/ioutil"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	"github.com/anchore/syft/syft/artifact"
+	"github.com/anchore/syft/syft/pkg"
+	"github.com/anchore/syft/syft/source"
+)
+
+func Test_Cataloger(t *testing.T) {
+	allParsedPaths := make(map[string]bool)
+	parser := func(resolver source.FileResolver, env *Environment, reader source.LocationReadCloser) ([]pkg.Package, []artifact.Relationship, error) {
+		allParsedPaths[reader.AccessPath()] = true
+		contents, err := ioutil.ReadAll(reader)
+		require.NoError(t, err)
+
+		if len(contents) == 0 {
+			return nil, nil, nil
+		}
+
+		p := pkg.Package{
+			Name:      string(contents),
+			Locations: source.NewLocationSet(reader.Location),
+		}
+		r := artifact.Relationship{
+			From: p,
+			To:   p,
+			Type: artifact.ContainsRelationship,
+		}
+
+		return []pkg.Package{p}, []artifact.Relationship{r}, nil
+	}
+
+	upstream := "some-other-cataloger"
+
+	expectedSelection := []string{"test-fixtures/last/path.txt", "test-fixtures/another-path.txt", "test-fixtures/a-path.txt", "test-fixtures/empty.txt"}
+	resolver := source.NewMockResolverForPaths(expectedSelection...)
+	cataloger := NewCataloger(upstream).
+		WithParserByPath(parser, "test-fixtures/another-path.txt", "test-fixtures/last/path.txt").
+		WithParserByGlobs(parser, "**/a-path.txt", "**/empty.txt")
+
+	actualPkgs, relationships, err := cataloger.Catalog(resolver)
+	assert.NoError(t, err)
+
+	expectedPkgs := make(map[string]pkg.Package)
+	for _, path := range expectedSelection {
+		require.True(t, allParsedPaths[path])
+		if path == "test-fixtures/empty.txt" {
+			continue // note: empty.txt won't become a package
+		}
+		expectedPkgs[path] = pkg.Package{
+			FoundBy: upstream,
+			Name:    fmt.Sprintf("%s file contents!", path),
+		}
+	}
+
+	assert.Len(t, allParsedPaths, len(expectedSelection))
+	assert.Len(t, actualPkgs, len(expectedPkgs))
+	assert.Len(t, relationships, len(actualPkgs))
+
+	for _, p := range actualPkgs {
+		ls := p.Locations.ToSlice()
+		require.NotEmpty(t, ls)
+		ref := ls[0]
+		exP, ok := expectedPkgs[ref.RealPath]
+		if !ok {
+			t.Errorf("missing expected pkg: ref=%+v", ref)
+			continue
+		}
+
+		// assigned by the generic cataloger
+		if p.FoundBy != exP.FoundBy {
+			t.Errorf("bad upstream: %s", p.FoundBy)
+		}
+
+		// assigned by the parser
+		if exP.Name != p.Name {
+			t.Errorf("bad contents mapping: %+v", p.Locations)
+		}
+	}
+}

syft/pkg/cataloger/generic/parser.go

@@ -0,0 +1,14 @@
+package generic
+
+import (
+	"github.com/anchore/syft/syft/artifact"
+	"github.com/anchore/syft/syft/linux"
+	"github.com/anchore/syft/syft/pkg"
+	"github.com/anchore/syft/syft/source"
+)
+
+type Environment struct {
+	LinuxRelease *linux.Release
+}
+
+type Parser func(source.FileResolver, *Environment, source.LocationReadCloser) ([]pkg.Package, []artifact.Relationship, error)

syft/pkg/cataloger/generic/test-fixtures/a-path.txt

@@ -0,0 +1 @@
+test-fixtures/a-path.txt file contents!

syft/pkg/cataloger/generic/test-fixtures/another-path.txt

@@ -0,0 +1 @@
+test-fixtures/another-path.txt file contents!

syft/pkg/cataloger/generic/test-fixtures/last/path.txt

@@ -0,0 +1 @@
+test-fixtures/last/path.txt file contents!