Upgrade generic cataloger (#1281)

* add second generation of generic cataloger

Signed-off-by: Alex Goodman <alex.goodman@anchore.com>

* upgrade aplm cataloger to use generic.Cataloger

Signed-off-by: Alex Goodman <alex.goodman@anchore.com>

* remove pacakge found-by attribute from the definition of a package ID

Signed-off-by: Alex Goodman <alex.goodman@anchore.com>

Signed-off-by: Alex Goodman <alex.goodman@anchore.com>
This commit is contained in:
Alex Goodman
2022-10-24 11:12:12 -04:00
committed by GitHub
parent 7a8b96abc2
commit b44f441c82
40 changed files with 495 additions and 245 deletions
+2 -30
View File
@@ -6,11 +6,11 @@ import (
"github.com/scylladb/go-set/strset"
"github.com/anchore/packageurl-go"
"github.com/anchore/syft/syft/file"
"github.com/anchore/syft/syft/linux"
)
var _ FileOwner = (*AlpmMetadata)(nil)
const AlpmDBGlob = "**/var/lib/pacman/local/**/desc"
type AlpmMetadata struct {
@@ -40,34 +40,6 @@ type AlpmFileRecord struct {
Digests []file.Digest `mapstructure:"digests" json:"digest,omitempty"`
}
// PackageURL returns the PURL for the specific Arch Linux package (see https://github.com/package-url/purl-spec)
func (m AlpmMetadata) PackageURL(distro *linux.Release) string {
qualifiers := map[string]string{
PURLQualifierArch: m.Architecture,
}
if m.BasePackage != "" {
qualifiers[PURLQualifierUpstream] = m.BasePackage
}
distroID := ""
if distro != nil {
distroID = distro.ID
}
return packageurl.NewPackageURL(
"alpm",
distroID,
m.Package,
m.Version,
purlQualifiers(
qualifiers,
distro,
),
"",
).ToString()
}
func (m AlpmMetadata) OwnedFiles() (result []string) {
s := strset.New()
for _, f := range m.Files {
+1 -1
View File
@@ -65,7 +65,7 @@ func (m ApkMetadata) PackageURL(distro *linux.Release) string {
"",
m.Package,
m.Version,
purlQualifiers(
PURLQualifiers(
qualifiers,
distro,
),
+4 -39
View File
@@ -1,48 +1,13 @@
package alpm
import (
"fmt"
"github.com/anchore/syft/internal"
"github.com/anchore/syft/syft/artifact"
"github.com/anchore/syft/syft/pkg"
"github.com/anchore/syft/syft/source"
"github.com/anchore/syft/syft/pkg/cataloger/generic"
)
const catalogerName = "alpmdb-cataloger"
type Cataloger struct{}
// NewAlpmdbCataloger returns a new ALPM DB cataloger object.
func NewAlpmdbCataloger() *Cataloger {
return &Cataloger{}
}
// Name returns a string that uniquely describes a cataloger
func (c *Cataloger) Name() string {
return catalogerName
}
// Catalog is given an object to resolve file references and content, this function returns any discovered Packages after analyzing rpm db installation.
func (c *Cataloger) Catalog(resolver source.FileResolver) ([]pkg.Package, []artifact.Relationship, error) {
fileMatches, err := resolver.FilesByGlob(pkg.AlpmDBGlob)
if err != nil {
return nil, nil, fmt.Errorf("failed to find rpmdb's by glob: %w", err)
}
var pkgs []pkg.Package
for _, location := range fileMatches {
dbContentReader, err := resolver.FileContentsByLocation(location)
if err != nil {
return nil, nil, err
}
discoveredPkgs, err := parseAlpmDB(resolver, location.RealPath, dbContentReader)
internal.CloseAndLogError(dbContentReader, location.VirtualPath)
if err != nil {
return nil, nil, fmt.Errorf("unable to catalog package=%+v: %w", location.RealPath, err)
}
pkgs = append(pkgs, discoveredPkgs...)
}
return pkgs, nil, nil
func NewAlpmdbCataloger() *generic.Cataloger {
return generic.NewCataloger(catalogerName).
WithParserByGlobs(parseAlpmDB, pkg.AlpmDBGlob)
}
+52
View File
@@ -0,0 +1,52 @@
package alpm
import (
"strings"
"github.com/anchore/packageurl-go"
"github.com/anchore/syft/syft/linux"
"github.com/anchore/syft/syft/pkg"
"github.com/anchore/syft/syft/source"
)
func newPackage(m pkg.AlpmMetadata, release *linux.Release, locations ...source.Location) pkg.Package {
p := pkg.Package{
Name: m.Package,
Version: m.Version,
Locations: source.NewLocationSet(locations...),
Type: pkg.AlpmPkg,
Licenses: strings.Split(m.License, " "),
PURL: packageURL(m, release),
MetadataType: pkg.AlpmMetadataType,
Metadata: m,
}
p.SetID()
return p
}
func packageURL(m pkg.AlpmMetadata, distro *linux.Release) string {
if distro == nil || distro.ID != "arch" {
// note: there is no namespace variation (like with debian ID_LIKE for ubuntu ID, for example)
return ""
}
qualifiers := map[string]string{
pkg.PURLQualifierArch: m.Architecture,
}
if m.BasePackage != "" {
qualifiers[pkg.PURLQualifierUpstream] = m.BasePackage
}
return packageurl.NewPackageURL(
"alpm", // `alpm` for Arch Linux and other users of the libalpm/pacman package manager. (see https://github.com/package-url/purl-spec/pull/164)
distro.ID,
m.Package,
m.Version,
pkg.PURLQualifiers(
qualifiers,
distro,
),
"",
).ToString()
}
@@ -1,4 +1,4 @@
package pkg
package alpm
import (
"testing"
@@ -7,18 +7,32 @@ import (
"github.com/anchore/packageurl-go"
"github.com/anchore/syft/syft/linux"
"github.com/anchore/syft/syft/pkg"
)
func TestAlpmMetadata_pURL(t *testing.T) {
func Test_PackageURL(t *testing.T) {
tests := []struct {
name string
metadata AlpmMetadata
metadata pkg.AlpmMetadata
distro linux.Release
expected string
}{
{
name: "bad distro id",
metadata: pkg.AlpmMetadata{
Package: "p",
Version: "v",
Architecture: "a",
},
distro: linux.Release{
ID: "something-else",
BuildID: "rolling",
},
expected: "",
},
{
name: "gocase",
metadata: AlpmMetadata{
metadata: pkg.AlpmMetadata{
Package: "p",
Version: "v",
Architecture: "a",
@@ -31,7 +45,7 @@ func TestAlpmMetadata_pURL(t *testing.T) {
},
{
name: "missing architecture",
metadata: AlpmMetadata{
metadata: pkg.AlpmMetadata{
Package: "p",
Version: "v",
},
@@ -41,7 +55,7 @@ func TestAlpmMetadata_pURL(t *testing.T) {
expected: "pkg:alpm/arch/p@v?distro=arch",
},
{
metadata: AlpmMetadata{
metadata: pkg.AlpmMetadata{
Package: "python",
Version: "3.10.0",
Architecture: "any",
@@ -53,7 +67,7 @@ func TestAlpmMetadata_pURL(t *testing.T) {
expected: "pkg:alpm/arch/python@3.10.0?arch=any&distro=arch-rolling",
},
{
metadata: AlpmMetadata{
metadata: pkg.AlpmMetadata{
Package: "g plus plus",
Version: "v84",
Architecture: "x86_64",
@@ -66,7 +80,7 @@ func TestAlpmMetadata_pURL(t *testing.T) {
},
{
name: "add source information as qualifier",
metadata: AlpmMetadata{
metadata: pkg.AlpmMetadata{
Package: "p",
Version: "v",
Architecture: "a",
@@ -82,12 +96,17 @@ func TestAlpmMetadata_pURL(t *testing.T) {
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
actual := test.metadata.PackageURL(&test.distro)
actual := packageURL(test.metadata, &test.distro)
if actual != test.expected {
dmp := diffmatchpatch.New()
diffs := dmp.DiffMain(test.expected, actual, true)
t.Errorf("diff: %s", dmp.DiffPrettyText(diffs))
}
if test.expected == "" {
return
}
// verify packageurl can parse
purl, err := packageurl.FromString(actual)
if err != nil {
+47 -59
View File
@@ -13,11 +13,15 @@ import (
"github.com/mitchellh/mapstructure"
"github.com/vbatts/go-mtree"
"github.com/anchore/syft/syft/artifact"
"github.com/anchore/syft/syft/file"
"github.com/anchore/syft/syft/pkg"
"github.com/anchore/syft/syft/pkg/cataloger/generic"
"github.com/anchore/syft/syft/source"
)
var _ generic.Parser = parseAlpmDB
var (
ignoredFiles = map[string]bool{
"/set": true,
@@ -27,16 +31,50 @@ var (
}
)
func newAlpmDBPackage(d *pkg.AlpmMetadata) *pkg.Package {
return &pkg.Package{
Name: d.Package,
Version: d.Version,
FoundBy: catalogerName,
Type: "alpm",
Licenses: strings.Split(d.License, " "),
MetadataType: pkg.AlpmMetadataType,
Metadata: *d,
func parseAlpmDB(resolver source.FileResolver, env *generic.Environment, reader source.LocationReadCloser) ([]pkg.Package, []artifact.Relationship, error) {
metadata, err := parseAlpmDBEntry(reader)
if err != nil {
return nil, nil, err
}
base := filepath.Dir(reader.RealPath)
r, err := getFileReader(filepath.Join(base, "mtree"), resolver)
if err != nil {
return nil, nil, err
}
pkgFiles, err := parseMtree(r)
if err != nil {
return nil, nil, err
}
// The replace the files found the the pacman database with the files from the mtree These contain more metadata and
// thus more useful.
metadata.Files = pkgFiles
// We only really do this to get any backup database entries from the files database
files := filepath.Join(base, "files")
_, err = getFileReader(files, resolver)
if err != nil {
return nil, nil, err
}
filesMetadata, err := parseAlpmDBEntry(reader)
if err != nil {
return nil, nil, err
} else if filesMetadata != nil {
metadata.Backup = filesMetadata.Backup
}
return []pkg.Package{
newPackage(*metadata, env.LinuxRelease, reader.Location),
}, nil, nil
}
func parseAlpmDBEntry(reader io.Reader) (*pkg.AlpmMetadata, error) {
scanner := newScanner(reader)
metadata, err := parseDatabase(scanner)
if err != nil {
return nil, err
}
return metadata, nil
}
func newScanner(reader io.Reader) *bufio.Scanner {
@@ -194,53 +232,3 @@ func parseMtree(r io.Reader) ([]pkg.AlpmFileRecord, error) {
}
return entries, nil
}
func parseAlpmDBEntry(reader io.Reader) (*pkg.AlpmMetadata, error) {
scanner := newScanner(reader)
metadata, err := parseDatabase(scanner)
if err != nil {
return nil, err
}
if metadata == nil {
return nil, nil
}
return metadata, nil
}
func parseAlpmDB(resolver source.FileResolver, desc string, reader io.Reader) ([]pkg.Package, error) {
metadata, err := parseAlpmDBEntry(reader)
if err != nil {
return nil, err
}
base := filepath.Dir(desc)
mtree := filepath.Join(base, "mtree")
r, err := getFileReader(mtree, resolver)
if err != nil {
return nil, err
}
pkgFiles, err := parseMtree(r)
if err != nil {
return nil, err
}
// The replace the files found the the pacman database with the files from the mtree These contain more metadata and
// thus more useful.
metadata.Files = pkgFiles
// We only really do this to get any backup database entries from the files database
files := filepath.Join(base, "files")
_, err = getFileReader(files, resolver)
if err != nil {
return nil, err
}
filesMetadata, err := parseAlpmDBEntry(reader)
if err != nil {
return nil, err
} else if filesMetadata != nil {
metadata.Backup = filesMetadata.Backup
}
p := *newAlpmDBPackage(metadata)
p.SetID()
return []pkg.Package{p}, nil
}
+151
View File
@@ -0,0 +1,151 @@
package generic
import (
"github.com/anchore/syft/internal"
"github.com/anchore/syft/internal/log"
"github.com/anchore/syft/syft/artifact"
"github.com/anchore/syft/syft/linux"
"github.com/anchore/syft/syft/pkg"
"github.com/anchore/syft/syft/source"
)
type processor func(resolver source.FileResolver, env Environment) []request
type request struct {
source.Location
Parser
}
// Cataloger implements the Catalog interface and is responsible for dispatching the proper parser function for
// a given path or glob pattern. This is intended to be reusable across many package cataloger types.
type Cataloger struct {
processor []processor
upstreamCataloger string
}
func (c *Cataloger) WithParserByGlobs(parser Parser, globs ...string) *Cataloger {
c.processor = append(c.processor,
func(resolver source.FileResolver, env Environment) []request {
var requests []request
for _, g := range globs {
// TODO: add more trace logging here
matches, err := resolver.FilesByGlob(g)
if err != nil {
log.Warnf("unable to process glob=%q: %+v", g, err)
continue
}
requests = append(requests, makeRequests(parser, matches)...)
}
return requests
},
)
return c
}
func (c *Cataloger) WithParserByMimeTypes(parser Parser, types ...string) *Cataloger {
c.processor = append(c.processor,
func(resolver source.FileResolver, env Environment) []request {
var requests []request
for _, t := range types {
// TODO: add more trace logging here
matches, err := resolver.FilesByMIMEType(t)
if err != nil {
log.Warnf("unable to process mimetype=%q: %+v", t, err)
continue
}
requests = append(requests, makeRequests(parser, matches)...)
}
return requests
},
)
return c
}
func (c *Cataloger) WithParserByPath(parser Parser, paths ...string) *Cataloger {
c.processor = append(c.processor,
func(resolver source.FileResolver, env Environment) []request {
var requests []request
for _, g := range paths {
// TODO: add more trace logging here
matches, err := resolver.FilesByPath(g)
if err != nil {
log.Warnf("unable to process path=%q: %+v", g, err)
continue
}
requests = append(requests, makeRequests(parser, matches)...)
}
return requests
},
)
return c
}
func makeRequests(parser Parser, locations []source.Location) []request {
var requests []request
for _, l := range locations {
requests = append(requests, request{
Location: l,
Parser: parser,
})
}
return requests
}
// NewCataloger if provided path-to-parser-function and glob-to-parser-function lookups creates a Cataloger
func NewCataloger(upstreamCataloger string) *Cataloger {
return &Cataloger{
upstreamCataloger: upstreamCataloger,
}
}
// Name returns a string that uniquely describes the upstream cataloger that this Generic Cataloger represents.
func (c *Cataloger) Name() string {
return c.upstreamCataloger
}
// Catalog is given an object to resolve file references and content, this function returns any discovered Packages after analyzing the catalog source.
func (c *Cataloger) Catalog(resolver source.FileResolver) ([]pkg.Package, []artifact.Relationship, error) {
var packages []pkg.Package
var relationships []artifact.Relationship
logger := log.Nested("cataloger", c.upstreamCataloger)
env := Environment{
// TODO: consider passing into the cataloger, this would affect the cataloger interface (and all implementations). This can be deferred until later.
LinuxRelease: linux.IdentifyRelease(resolver),
}
for _, req := range c.selectFiles(resolver) {
location, parser := req.Location, req.Parser
contentReader, err := resolver.FileContentsByLocation(location)
if err != nil {
logger.WithFields("location", location.RealPath, "error", err).Warn("unable to fetch contents")
continue
}
discoveredPackages, discoveredRelationships, err := parser(resolver, &env, source.NewLocationReadCloser(location, contentReader))
internal.CloseAndLogError(contentReader, location.VirtualPath)
if err != nil {
logger.WithFields("location", location.RealPath, "error", err).Warnf("cataloger failed")
continue
}
for _, p := range discoveredPackages {
p.FoundBy = c.upstreamCataloger
packages = append(packages, p)
}
relationships = append(relationships, discoveredRelationships...)
}
return packages, relationships, nil
}
// selectFiles takes a set of file trees and resolves and file references of interest for future cataloging
func (c *Cataloger) selectFiles(resolver source.FileResolver) []request {
var requests []request
for _, proc := range c.processor {
requests = append(requests, proc(resolver, Environment{})...)
}
return requests
}
@@ -0,0 +1,87 @@
package generic
import (
"fmt"
"io/ioutil"
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/anchore/syft/syft/artifact"
"github.com/anchore/syft/syft/pkg"
"github.com/anchore/syft/syft/source"
)
func Test_Cataloger(t *testing.T) {
allParsedPaths := make(map[string]bool)
parser := func(resolver source.FileResolver, env *Environment, reader source.LocationReadCloser) ([]pkg.Package, []artifact.Relationship, error) {
allParsedPaths[reader.AccessPath()] = true
contents, err := ioutil.ReadAll(reader)
require.NoError(t, err)
if len(contents) == 0 {
return nil, nil, nil
}
p := pkg.Package{
Name: string(contents),
Locations: source.NewLocationSet(reader.Location),
}
r := artifact.Relationship{
From: p,
To: p,
Type: artifact.ContainsRelationship,
}
return []pkg.Package{p}, []artifact.Relationship{r}, nil
}
upstream := "some-other-cataloger"
expectedSelection := []string{"test-fixtures/last/path.txt", "test-fixtures/another-path.txt", "test-fixtures/a-path.txt", "test-fixtures/empty.txt"}
resolver := source.NewMockResolverForPaths(expectedSelection...)
cataloger := NewCataloger(upstream).
WithParserByPath(parser, "test-fixtures/another-path.txt", "test-fixtures/last/path.txt").
WithParserByGlobs(parser, "**/a-path.txt", "**/empty.txt")
actualPkgs, relationships, err := cataloger.Catalog(resolver)
assert.NoError(t, err)
expectedPkgs := make(map[string]pkg.Package)
for _, path := range expectedSelection {
require.True(t, allParsedPaths[path])
if path == "test-fixtures/empty.txt" {
continue // note: empty.txt won't become a package
}
expectedPkgs[path] = pkg.Package{
FoundBy: upstream,
Name: fmt.Sprintf("%s file contents!", path),
}
}
assert.Len(t, allParsedPaths, len(expectedSelection))
assert.Len(t, actualPkgs, len(expectedPkgs))
assert.Len(t, relationships, len(actualPkgs))
for _, p := range actualPkgs {
ls := p.Locations.ToSlice()
require.NotEmpty(t, ls)
ref := ls[0]
exP, ok := expectedPkgs[ref.RealPath]
if !ok {
t.Errorf("missing expected pkg: ref=%+v", ref)
continue
}
// assigned by the generic cataloger
if p.FoundBy != exP.FoundBy {
t.Errorf("bad upstream: %s", p.FoundBy)
}
// assigned by the parser
if exP.Name != p.Name {
t.Errorf("bad contents mapping: %+v", p.Locations)
}
}
}
+14
View File
@@ -0,0 +1,14 @@
package generic
import (
"github.com/anchore/syft/syft/artifact"
"github.com/anchore/syft/syft/linux"
"github.com/anchore/syft/syft/pkg"
"github.com/anchore/syft/syft/source"
)
type Environment struct {
LinuxRelease *linux.Release
}
type Parser func(source.FileResolver, *Environment, source.LocationReadCloser) ([]pkg.Package, []artifact.Relationship, error)
@@ -0,0 +1 @@
test-fixtures/a-path.txt file contents!
@@ -0,0 +1 @@
test-fixtures/another-path.txt file contents!
@@ -0,0 +1 @@
test-fixtures/last/path.txt file contents!
+1 -1
View File
@@ -65,7 +65,7 @@ func (m DpkgMetadata) PackageURL(distro *linux.Release) string {
namespace,
m.Package,
m.Version,
purlQualifiers(
PURLQualifiers(
qualifiers,
distro,
),
+1 -1
View File
@@ -17,7 +17,7 @@ type Package struct {
id artifact.ID `hash:"ignore"`
Name string // the package name
Version string // the version of the package
FoundBy string `cyclonedx:"foundBy"` // the specific cataloger that discovered this package
FoundBy string `hash:"ignore" cyclonedx:"foundBy"` // the specific cataloger that discovered this package
Locations source.LocationSet // the locations that lead to the discovery of this package (note: this is not necessarily the locations that make up this package)
Licenses []string // licenses discovered with the package metadata
Language Language `cyclonedx:"language"` // the language ecosystem this package belongs to (e.g. JavaScript, Python, etc)
-8
View File
@@ -190,14 +190,6 @@ func TestIDUniqueness(t *testing.T) {
},
expectedIDComparison: assert.NotEqual,
},
{
name: "foundBy is reflected",
transform: func(pkg Package) Package {
pkg.FoundBy = "new!"
return pkg
},
expectedIDComparison: assert.NotEqual,
},
{
name: "metadata mutation is reflected",
transform: func(pkg Package) Package {
+1 -1
View File
@@ -80,7 +80,7 @@ func (m RpmMetadata) PackageURL(distro *linux.Release) string {
// for purl the epoch is a qualifier, not part of the version
// see https://github.com/package-url/purl-spec/blob/master/PURL-TYPES.rst under the RPM section
fmt.Sprintf("%s-%s", m.Version, m.Release),
purlQualifiers(
PURLQualifiers(
qualifiers,
distro,
),
+1 -1
View File
@@ -67,7 +67,7 @@ func URL(p Package, release *linux.Release) string {
).ToString()
}
func purlQualifiers(vars map[string]string, release *linux.Release) (q packageurl.Qualifiers) {
func PURLQualifiers(vars map[string]string, release *linux.Release) (q packageurl.Qualifiers) {
keys := make([]string, 0, len(vars))
for k := range vars {
keys = append(keys, k)
+1 -18
View File
@@ -200,24 +200,6 @@ func TestPackageURL(t *testing.T) {
expected: "pkg:maven/g.id/a@v",
},
{
name: "alpm",
distro: &linux.Release{
ID: "arch",
BuildID: "rolling",
},
pkg: Package{
Name: "linux",
Version: "5.10.0",
Type: AlpmPkg,
Metadata: AlpmMetadata{
Package: "linux",
Version: "5.10.0",
},
},
expected: "pkg:alpm/arch/linux@5.10.0?distro=arch-rolling",
},
{
name: "cocoapods",
pkg: Package{
@@ -288,6 +270,7 @@ func TestPackageURL(t *testing.T) {
// testing microsoft packages is not valid for purl at this time
expectedTypes.Remove(string(KbPkg))
expectedTypes.Remove(string(PortagePkg))
expectedTypes.Remove(string(AlpmPkg))
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {