From 8095cd9980cb1eebd8b0f46d19b0989885fb82c6 Mon Sep 17 00:00:00 2001 From: Alex Goodman Date: Thu, 5 Nov 2020 13:46:20 -0500 Subject: [PATCH] add rpmdb file info to cataloger Signed-off-by: Alex Goodman --- go.mod | 2 +- go.sum | 2 + syft/cataloger/rpmdb/cataloger.go | 48 ++++++- syft/cataloger/rpmdb/parse_rpmdb.go | 31 ++++- syft/cataloger/rpmdb/parse_rpmdb_test.go | 163 +++++++++++++++++------ syft/pkg/rpmdb_metadata.go | 28 ++-- 6 files changed, 213 insertions(+), 61 deletions(-) diff --git a/go.mod b/go.mod index 7131866a6..1c3385836 100644 --- a/go.mod +++ b/go.mod @@ -11,7 +11,7 @@ require ( github.com/bmatcuk/doublestar v1.3.3 github.com/docker/docker v17.12.0-ce-rc1.0.20200309214505-aa6a9891b09c+incompatible github.com/dustin/go-humanize v1.0.0 - github.com/go-test/deep v1.0.6 + github.com/go-test/deep v1.0.7 github.com/google/uuid v1.1.1 github.com/gookit/color v1.2.7 github.com/hashicorp/go-multierror v1.1.0 diff --git a/go.sum b/go.sum index d1b9dcec4..71719b51e 100644 --- a/go.sum +++ b/go.sum @@ -299,6 +299,8 @@ github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LB github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/go-test/deep v1.0.6 h1:UHSEyLZUwX9Qoi99vVwvewiMC8mM2bf7XEM2nqvzEn8= github.com/go-test/deep v1.0.6/go.mod h1:QV8Hv/iy04NyLBxAdO9njL0iVPN1S4d/A3NVv1V36o8= +github.com/go-test/deep v1.0.7 h1:/VSMRlnY/JSyqxQUzQLKVMAskpY/NZKFA5j2P+0pP2M= +github.com/go-test/deep v1.0.7/go.mod h1:QV8Hv/iy04NyLBxAdO9njL0iVPN1S4d/A3NVv1V36o8= github.com/go-toolsmith/astcast v1.0.0 h1:JojxlmI6STnFVG9yOImLeGREv8W2ocNUM+iOhR6jE7g= github.com/go-toolsmith/astcast v1.0.0/go.mod h1:mt2OdQTeAQcY4DQgPSArJjHCcOwlX+Wl/kwN+LbLGQ4= github.com/go-toolsmith/astcopy v1.0.0 h1:OMgl1b1MEpjFQ1m5ztEO06rz5CUd3oBv9RF7+DyvdG8= diff --git a/syft/cataloger/rpmdb/cataloger.go b/syft/cataloger/rpmdb/cataloger.go index 7c331dade..25d57d610 100644 --- a/syft/cataloger/rpmdb/cataloger.go +++ b/syft/cataloger/rpmdb/cataloger.go @@ -4,13 +4,49 @@ Package rpmdb provides a concrete Cataloger implementation for RPM "Package" DB package rpmdb import ( - "github.com/anchore/syft/syft/cataloger/common" + "fmt" + "strings" + + "github.com/anchore/syft/syft/pkg" + "github.com/anchore/syft/syft/scope" ) +const ( + packagesGlob = "**/var/lib/rpm/Packages" +) + +type Cataloger struct{} + // NewRpmdbCataloger returns a new RPM DB cataloger object. -func NewRpmdbCataloger() *common.GenericCataloger { - globParsers := map[string]common.ParserFn{ - "**/var/lib/rpm/Packages": parseRpmDB, - } - return common.NewGenericCataloger(nil, globParsers, "rpmdb-cataloger") +func NewRpmdbCataloger() *Cataloger { + return &Cataloger{} +} + +// Name returns a string that uniquely describes a cataloger +func (c *Cataloger) Name() string { + return "rpmdb-cataloger" +} + +// Catalog is given an object to resolve file references and content, this function returns any discovered Packages after analyzing python egg and wheel installations. +func (c *Cataloger) Catalog(resolver scope.Resolver) ([]pkg.Package, error) { + + fileMatches, err := resolver.FilesByGlob(packagesGlob) + if err != nil { + return nil, fmt.Errorf("failed to find rpmdb's by glob") + } + + var pkgs []pkg.Package + for _, ref := range fileMatches { + + dbContents, err := resolver.FileContentsByRef(ref) + if err != nil { + return nil, err + } + + pkgs, err = parseRpmDB(resolver, strings.NewReader(dbContents)) + if err != nil { + return nil, fmt.Errorf("unable to catalog rpmdb package=%+v: %w", ref.Path, err) + } + } + return pkgs, nil } diff --git a/syft/cataloger/rpmdb/parse_rpmdb.go b/syft/cataloger/rpmdb/parse_rpmdb.go index efebae2a4..59bd1f222 100644 --- a/syft/cataloger/rpmdb/parse_rpmdb.go +++ b/syft/cataloger/rpmdb/parse_rpmdb.go @@ -6,18 +6,18 @@ import ( "io/ioutil" "os" + "github.com/anchore/stereoscope/pkg/file" + + "github.com/anchore/syft/syft/scope" + rpmdb "github.com/anchore/go-rpmdb/pkg" "github.com/anchore/syft/internal" "github.com/anchore/syft/internal/log" - "github.com/anchore/syft/syft/cataloger/common" "github.com/anchore/syft/syft/pkg" ) -// integrity check -var _ common.ParserFn = parseRpmDB - // parseApkDb parses an "Packages" RPM DB and returns the Packages listed within it. -func parseRpmDB(_ string, reader io.Reader) ([]pkg.Package, error) { +func parseRpmDB(resolver scope.FileResolver, reader io.Reader) ([]pkg.Package, error) { f, err := ioutil.TempFile("", internal.ApplicationName+"-rpmdb") if err != nil { return nil, fmt.Errorf("failed to create temp rpmdb file: %w", err) @@ -48,6 +48,26 @@ func parseRpmDB(_ string, reader io.Reader) ([]pkg.Package, error) { allPkgs := make([]pkg.Package, 0) for _, entry := range pkgList { + var records = make([]pkg.RpmdbFileRecord, 0) + + for _, record := range entry.Files { + refs, err := resolver.FilesByPath(file.Path(record.Path)) + if err != nil { + return nil, fmt.Errorf("failed to resolve path=%+v: %w", record.Path, err) + } + //only persist RPMDB file records which exist in the image/directory, otherwise ignore them + if len(refs) == 0 { + continue + } + + records = append(records, pkg.RpmdbFileRecord{ + Path: record.Path, + Mode: pkg.RpmdbFileMode(record.Mode), + Size: int(record.Size), + SHA256: record.SHA256, + }) + } + p := pkg.Package{ Name: entry.Name, Version: fmt.Sprintf("%s-%s", entry.Version, entry.Release), // this is what engine does @@ -64,6 +84,7 @@ func parseRpmDB(_ string, reader io.Reader) ([]pkg.Package, error) { Vendor: entry.Vendor, License: entry.License, Size: entry.Size, + Files: records, }, } diff --git a/syft/cataloger/rpmdb/parse_rpmdb_test.go b/syft/cataloger/rpmdb/parse_rpmdb_test.go index bf81f111f..13348f783 100644 --- a/syft/cataloger/rpmdb/parse_rpmdb_test.go +++ b/syft/cataloger/rpmdb/parse_rpmdb_test.go @@ -1,58 +1,141 @@ package rpmdb import ( + "fmt" "os" "testing" + "github.com/anchore/stereoscope/pkg/file" "github.com/anchore/syft/syft/pkg" "github.com/go-test/deep" ) +type rpmdbTestFileResolverMock struct { + ignorePaths bool +} + +func newTestFileResolver(ignorePaths bool) *rpmdbTestFileResolverMock { + return &rpmdbTestFileResolverMock{ + ignorePaths: ignorePaths, + } +} + +func (r *rpmdbTestFileResolverMock) FilesByPath(paths ...file.Path) ([]file.Reference, error) { + if r.ignorePaths { + // act as if no paths exist + return nil, nil + } + // act as if all files exist + var refs = make([]file.Reference, len(paths)) + for i, p := range paths { + refs[i] = file.NewFileReference(p) + } + return refs, nil +} + +func (r *rpmdbTestFileResolverMock) FilesByGlob(_ ...string) ([]file.Reference, error) { + return nil, fmt.Errorf("not implemented") +} +func (r *rpmdbTestFileResolverMock) RelativeFileByPath(_ file.Reference, path string) (*file.Reference, error) { + return nil, fmt.Errorf("not implemented") +} + func TestParseRpmDB(t *testing.T) { - expected := map[string]pkg.Package{ - "dive": { - Name: "dive", - Version: "0.9.2-1", - Type: pkg.RpmPkg, - MetadataType: pkg.RpmdbMetadataType, - Metadata: pkg.RpmdbMetadata{ - Name: "dive", - Epoch: 0, - Arch: "x86_64", - Release: "1", - Version: "0.9.2", - SourceRpm: "dive-0.9.2-1.src.rpm", - Size: 12406784, - License: "MIT", - Vendor: "", + tests := []struct { + fixture string + expected map[string]pkg.Package + ignorePaths bool + }{ + { + fixture: "test-fixtures/Packages", + // we only surface package paths for files that exist (here we DO NOT expect a path) + ignorePaths: true, + expected: map[string]pkg.Package{ + "dive": { + Name: "dive", + Version: "0.9.2-1", + Type: pkg.RpmPkg, + MetadataType: pkg.RpmdbMetadataType, + Metadata: pkg.RpmdbMetadata{ + Name: "dive", + Epoch: 0, + Arch: "x86_64", + Release: "1", + Version: "0.9.2", + SourceRpm: "dive-0.9.2-1.src.rpm", + Size: 12406784, + License: "MIT", + Vendor: "", + Files: []pkg.RpmdbFileRecord{}, + }, + }, + }, + }, + { + fixture: "test-fixtures/Packages", + // we only surface package paths for files that exist (here we expect a path) + ignorePaths: false, + expected: map[string]pkg.Package{ + "dive": { + Name: "dive", + Version: "0.9.2-1", + Type: pkg.RpmPkg, + MetadataType: pkg.RpmdbMetadataType, + Metadata: pkg.RpmdbMetadata{ + Name: "dive", + Epoch: 0, + Arch: "x86_64", + Release: "1", + Version: "0.9.2", + SourceRpm: "dive-0.9.2-1.src.rpm", + Size: 12406784, + License: "MIT", + Vendor: "", + Files: []pkg.RpmdbFileRecord{ + { + Path: "/usr/local/bin/dive", + Mode: 33261, + Size: 12406784, + SHA256: "81d29f327ba23096b3c52ff6fe1c425641e618bc87b5c05ee377edc650afaa55", + }, + }, + }, + }, }, }, } - fixture, err := os.Open("test-fixtures/Packages") - if err != nil { - t.Fatalf("failed to open fixture: %+v", err) - } - - actual, err := parseRpmDB(fixture.Name(), fixture) - if err != nil { - t.Fatalf("failed to parse rpmdb: %+v", err) - } - - if len(actual) != len(expected) { - for _, a := range actual { - t.Log(" ", a) - } - t.Fatalf("unexpected package count: %d!=%d", len(actual), len(expected)) - } - - for _, a := range actual { - e := expected[a.Name] - diffs := deep.Equal(a, e) - if len(diffs) > 0 { - for _, d := range diffs { - t.Errorf("diff: %+v", d) + for _, test := range tests { + t.Run(test.fixture, func(t *testing.T) { + fixture, err := os.Open(test.fixture) + if err != nil { + t.Fatalf("failed to open fixture: %+v", err) } - } + + fileResolver := newTestFileResolver(test.ignorePaths) + + actual, err := parseRpmDB(fileResolver, fixture) + if err != nil { + t.Fatalf("failed to parse rpmdb: %+v", err) + } + + if len(actual) != len(test.expected) { + for _, a := range actual { + t.Log(" ", a) + } + t.Fatalf("unexpected package count: %d!=%d", len(actual), len(test.expected)) + } + + for _, a := range actual { + e := test.expected[a.Name] + diffs := deep.Equal(a, e) + if len(diffs) > 0 { + for _, d := range diffs { + t.Errorf("diff: %+v", d) + } + } + } + }) } + } diff --git a/syft/pkg/rpmdb_metadata.go b/syft/pkg/rpmdb_metadata.go index 85b7d9bda..c9823c1a8 100644 --- a/syft/pkg/rpmdb_metadata.go +++ b/syft/pkg/rpmdb_metadata.go @@ -9,17 +9,27 @@ import ( // RpmdbMetadata represents all captured data for a RPM DB package entry. type RpmdbMetadata struct { - Name string `json:"name"` - Version string `json:"version"` - Epoch int `json:"epoch"` - Arch string `json:"architecture"` - Release string `json:"release"` - SourceRpm string `json:"sourceRpm"` - Size int `json:"size"` - License string `json:"license"` - Vendor string `json:"vendor"` + Name string `json:"name"` + Version string `json:"version"` + Epoch int `json:"epoch"` + Arch string `json:"architecture"` + Release string `json:"release"` + SourceRpm string `json:"sourceRpm"` + Size int `json:"size"` + License string `json:"license"` + Vendor string `json:"vendor"` + Files []RpmdbFileRecord `json:"files"` } +type RpmdbFileRecord struct { + Path string `json:"path"` + Mode RpmdbFileMode `json:"mode"` + Size int `json:"size"` + SHA256 string `json:"sha256"` +} + +type RpmdbFileMode uint16 + func (m RpmdbMetadata) PackageURL(d distro.Distro) string { pURL := packageurl.NewPackageURL( packageurl.TypeRPM,