From 8095cd9980cb1eebd8b0f46d19b0989885fb82c6 Mon Sep 17 00:00:00 2001 From: Alex Goodman Date: Thu, 5 Nov 2020 13:46:20 -0500 Subject: [PATCH 1/3] add rpmdb file info to cataloger Signed-off-by: Alex Goodman --- go.mod | 2 +- go.sum | 2 + syft/cataloger/rpmdb/cataloger.go | 48 ++++++- syft/cataloger/rpmdb/parse_rpmdb.go | 31 ++++- syft/cataloger/rpmdb/parse_rpmdb_test.go | 163 +++++++++++++++++------ syft/pkg/rpmdb_metadata.go | 28 ++-- 6 files changed, 213 insertions(+), 61 deletions(-) diff --git a/go.mod b/go.mod index 7131866a6..1c3385836 100644 --- a/go.mod +++ b/go.mod @@ -11,7 +11,7 @@ require ( github.com/bmatcuk/doublestar v1.3.3 github.com/docker/docker v17.12.0-ce-rc1.0.20200309214505-aa6a9891b09c+incompatible github.com/dustin/go-humanize v1.0.0 - github.com/go-test/deep v1.0.6 + github.com/go-test/deep v1.0.7 github.com/google/uuid v1.1.1 github.com/gookit/color v1.2.7 github.com/hashicorp/go-multierror v1.1.0 diff --git a/go.sum b/go.sum index d1b9dcec4..71719b51e 100644 --- a/go.sum +++ b/go.sum @@ -299,6 +299,8 @@ github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LB github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/go-test/deep v1.0.6 h1:UHSEyLZUwX9Qoi99vVwvewiMC8mM2bf7XEM2nqvzEn8= github.com/go-test/deep v1.0.6/go.mod h1:QV8Hv/iy04NyLBxAdO9njL0iVPN1S4d/A3NVv1V36o8= +github.com/go-test/deep v1.0.7 h1:/VSMRlnY/JSyqxQUzQLKVMAskpY/NZKFA5j2P+0pP2M= +github.com/go-test/deep v1.0.7/go.mod h1:QV8Hv/iy04NyLBxAdO9njL0iVPN1S4d/A3NVv1V36o8= github.com/go-toolsmith/astcast v1.0.0 h1:JojxlmI6STnFVG9yOImLeGREv8W2ocNUM+iOhR6jE7g= github.com/go-toolsmith/astcast v1.0.0/go.mod h1:mt2OdQTeAQcY4DQgPSArJjHCcOwlX+Wl/kwN+LbLGQ4= github.com/go-toolsmith/astcopy v1.0.0 h1:OMgl1b1MEpjFQ1m5ztEO06rz5CUd3oBv9RF7+DyvdG8= diff --git a/syft/cataloger/rpmdb/cataloger.go b/syft/cataloger/rpmdb/cataloger.go index 7c331dade..25d57d610 100644 --- a/syft/cataloger/rpmdb/cataloger.go +++ b/syft/cataloger/rpmdb/cataloger.go @@ -4,13 +4,49 @@ Package rpmdb provides a concrete Cataloger implementation for RPM "Package" DB package rpmdb import ( - "github.com/anchore/syft/syft/cataloger/common" + "fmt" + "strings" + + "github.com/anchore/syft/syft/pkg" + "github.com/anchore/syft/syft/scope" ) +const ( + packagesGlob = "**/var/lib/rpm/Packages" +) + +type Cataloger struct{} + // NewRpmdbCataloger returns a new RPM DB cataloger object. -func NewRpmdbCataloger() *common.GenericCataloger { - globParsers := map[string]common.ParserFn{ - "**/var/lib/rpm/Packages": parseRpmDB, - } - return common.NewGenericCataloger(nil, globParsers, "rpmdb-cataloger") +func NewRpmdbCataloger() *Cataloger { + return &Cataloger{} +} + +// Name returns a string that uniquely describes a cataloger +func (c *Cataloger) Name() string { + return "rpmdb-cataloger" +} + +// Catalog is given an object to resolve file references and content, this function returns any discovered Packages after analyzing python egg and wheel installations. +func (c *Cataloger) Catalog(resolver scope.Resolver) ([]pkg.Package, error) { + + fileMatches, err := resolver.FilesByGlob(packagesGlob) + if err != nil { + return nil, fmt.Errorf("failed to find rpmdb's by glob") + } + + var pkgs []pkg.Package + for _, ref := range fileMatches { + + dbContents, err := resolver.FileContentsByRef(ref) + if err != nil { + return nil, err + } + + pkgs, err = parseRpmDB(resolver, strings.NewReader(dbContents)) + if err != nil { + return nil, fmt.Errorf("unable to catalog rpmdb package=%+v: %w", ref.Path, err) + } + } + return pkgs, nil } diff --git a/syft/cataloger/rpmdb/parse_rpmdb.go b/syft/cataloger/rpmdb/parse_rpmdb.go index efebae2a4..59bd1f222 100644 --- a/syft/cataloger/rpmdb/parse_rpmdb.go +++ b/syft/cataloger/rpmdb/parse_rpmdb.go @@ -6,18 +6,18 @@ import ( "io/ioutil" "os" + "github.com/anchore/stereoscope/pkg/file" + + "github.com/anchore/syft/syft/scope" + rpmdb "github.com/anchore/go-rpmdb/pkg" "github.com/anchore/syft/internal" "github.com/anchore/syft/internal/log" - "github.com/anchore/syft/syft/cataloger/common" "github.com/anchore/syft/syft/pkg" ) -// integrity check -var _ common.ParserFn = parseRpmDB - // parseApkDb parses an "Packages" RPM DB and returns the Packages listed within it. -func parseRpmDB(_ string, reader io.Reader) ([]pkg.Package, error) { +func parseRpmDB(resolver scope.FileResolver, reader io.Reader) ([]pkg.Package, error) { f, err := ioutil.TempFile("", internal.ApplicationName+"-rpmdb") if err != nil { return nil, fmt.Errorf("failed to create temp rpmdb file: %w", err) @@ -48,6 +48,26 @@ func parseRpmDB(_ string, reader io.Reader) ([]pkg.Package, error) { allPkgs := make([]pkg.Package, 0) for _, entry := range pkgList { + var records = make([]pkg.RpmdbFileRecord, 0) + + for _, record := range entry.Files { + refs, err := resolver.FilesByPath(file.Path(record.Path)) + if err != nil { + return nil, fmt.Errorf("failed to resolve path=%+v: %w", record.Path, err) + } + //only persist RPMDB file records which exist in the image/directory, otherwise ignore them + if len(refs) == 0 { + continue + } + + records = append(records, pkg.RpmdbFileRecord{ + Path: record.Path, + Mode: pkg.RpmdbFileMode(record.Mode), + Size: int(record.Size), + SHA256: record.SHA256, + }) + } + p := pkg.Package{ Name: entry.Name, Version: fmt.Sprintf("%s-%s", entry.Version, entry.Release), // this is what engine does @@ -64,6 +84,7 @@ func parseRpmDB(_ string, reader io.Reader) ([]pkg.Package, error) { Vendor: entry.Vendor, License: entry.License, Size: entry.Size, + Files: records, }, } diff --git a/syft/cataloger/rpmdb/parse_rpmdb_test.go b/syft/cataloger/rpmdb/parse_rpmdb_test.go index bf81f111f..13348f783 100644 --- a/syft/cataloger/rpmdb/parse_rpmdb_test.go +++ b/syft/cataloger/rpmdb/parse_rpmdb_test.go @@ -1,58 +1,141 @@ package rpmdb import ( + "fmt" "os" "testing" + "github.com/anchore/stereoscope/pkg/file" "github.com/anchore/syft/syft/pkg" "github.com/go-test/deep" ) +type rpmdbTestFileResolverMock struct { + ignorePaths bool +} + +func newTestFileResolver(ignorePaths bool) *rpmdbTestFileResolverMock { + return &rpmdbTestFileResolverMock{ + ignorePaths: ignorePaths, + } +} + +func (r *rpmdbTestFileResolverMock) FilesByPath(paths ...file.Path) ([]file.Reference, error) { + if r.ignorePaths { + // act as if no paths exist + return nil, nil + } + // act as if all files exist + var refs = make([]file.Reference, len(paths)) + for i, p := range paths { + refs[i] = file.NewFileReference(p) + } + return refs, nil +} + +func (r *rpmdbTestFileResolverMock) FilesByGlob(_ ...string) ([]file.Reference, error) { + return nil, fmt.Errorf("not implemented") +} +func (r *rpmdbTestFileResolverMock) RelativeFileByPath(_ file.Reference, path string) (*file.Reference, error) { + return nil, fmt.Errorf("not implemented") +} + func TestParseRpmDB(t *testing.T) { - expected := map[string]pkg.Package{ - "dive": { - Name: "dive", - Version: "0.9.2-1", - Type: pkg.RpmPkg, - MetadataType: pkg.RpmdbMetadataType, - Metadata: pkg.RpmdbMetadata{ - Name: "dive", - Epoch: 0, - Arch: "x86_64", - Release: "1", - Version: "0.9.2", - SourceRpm: "dive-0.9.2-1.src.rpm", - Size: 12406784, - License: "MIT", - Vendor: "", + tests := []struct { + fixture string + expected map[string]pkg.Package + ignorePaths bool + }{ + { + fixture: "test-fixtures/Packages", + // we only surface package paths for files that exist (here we DO NOT expect a path) + ignorePaths: true, + expected: map[string]pkg.Package{ + "dive": { + Name: "dive", + Version: "0.9.2-1", + Type: pkg.RpmPkg, + MetadataType: pkg.RpmdbMetadataType, + Metadata: pkg.RpmdbMetadata{ + Name: "dive", + Epoch: 0, + Arch: "x86_64", + Release: "1", + Version: "0.9.2", + SourceRpm: "dive-0.9.2-1.src.rpm", + Size: 12406784, + License: "MIT", + Vendor: "", + Files: []pkg.RpmdbFileRecord{}, + }, + }, + }, + }, + { + fixture: "test-fixtures/Packages", + // we only surface package paths for files that exist (here we expect a path) + ignorePaths: false, + expected: map[string]pkg.Package{ + "dive": { + Name: "dive", + Version: "0.9.2-1", + Type: pkg.RpmPkg, + MetadataType: pkg.RpmdbMetadataType, + Metadata: pkg.RpmdbMetadata{ + Name: "dive", + Epoch: 0, + Arch: "x86_64", + Release: "1", + Version: "0.9.2", + SourceRpm: "dive-0.9.2-1.src.rpm", + Size: 12406784, + License: "MIT", + Vendor: "", + Files: []pkg.RpmdbFileRecord{ + { + Path: "/usr/local/bin/dive", + Mode: 33261, + Size: 12406784, + SHA256: "81d29f327ba23096b3c52ff6fe1c425641e618bc87b5c05ee377edc650afaa55", + }, + }, + }, + }, }, }, } - fixture, err := os.Open("test-fixtures/Packages") - if err != nil { - t.Fatalf("failed to open fixture: %+v", err) - } - - actual, err := parseRpmDB(fixture.Name(), fixture) - if err != nil { - t.Fatalf("failed to parse rpmdb: %+v", err) - } - - if len(actual) != len(expected) { - for _, a := range actual { - t.Log(" ", a) - } - t.Fatalf("unexpected package count: %d!=%d", len(actual), len(expected)) - } - - for _, a := range actual { - e := expected[a.Name] - diffs := deep.Equal(a, e) - if len(diffs) > 0 { - for _, d := range diffs { - t.Errorf("diff: %+v", d) + for _, test := range tests { + t.Run(test.fixture, func(t *testing.T) { + fixture, err := os.Open(test.fixture) + if err != nil { + t.Fatalf("failed to open fixture: %+v", err) } - } + + fileResolver := newTestFileResolver(test.ignorePaths) + + actual, err := parseRpmDB(fileResolver, fixture) + if err != nil { + t.Fatalf("failed to parse rpmdb: %+v", err) + } + + if len(actual) != len(test.expected) { + for _, a := range actual { + t.Log(" ", a) + } + t.Fatalf("unexpected package count: %d!=%d", len(actual), len(test.expected)) + } + + for _, a := range actual { + e := test.expected[a.Name] + diffs := deep.Equal(a, e) + if len(diffs) > 0 { + for _, d := range diffs { + t.Errorf("diff: %+v", d) + } + } + } + }) } + } diff --git a/syft/pkg/rpmdb_metadata.go b/syft/pkg/rpmdb_metadata.go index 85b7d9bda..c9823c1a8 100644 --- a/syft/pkg/rpmdb_metadata.go +++ b/syft/pkg/rpmdb_metadata.go @@ -9,17 +9,27 @@ import ( // RpmdbMetadata represents all captured data for a RPM DB package entry. type RpmdbMetadata struct { - Name string `json:"name"` - Version string `json:"version"` - Epoch int `json:"epoch"` - Arch string `json:"architecture"` - Release string `json:"release"` - SourceRpm string `json:"sourceRpm"` - Size int `json:"size"` - License string `json:"license"` - Vendor string `json:"vendor"` + Name string `json:"name"` + Version string `json:"version"` + Epoch int `json:"epoch"` + Arch string `json:"architecture"` + Release string `json:"release"` + SourceRpm string `json:"sourceRpm"` + Size int `json:"size"` + License string `json:"license"` + Vendor string `json:"vendor"` + Files []RpmdbFileRecord `json:"files"` } +type RpmdbFileRecord struct { + Path string `json:"path"` + Mode RpmdbFileMode `json:"mode"` + Size int `json:"size"` + SHA256 string `json:"sha256"` +} + +type RpmdbFileMode uint16 + func (m RpmdbMetadata) PackageURL(d distro.Distro) string { pURL := packageurl.NewPackageURL( packageurl.TypeRPM, From 0205e72be97ab2d27cb04fb8f7114c137b98c5d2 Mon Sep 17 00:00:00 2001 From: Alex Goodman Date: Fri, 6 Nov 2020 07:21:38 -0500 Subject: [PATCH 2/3] ensure resolvers ignore directories for "FilesBy*" methods Signed-off-by: Alex Goodman --- syft/scope/resolvers/all_layers_resolver.go | 30 ++++++++++++++- .../resolvers/all_layers_resolver_test.go | 13 ++++++- syft/scope/resolvers/directory_resolver.go | 11 +++++- .../resolvers/directory_resolver_test.go | 6 +++ syft/scope/resolvers/image_squash_resolver.go | 32 +++++++++++++++- .../resolvers/image_squash_resolver_test.go | 37 +++++++++++++++++-- syft/scope/scope_test.go | 8 +++- 7 files changed, 129 insertions(+), 8 deletions(-) diff --git a/syft/scope/resolvers/all_layers_resolver.go b/syft/scope/resolvers/all_layers_resolver.go index 049d4a19d..ab732cf17 100644 --- a/syft/scope/resolvers/all_layers_resolver.go +++ b/syft/scope/resolvers/all_layers_resolver.go @@ -67,12 +67,26 @@ func (r *AllLayersResolver) FilesByPath(paths ...file.Path) ([]file.Reference, e for _, path := range paths { for idx, layerIdx := range r.layers { - ref := r.img.Layers[layerIdx].Tree.File(path) + tree := r.img.Layers[layerIdx].Tree + ref := tree.File(path) if ref == nil { // no file found, keep looking through layers continue } + // don't consider directories (special case: there is no path information for /) + if ref.Path == "/" { + continue + } else if r.img.FileCatalog.Exists(*ref) { + metadata, err := r.img.FileCatalog.Get(*ref) + if err != nil { + return nil, fmt.Errorf("unable to get file metadata for path=%q: %w", ref.Path, err) + } + if metadata.Metadata.IsDir { + continue + } + } + results, err := r.fileByRef(*ref, uniqueFileIDs, idx) if err != nil { return nil, err @@ -97,6 +111,20 @@ func (r *AllLayersResolver) FilesByGlob(patterns ...string) ([]file.Reference, e } for _, ref := range refs { + + // don't consider directories (special case: there is no path information for /) + if ref.Path == "/" { + continue + } else if r.img.FileCatalog.Exists(ref) { + metadata, err := r.img.FileCatalog.Get(ref) + if err != nil { + return nil, fmt.Errorf("unable to get file metadata for path=%q: %w", ref.Path, err) + } + if metadata.Metadata.IsDir { + continue + } + } + results, err := r.fileByRef(ref, uniqueFileIDs, idx) if err != nil { return nil, err diff --git a/syft/scope/resolvers/all_layers_resolver_test.go b/syft/scope/resolvers/all_layers_resolver_test.go index 97d30c0e3..e99ebd8a6 100644 --- a/syft/scope/resolvers/all_layers_resolver_test.go +++ b/syft/scope/resolvers/all_layers_resolver_test.go @@ -1,9 +1,10 @@ package resolvers import ( - "github.com/anchore/stereoscope/pkg/imagetest" "testing" + "github.com/anchore/stereoscope/pkg/imagetest" + "github.com/anchore/stereoscope/pkg/file" ) @@ -80,6 +81,11 @@ func TestAllLayersResolver_FilesByPath(t *testing.T) { }, }, }, + { + name: "ignore directories", + linkPath: "/bin", + resolutions: []resolution{}, + }, } for _, c := range cases { t.Run(c.name, func(t *testing.T) { @@ -188,6 +194,11 @@ func TestAllLayersResolver_FilesByGlob(t *testing.T) { }, }, }, + { + name: "ignore directories", + glob: "**/bin", + resolutions: []resolution{}, + }, } for _, c := range cases { t.Run(c.name, func(t *testing.T) { diff --git a/syft/scope/resolvers/directory_resolver.go b/syft/scope/resolvers/directory_resolver.go index 3d8d4ed07..5677c6dde 100644 --- a/syft/scope/resolvers/directory_resolver.go +++ b/syft/scope/resolvers/directory_resolver.go @@ -33,12 +33,18 @@ func (s DirectoryResolver) FilesByPath(userPaths ...file.Path) ([]file.Reference // a path relative to root should be prefixed with the resolvers directory path, otherwise it should be left as is userStrPath = path.Join(s.Path, userStrPath) } - _, err := os.Stat(userStrPath) + fileMeta, err := os.Stat(userStrPath) if os.IsNotExist(err) { continue } else if err != nil { log.Errorf("path (%s) is not valid: %v", userStrPath, err) } + + // don't consider directories + if fileMeta.IsDir() { + continue + } + references = append(references, file.NewFileReference(file.Path(userStrPath))) } @@ -69,9 +75,12 @@ func (s DirectoryResolver) FilesByGlob(patterns ...string) ([]file.Reference, er if err != nil { continue } + + // don't consider directories if fileMeta.IsDir() { continue } + matchedPath := file.Path(match) result = append(result, file.NewFileReference(matchedPath)) } diff --git a/syft/scope/resolvers/directory_resolver_test.go b/syft/scope/resolvers/directory_resolver_test.go index 5f2b93e19..809148da8 100644 --- a/syft/scope/resolvers/directory_resolver_test.go +++ b/syft/scope/resolvers/directory_resolver_test.go @@ -48,6 +48,12 @@ func TestDirectoryResolver_FilesByPath(t *testing.T) { expected: "test-fixtures/image-symlinks/file-1.txt", refCount: 1, }, + { + name: "directories ignored", + root: "./test-fixtures/", + input: "/image-symlinks", + refCount: 0, + }, } for _, c := range cases { t.Run(c.name, func(t *testing.T) { diff --git a/syft/scope/resolvers/image_squash_resolver.go b/syft/scope/resolvers/image_squash_resolver.go index 1e398f0ac..47ae8507f 100644 --- a/syft/scope/resolvers/image_squash_resolver.go +++ b/syft/scope/resolvers/image_squash_resolver.go @@ -26,16 +26,32 @@ func (r *ImageSquashResolver) FilesByPath(paths ...file.Path) ([]file.Reference, uniqueFiles := make([]file.Reference, 0) for _, path := range paths { - ref := r.img.SquashedTree().File(path) + tree := r.img.SquashedTree() + ref := tree.File(path) if ref == nil { // no file found, keep looking through layers continue } + // don't consider directories (special case: there is no path information for /) + if ref.Path == "/" { + continue + } else if r.img.FileCatalog.Exists(*ref) { + metadata, err := r.img.FileCatalog.Get(*ref) + if err != nil { + return nil, fmt.Errorf("unable to get file metadata for path=%q: %w", ref.Path, err) + } + if metadata.Metadata.IsDir { + continue + } + } + + // a file may be a symlink, process it as such and resolve it resolvedRef, err := r.img.ResolveLinkByImageSquash(*ref) if err != nil { return nil, fmt.Errorf("failed to resolve link from img (ref=%+v): %w", ref, err) } + if resolvedRef != nil && !uniqueFileIDs.Contains(*resolvedRef) { uniqueFileIDs.Add(*resolvedRef) uniqueFiles = append(uniqueFiles, *resolvedRef) @@ -57,6 +73,20 @@ func (r *ImageSquashResolver) FilesByGlob(patterns ...string) ([]file.Reference, } for _, ref := range refs { + + // don't consider directories (special case: there is no path information for /) + if ref.Path == "/" { + continue + } else if r.img.FileCatalog.Exists(ref) { + metadata, err := r.img.FileCatalog.Get(ref) + if err != nil { + return nil, fmt.Errorf("unable to get file metadata for path=%q: %w", ref.Path, err) + } + if metadata.Metadata.IsDir { + continue + } + } + resolvedRefs, err := r.FilesByPath(ref.Path) if err != nil { return nil, fmt.Errorf("failed to find files by path (ref=%+v): %w", ref, err) diff --git a/syft/scope/resolvers/image_squash_resolver_test.go b/syft/scope/resolvers/image_squash_resolver_test.go index b3530b566..dee432125 100644 --- a/syft/scope/resolvers/image_squash_resolver_test.go +++ b/syft/scope/resolvers/image_squash_resolver_test.go @@ -1,9 +1,10 @@ package resolvers import ( - "github.com/anchore/stereoscope/pkg/imagetest" "testing" + "github.com/anchore/stereoscope/pkg/imagetest" + "github.com/anchore/stereoscope/pkg/file" ) @@ -44,6 +45,11 @@ func TestImageSquashResolver_FilesByPath(t *testing.T) { resolveLayer: 8, resolvePath: "/link-dead", }, + { + name: "ignore directories", + linkPath: "/bin", + resolvePath: "", + }, } for _, c := range cases { t.Run(c.name, func(t *testing.T) { @@ -60,10 +66,20 @@ func TestImageSquashResolver_FilesByPath(t *testing.T) { t.Fatalf("could not use resolver: %+v", err) } - if len(refs) != 1 { + expectedRefs := 1 + if c.resolvePath == "" { + expectedRefs = 0 + } + + if len(refs) != expectedRefs { t.Fatalf("unexpected number of resolutions: %d", len(refs)) } + if expectedRefs == 0 { + // nothing else to assert + return + } + actual := refs[0] if actual.Path != file.Path(c.resolvePath) { @@ -119,6 +135,11 @@ func TestImageSquashResolver_FilesByGlob(t *testing.T) { resolveLayer: 8, resolvePath: "/link-dead", }, + { + name: "ignore directories", + glob: "**/bin", + resolvePath: "", + }, } for _, c := range cases { t.Run(c.name, func(t *testing.T) { @@ -135,10 +156,20 @@ func TestImageSquashResolver_FilesByGlob(t *testing.T) { t.Fatalf("could not use resolver: %+v", err) } - if len(refs) != 1 { + expectedRefs := 1 + if c.resolvePath == "" { + expectedRefs = 0 + } + + if len(refs) != expectedRefs { t.Fatalf("unexpected number of resolutions: %d", len(refs)) } + if expectedRefs == 0 { + // nothing else to assert + return + } + actual := refs[0] if actual.Path != file.Path(c.resolvePath) { diff --git a/syft/scope/scope_test.go b/syft/scope/scope_test.go index 1597ed7f2..b6656fc07 100644 --- a/syft/scope/scope_test.go +++ b/syft/scope/scope_test.go @@ -61,9 +61,15 @@ func TestDirectoryScope(t *testing.T) { { desc: "path detected", input: "test-fixtures", - inputPaths: []file.Path{file.Path("test-fixtures/path-detected")}, + inputPaths: []file.Path{file.Path("test-fixtures/path-detected/.vimrc")}, expRefs: 1, }, + { + desc: "directory ignored", + input: "test-fixtures", + inputPaths: []file.Path{file.Path("test-fixtures/path-detected")}, + expRefs: 0, + }, { desc: "no files-by-path detected", input: "test-fixtures", From 6aba2f48d4611a1414fb13ce58af6bd9d4566317 Mon Sep 17 00:00:00 2001 From: Alex Goodman Date: Fri, 6 Nov 2020 10:44:27 -0500 Subject: [PATCH 3/3] split out rpmdb file processing to helper + lint fixes Signed-off-by: Alex Goodman --- go.mod | 4 +- go.sum | 4 ++ syft/cataloger/rpmdb/cataloger.go | 6 +-- syft/cataloger/rpmdb/parse_rpmdb.go | 44 +++++++++++-------- syft/cataloger/rpmdb/parse_rpmdb_test.go | 4 +- syft/scope/resolvers/all_layers_resolver.go | 1 - syft/scope/resolvers/image_squash_resolver.go | 1 - 7 files changed, 36 insertions(+), 28 deletions(-) diff --git a/go.mod b/go.mod index 1c3385836..42ea39ed9 100644 --- a/go.mod +++ b/go.mod @@ -4,10 +4,10 @@ go 1.14 require ( github.com/adrg/xdg v0.2.1 - github.com/anchore/go-rpmdb v0.0.0-20200811175839-cbc751c28e8e + github.com/anchore/go-rpmdb v0.0.0-20201106153645-0043963c2e12 github.com/anchore/go-testutils v0.0.0-20200925183923-d5f45b0d3c04 github.com/anchore/go-version v1.2.2-0.20200701162849-18adb9c92b9b - github.com/anchore/stereoscope v0.0.0-20200925184903-c82da54e98fe + github.com/anchore/stereoscope v0.0.0-20201106140100-12e75c48f409 github.com/bmatcuk/doublestar v1.3.3 github.com/docker/docker v17.12.0-ce-rc1.0.20200309214505-aa6a9891b09c+incompatible github.com/dustin/go-humanize v1.0.0 diff --git a/go.sum b/go.sum index 71719b51e..55021990b 100644 --- a/go.sum +++ b/go.sum @@ -126,12 +126,16 @@ github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRF github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho= github.com/anchore/go-rpmdb v0.0.0-20200811175839-cbc751c28e8e h1:kty6r0R2JeaNPeWKSYDC+HW3hkqwFh4PP5TQ8pUPYFw= github.com/anchore/go-rpmdb v0.0.0-20200811175839-cbc751c28e8e/go.mod h1:iYuIG0Nai8dR0ri3LhZQKUyO1loxUWAGvoWhXDmjy1A= +github.com/anchore/go-rpmdb v0.0.0-20201106153645-0043963c2e12 h1:xbeIbn5F52JVx3RUIajxCj8b0y+9lywspql4sFhcxWQ= +github.com/anchore/go-rpmdb v0.0.0-20201106153645-0043963c2e12/go.mod h1:juoyWXIj7sJ1IDl4E/KIfyLtovbs5XQVSIdaQifFQT8= github.com/anchore/go-testutils v0.0.0-20200925183923-d5f45b0d3c04 h1:VzprUTpc0vW0nnNKJfJieyH/TZ9UYAnTZs5/gHTdAe8= github.com/anchore/go-testutils v0.0.0-20200925183923-d5f45b0d3c04/go.mod h1:6dK64g27Qi1qGQZ67gFmBFvEHScy0/C8qhQhNe5B5pQ= github.com/anchore/go-version v1.2.2-0.20200701162849-18adb9c92b9b h1:e1bmaoJfZVsCYMrIZBpFxwV26CbsuoEh5muXD5I1Ods= github.com/anchore/go-version v1.2.2-0.20200701162849-18adb9c92b9b/go.mod h1:Bkc+JYWjMCF8OyZ340IMSIi2Ebf3uwByOk6ho4wne1E= github.com/anchore/stereoscope v0.0.0-20200925184903-c82da54e98fe h1:m4NSyTo2fVUoUHAV/ZVqE/PFMr/y8oz9HRrhWLk9It0= github.com/anchore/stereoscope v0.0.0-20200925184903-c82da54e98fe/go.mod h1:2Jja/4l0zYggW52og+nn0rut4i+OYjCf9vTyrM8RT4E= +github.com/anchore/stereoscope v0.0.0-20201106140100-12e75c48f409 h1:xKSpDRjmYrEFrdMeDh4AuSUAFc99pdro6YFBKxy2um0= +github.com/anchore/stereoscope v0.0.0-20201106140100-12e75c48f409/go.mod h1:2Jja/4l0zYggW52og+nn0rut4i+OYjCf9vTyrM8RT4E= github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883/go.mod h1:rCTlJbsFo29Kk6CurOXKm700vrz8f0KW0JNfpkRJY/8= github.com/apex/log v1.1.4/go.mod h1:AlpoD9aScyQfJDVHmLMEcx4oU6LqzkWp4Mg9GdAcEvQ= github.com/apex/log v1.3.0 h1:1fyfbPvUwD10nMoh3hY6MXzvZShJQn9/ck7ATgAt5pA= diff --git a/syft/cataloger/rpmdb/cataloger.go b/syft/cataloger/rpmdb/cataloger.go index 25d57d610..e134b2bfb 100644 --- a/syft/cataloger/rpmdb/cataloger.go +++ b/syft/cataloger/rpmdb/cataloger.go @@ -27,17 +27,15 @@ func (c *Cataloger) Name() string { return "rpmdb-cataloger" } -// Catalog is given an object to resolve file references and content, this function returns any discovered Packages after analyzing python egg and wheel installations. +// Catalog is given an object to resolve file references and content, this function returns any discovered Packages after analyzing rpm db installation. func (c *Cataloger) Catalog(resolver scope.Resolver) ([]pkg.Package, error) { - fileMatches, err := resolver.FilesByGlob(packagesGlob) if err != nil { - return nil, fmt.Errorf("failed to find rpmdb's by glob") + return nil, fmt.Errorf("failed to find rpmdb's by glob: %w", err) } var pkgs []pkg.Package for _, ref := range fileMatches { - dbContents, err := resolver.FileContentsByRef(ref) if err != nil { return nil, err diff --git a/syft/cataloger/rpmdb/parse_rpmdb.go b/syft/cataloger/rpmdb/parse_rpmdb.go index 59bd1f222..9fc474583 100644 --- a/syft/cataloger/rpmdb/parse_rpmdb.go +++ b/syft/cataloger/rpmdb/parse_rpmdb.go @@ -48,24 +48,9 @@ func parseRpmDB(resolver scope.FileResolver, reader io.Reader) ([]pkg.Package, e allPkgs := make([]pkg.Package, 0) for _, entry := range pkgList { - var records = make([]pkg.RpmdbFileRecord, 0) - - for _, record := range entry.Files { - refs, err := resolver.FilesByPath(file.Path(record.Path)) - if err != nil { - return nil, fmt.Errorf("failed to resolve path=%+v: %w", record.Path, err) - } - //only persist RPMDB file records which exist in the image/directory, otherwise ignore them - if len(refs) == 0 { - continue - } - - records = append(records, pkg.RpmdbFileRecord{ - Path: record.Path, - Mode: pkg.RpmdbFileMode(record.Mode), - Size: int(record.Size), - SHA256: record.SHA256, - }) + records, err := extractRpmdbFileRecords(resolver, entry) + if err != nil { + return nil, err } p := pkg.Package{ @@ -93,3 +78,26 @@ func parseRpmDB(resolver scope.FileResolver, reader io.Reader) ([]pkg.Package, e return allPkgs, nil } + +func extractRpmdbFileRecords(resolver scope.FileResolver, entry *rpmdb.PackageInfo) ([]pkg.RpmdbFileRecord, error) { + var records = make([]pkg.RpmdbFileRecord, 0) + + for _, record := range entry.Files { + refs, err := resolver.FilesByPath(file.Path(record.Path)) + if err != nil { + return nil, fmt.Errorf("failed to resolve path=%+v: %w", record.Path, err) + } + //only persist RPMDB file records which exist in the image/directory, otherwise ignore them + if len(refs) == 0 { + continue + } + + records = append(records, pkg.RpmdbFileRecord{ + Path: record.Path, + Mode: pkg.RpmdbFileMode(record.Mode), + Size: int(record.Size), + SHA256: record.SHA256, + }) + } + return records, nil +} diff --git a/syft/cataloger/rpmdb/parse_rpmdb_test.go b/syft/cataloger/rpmdb/parse_rpmdb_test.go index 13348f783..6d1988932 100644 --- a/syft/cataloger/rpmdb/parse_rpmdb_test.go +++ b/syft/cataloger/rpmdb/parse_rpmdb_test.go @@ -33,10 +33,10 @@ func (r *rpmdbTestFileResolverMock) FilesByPath(paths ...file.Path) ([]file.Refe return refs, nil } -func (r *rpmdbTestFileResolverMock) FilesByGlob(_ ...string) ([]file.Reference, error) { +func (r *rpmdbTestFileResolverMock) FilesByGlob(...string) ([]file.Reference, error) { return nil, fmt.Errorf("not implemented") } -func (r *rpmdbTestFileResolverMock) RelativeFileByPath(_ file.Reference, path string) (*file.Reference, error) { +func (r *rpmdbTestFileResolverMock) RelativeFileByPath(file.Reference, string) (*file.Reference, error) { return nil, fmt.Errorf("not implemented") } diff --git a/syft/scope/resolvers/all_layers_resolver.go b/syft/scope/resolvers/all_layers_resolver.go index ab732cf17..a1c57ccda 100644 --- a/syft/scope/resolvers/all_layers_resolver.go +++ b/syft/scope/resolvers/all_layers_resolver.go @@ -111,7 +111,6 @@ func (r *AllLayersResolver) FilesByGlob(patterns ...string) ([]file.Reference, e } for _, ref := range refs { - // don't consider directories (special case: there is no path information for /) if ref.Path == "/" { continue diff --git a/syft/scope/resolvers/image_squash_resolver.go b/syft/scope/resolvers/image_squash_resolver.go index 47ae8507f..be1b7fe80 100644 --- a/syft/scope/resolvers/image_squash_resolver.go +++ b/syft/scope/resolvers/image_squash_resolver.go @@ -73,7 +73,6 @@ func (r *ImageSquashResolver) FilesByGlob(patterns ...string) ([]file.Reference, } for _, ref := range refs { - // don't consider directories (special case: there is no path information for /) if ref.Path == "/" { continue