mirror of
https://github.com/anchore/syft.git
synced 2026-11-02 07:26:34 +01:00
Speed up cataloging by replacing globs searching with index lookups (#1510)
* replace raw globs with index equivelent operations Signed-off-by: Alex Goodman <alex.goodman@anchore.com> * add cataloger test for alpm cataloger Signed-off-by: Alex Goodman <alex.goodman@anchore.com> * fix import sorting for binary cataloger Signed-off-by: Alex Goodman <alex.goodman@anchore.com> * fix linting for mock resolver Signed-off-by: Alex Goodman <alex.goodman@anchore.com> * separate portage cataloger parser impl from cataloger Signed-off-by: Alex Goodman <alex.goodman@anchore.com> * enhance cataloger pkgtest utils to account for resolver responses Signed-off-by: Alex Goodman <alex.goodman@anchore.com> * add glob-based cataloger tests for alpm cataloger Signed-off-by: Alex Goodman <alex.goodman@anchore.com> * add glob-based cataloger tests for apkdb cataloger Signed-off-by: Alex Goodman <alex.goodman@anchore.com> * add glob-based cataloger tests for dpkg cataloger Signed-off-by: Alex Goodman <alex.goodman@anchore.com> * add glob-based cataloger tests for cpp cataloger Signed-off-by: Alex Goodman <alex.goodman@anchore.com> * add glob-based cataloger tests for dart cataloger Signed-off-by: Alex Goodman <alex.goodman@anchore.com> * add glob-based cataloger tests for dotnet cataloger Signed-off-by: Alex Goodman <alex.goodman@anchore.com> * add glob-based cataloger tests for elixir cataloger Signed-off-by: Alex Goodman <alex.goodman@anchore.com> * add glob-based cataloger tests for erlang cataloger Signed-off-by: Alex Goodman <alex.goodman@anchore.com> * add glob-based cataloger tests for golang cataloger Signed-off-by: Alex Goodman <alex.goodman@anchore.com> * add glob-based cataloger tests for haskell cataloger Signed-off-by: Alex Goodman <alex.goodman@anchore.com> * add glob-based cataloger tests for java cataloger Signed-off-by: Alex Goodman <alex.goodman@anchore.com> * add glob-based cataloger tests for javascript cataloger Signed-off-by: Alex Goodman <alex.goodman@anchore.com> * add glob-based cataloger tests for php cataloger Signed-off-by: Alex Goodman <alex.goodman@anchore.com> * add glob-based cataloger tests for portage cataloger Signed-off-by: Alex Goodman <alex.goodman@anchore.com> * add glob-based cataloger tests for python cataloger Signed-off-by: Alex Goodman <alex.goodman@anchore.com> * add glob-based cataloger tests for rpm cataloger Signed-off-by: Alex Goodman <alex.goodman@anchore.com> * add glob-based cataloger tests for rust cataloger Signed-off-by: Alex Goodman <alex.goodman@anchore.com> * add glob-based cataloger tests for sbom cataloger Signed-off-by: Alex Goodman <alex.goodman@anchore.com> * add glob-based cataloger tests for swift cataloger Signed-off-by: Alex Goodman <alex.goodman@anchore.com> * allow generic catloger to run all mimetype searches at once Signed-off-by: Alex Goodman <alex.goodman@anchore.com> * remove stutter from php and javascript cataloger constructors Signed-off-by: Alex Goodman <alex.goodman@anchore.com> * bump stereoscope Signed-off-by: Alex Goodman <alex.goodman@anchore.com> * add tests for generic.Search Signed-off-by: Alex Goodman <alex.goodman@anchore.com> * add exceptions for java archive git ignore entries Signed-off-by: Alex Goodman <alex.goodman@anchore.com> * enhance basename and extension resolver methods to be variadic Signed-off-by: Alex Goodman <alex.goodman@anchore.com> * dont allow * prefix on extension searches Signed-off-by: Alex Goodman <alex.goodman@anchore.com> * add glob-based cataloger tests for ruby cataloger Signed-off-by: Alex Goodman <alex.goodman@anchore.com> * remove unnecessary string casting Signed-off-by: Alex Goodman <alex.goodman@anchore.com> * incorporate surfacing of leaf link resolitions from stereoscope results Signed-off-by: Alex Goodman <alex.goodman@anchore.com> * [wip] switch to stereoscope file metadata Signed-off-by: Alex Goodman <alex.goodman@anchore.com> * [wip + failing] revert to old globs but keep new resolvers Signed-off-by: Alex Goodman <alex.goodman@anchore.com> * index files, links, and dirs within the directory resolver Signed-off-by: Alex Goodman <alex.goodman@anchore.com> * fix several resolver bugs and inconsistencies Signed-off-by: Alex Goodman <alex.goodman@anchore.com> * move format testutils to internal package Signed-off-by: Alex Goodman <alex.goodman@anchore.com> * update syft json to account for file type string normalization Signed-off-by: Alex Goodman <alex.goodman@anchore.com> * split up directory resolver from indexing Signed-off-by: Alex Goodman <alex.goodman@anchore.com> * update docs to include details about searching Signed-off-by: Alex Goodman <alex.goodman@anchore.com> * [wip] bump stereoscope to development version Signed-off-by: Alex Goodman <alex.goodman@anchore.com> * fix linting Signed-off-by: Alex Goodman <alex.goodman@anchore.com> * adjust symlinks fixture to be fixed to digest Signed-off-by: Alex Goodman <alex.goodman@anchore.com> * fix all-locations resolver tests Signed-off-by: Alex Goodman <alex.goodman@anchore.com> * fix test fixture reference Signed-off-by: Alex Goodman <alex.goodman@anchore.com> * rename file.Type Signed-off-by: Alex Goodman <alex.goodman@anchore.com> * bump stereoscope Signed-off-by: Alex Goodman <alex.goodman@anchore.com> * fix PR comment to exclude extra * Signed-off-by: Alex Goodman <alex.goodman@anchore.com> * bump to dev version of stereoscope Signed-off-by: Alex Goodman <alex.goodman@anchore.com> * bump to final version of stereoscope Signed-off-by: Alex Goodman <alex.goodman@anchore.com> * move observing resolver to pkgtest Signed-off-by: Alex Goodman <alex.goodman@anchore.com> --------- Signed-off-by: Alex Goodman <alex.goodman@anchore.com>
This commit is contained in:
Alex Goodman
@@ -32,6 +32,7 @@ func (c Cataloger) Catalog(resolver source.FileResolver) ([]pkg.Package, []artif
|
||||
var relationships []artifact.Relationship
|
||||
|
||||
for _, cls := range defaultClassifiers {
|
||||
log.WithFields("classifier", cls.Class).Trace("cataloging binaries")
|
||||
pkgs, err := catalog(resolver, cls)
|
||||
if err != nil {
|
||||
log.WithFields("error", err, "classifier", cls.Class).Warn("unable to catalog binary package: %w", err)
|
||||
|
||||
@@ -13,7 +13,7 @@ import (
|
||||
"github.com/anchore/syft/syft/source"
|
||||
)
|
||||
|
||||
func TestClassifierCataloger_DefaultClassifiers_PositiveCases(t *testing.T) {
|
||||
func Test_Cataloger_DefaultClassifiers_PositiveCases(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
fixtureDir string
|
||||
@@ -440,7 +440,7 @@ func TestClassifierCataloger_DefaultClassifiers_PositiveCases(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestClassifierCataloger_DefaultClassifiers_PositiveCases_Image(t *testing.T) {
|
||||
func Test_Cataloger_DefaultClassifiers_PositiveCases_Image(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
fixtureImage string
|
||||
@@ -521,39 +521,57 @@ func assertPackagesAreEqual(t *testing.T, expected pkg.Package, p pkg.Package) {
|
||||
}
|
||||
|
||||
type panicyResolver struct {
|
||||
globCalled bool
|
||||
searchCalled bool
|
||||
}
|
||||
|
||||
func (p panicyResolver) FileContentsByLocation(location source.Location) (io.ReadCloser, error) {
|
||||
func (p *panicyResolver) FilesByExtension(_ ...string) ([]source.Location, error) {
|
||||
p.searchCalled = true
|
||||
return nil, errors.New("not implemented")
|
||||
}
|
||||
|
||||
func (p panicyResolver) HasPath(s string) bool {
|
||||
func (p *panicyResolver) FilesByBasename(_ ...string) ([]source.Location, error) {
|
||||
p.searchCalled = true
|
||||
return nil, errors.New("not implemented")
|
||||
}
|
||||
|
||||
func (p *panicyResolver) FilesByBasenameGlob(_ ...string) ([]source.Location, error) {
|
||||
p.searchCalled = true
|
||||
return nil, errors.New("not implemented")
|
||||
}
|
||||
|
||||
func (p *panicyResolver) FileContentsByLocation(_ source.Location) (io.ReadCloser, error) {
|
||||
p.searchCalled = true
|
||||
return nil, errors.New("not implemented")
|
||||
}
|
||||
|
||||
func (p *panicyResolver) HasPath(s string) bool {
|
||||
return true
|
||||
}
|
||||
|
||||
func (p panicyResolver) FilesByPath(paths ...string) ([]source.Location, error) {
|
||||
func (p *panicyResolver) FilesByPath(_ ...string) ([]source.Location, error) {
|
||||
p.searchCalled = true
|
||||
return nil, errors.New("not implemented")
|
||||
}
|
||||
|
||||
func (p *panicyResolver) FilesByGlob(patterns ...string) ([]source.Location, error) {
|
||||
p.globCalled = true
|
||||
func (p *panicyResolver) FilesByGlob(_ ...string) ([]source.Location, error) {
|
||||
p.searchCalled = true
|
||||
return nil, errors.New("not implemented")
|
||||
}
|
||||
|
||||
func (p panicyResolver) FilesByMIMEType(types ...string) ([]source.Location, error) {
|
||||
func (p *panicyResolver) FilesByMIMEType(_ ...string) ([]source.Location, error) {
|
||||
p.searchCalled = true
|
||||
return nil, errors.New("not implemented")
|
||||
}
|
||||
|
||||
func (p panicyResolver) RelativeFileByPath(_ source.Location, path string) *source.Location {
|
||||
func (p *panicyResolver) RelativeFileByPath(_ source.Location, _ string) *source.Location {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (p panicyResolver) AllLocations() <-chan source.Location {
|
||||
func (p *panicyResolver) AllLocations() <-chan source.Location {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (p panicyResolver) FileMetadataByLocation(location source.Location) (source.FileMetadata, error) {
|
||||
func (p *panicyResolver) FileMetadataByLocation(_ source.Location) (source.FileMetadata, error) {
|
||||
return source.FileMetadata{}, errors.New("not implemented")
|
||||
}
|
||||
|
||||
@@ -563,5 +581,5 @@ func Test_Cataloger_ResilientToErrors(t *testing.T) {
|
||||
resolver := &panicyResolver{}
|
||||
_, _, err := c.Catalog(resolver)
|
||||
assert.NoError(t, err)
|
||||
assert.True(t, resolver.globCalled)
|
||||
assert.True(t, resolver.searchCalled)
|
||||
}
|
||||
|
||||
@@ -21,7 +21,7 @@ func Test_ClassifierCPEs(t *testing.T) {
|
||||
fixture: "test-fixtures/version.txt",
|
||||
classifier: classifier{
|
||||
Package: "some-app",
|
||||
FileGlob: ".*/version.txt",
|
||||
FileGlob: "**/version.txt",
|
||||
EvidenceMatcher: fileContentsVersionMatcher(`(?m)my-verison:(?P<version>[0-9.]+)`),
|
||||
CPEs: []cpe.CPE{},
|
||||
},
|
||||
@@ -32,7 +32,7 @@ func Test_ClassifierCPEs(t *testing.T) {
|
||||
fixture: "test-fixtures/version.txt",
|
||||
classifier: classifier{
|
||||
Package: "some-app",
|
||||
FileGlob: ".*/version.txt",
|
||||
FileGlob: "**/version.txt",
|
||||
EvidenceMatcher: fileContentsVersionMatcher(`(?m)my-verison:(?P<version>[0-9.]+)`),
|
||||
CPEs: []cpe.CPE{
|
||||
cpe.Must("cpe:2.3:a:some:app:*:*:*:*:*:*:*:*"),
|
||||
@@ -47,7 +47,7 @@ func Test_ClassifierCPEs(t *testing.T) {
|
||||
fixture: "test-fixtures/version.txt",
|
||||
classifier: classifier{
|
||||
Package: "some-app",
|
||||
FileGlob: ".*/version.txt",
|
||||
FileGlob: "**/version.txt",
|
||||
EvidenceMatcher: fileContentsVersionMatcher(`(?m)my-verison:(?P<version>[0-9.]+)`),
|
||||
CPEs: []cpe.CPE{
|
||||
cpe.Must("cpe:2.3:a:some:app:*:*:*:*:*:*:*:*"),
|
||||
|
||||
Reference in New Issue
Block a user