mirror of
https://github.com/anchore/syft.git
synced 2026-11-05 00:46:38 +01:00
Speed up cataloging by replacing globs searching with index lookups (#1510)
* replace raw globs with index equivelent operations Signed-off-by: Alex Goodman <alex.goodman@anchore.com> * add cataloger test for alpm cataloger Signed-off-by: Alex Goodman <alex.goodman@anchore.com> * fix import sorting for binary cataloger Signed-off-by: Alex Goodman <alex.goodman@anchore.com> * fix linting for mock resolver Signed-off-by: Alex Goodman <alex.goodman@anchore.com> * separate portage cataloger parser impl from cataloger Signed-off-by: Alex Goodman <alex.goodman@anchore.com> * enhance cataloger pkgtest utils to account for resolver responses Signed-off-by: Alex Goodman <alex.goodman@anchore.com> * add glob-based cataloger tests for alpm cataloger Signed-off-by: Alex Goodman <alex.goodman@anchore.com> * add glob-based cataloger tests for apkdb cataloger Signed-off-by: Alex Goodman <alex.goodman@anchore.com> * add glob-based cataloger tests for dpkg cataloger Signed-off-by: Alex Goodman <alex.goodman@anchore.com> * add glob-based cataloger tests for cpp cataloger Signed-off-by: Alex Goodman <alex.goodman@anchore.com> * add glob-based cataloger tests for dart cataloger Signed-off-by: Alex Goodman <alex.goodman@anchore.com> * add glob-based cataloger tests for dotnet cataloger Signed-off-by: Alex Goodman <alex.goodman@anchore.com> * add glob-based cataloger tests for elixir cataloger Signed-off-by: Alex Goodman <alex.goodman@anchore.com> * add glob-based cataloger tests for erlang cataloger Signed-off-by: Alex Goodman <alex.goodman@anchore.com> * add glob-based cataloger tests for golang cataloger Signed-off-by: Alex Goodman <alex.goodman@anchore.com> * add glob-based cataloger tests for haskell cataloger Signed-off-by: Alex Goodman <alex.goodman@anchore.com> * add glob-based cataloger tests for java cataloger Signed-off-by: Alex Goodman <alex.goodman@anchore.com> * add glob-based cataloger tests for javascript cataloger Signed-off-by: Alex Goodman <alex.goodman@anchore.com> * add glob-based cataloger tests for php cataloger Signed-off-by: Alex Goodman <alex.goodman@anchore.com> * add glob-based cataloger tests for portage cataloger Signed-off-by: Alex Goodman <alex.goodman@anchore.com> * add glob-based cataloger tests for python cataloger Signed-off-by: Alex Goodman <alex.goodman@anchore.com> * add glob-based cataloger tests for rpm cataloger Signed-off-by: Alex Goodman <alex.goodman@anchore.com> * add glob-based cataloger tests for rust cataloger Signed-off-by: Alex Goodman <alex.goodman@anchore.com> * add glob-based cataloger tests for sbom cataloger Signed-off-by: Alex Goodman <alex.goodman@anchore.com> * add glob-based cataloger tests for swift cataloger Signed-off-by: Alex Goodman <alex.goodman@anchore.com> * allow generic catloger to run all mimetype searches at once Signed-off-by: Alex Goodman <alex.goodman@anchore.com> * remove stutter from php and javascript cataloger constructors Signed-off-by: Alex Goodman <alex.goodman@anchore.com> * bump stereoscope Signed-off-by: Alex Goodman <alex.goodman@anchore.com> * add tests for generic.Search Signed-off-by: Alex Goodman <alex.goodman@anchore.com> * add exceptions for java archive git ignore entries Signed-off-by: Alex Goodman <alex.goodman@anchore.com> * enhance basename and extension resolver methods to be variadic Signed-off-by: Alex Goodman <alex.goodman@anchore.com> * dont allow * prefix on extension searches Signed-off-by: Alex Goodman <alex.goodman@anchore.com> * add glob-based cataloger tests for ruby cataloger Signed-off-by: Alex Goodman <alex.goodman@anchore.com> * remove unnecessary string casting Signed-off-by: Alex Goodman <alex.goodman@anchore.com> * incorporate surfacing of leaf link resolitions from stereoscope results Signed-off-by: Alex Goodman <alex.goodman@anchore.com> * [wip] switch to stereoscope file metadata Signed-off-by: Alex Goodman <alex.goodman@anchore.com> * [wip + failing] revert to old globs but keep new resolvers Signed-off-by: Alex Goodman <alex.goodman@anchore.com> * index files, links, and dirs within the directory resolver Signed-off-by: Alex Goodman <alex.goodman@anchore.com> * fix several resolver bugs and inconsistencies Signed-off-by: Alex Goodman <alex.goodman@anchore.com> * move format testutils to internal package Signed-off-by: Alex Goodman <alex.goodman@anchore.com> * update syft json to account for file type string normalization Signed-off-by: Alex Goodman <alex.goodman@anchore.com> * split up directory resolver from indexing Signed-off-by: Alex Goodman <alex.goodman@anchore.com> * update docs to include details about searching Signed-off-by: Alex Goodman <alex.goodman@anchore.com> * [wip] bump stereoscope to development version Signed-off-by: Alex Goodman <alex.goodman@anchore.com> * fix linting Signed-off-by: Alex Goodman <alex.goodman@anchore.com> * adjust symlinks fixture to be fixed to digest Signed-off-by: Alex Goodman <alex.goodman@anchore.com> * fix all-locations resolver tests Signed-off-by: Alex Goodman <alex.goodman@anchore.com> * fix test fixture reference Signed-off-by: Alex Goodman <alex.goodman@anchore.com> * rename file.Type Signed-off-by: Alex Goodman <alex.goodman@anchore.com> * bump stereoscope Signed-off-by: Alex Goodman <alex.goodman@anchore.com> * fix PR comment to exclude extra * Signed-off-by: Alex Goodman <alex.goodman@anchore.com> * bump to dev version of stereoscope Signed-off-by: Alex Goodman <alex.goodman@anchore.com> * bump to final version of stereoscope Signed-off-by: Alex Goodman <alex.goodman@anchore.com> * move observing resolver to pkgtest Signed-off-by: Alex Goodman <alex.goodman@anchore.com> --------- Signed-off-by: Alex Goodman <alex.goodman@anchore.com>
This commit is contained in:
Alex Goodman
@@ -4,11 +4,7 @@ import (
|
||||
"github.com/anchore/syft/syft/pkg/cataloger/generic"
|
||||
)
|
||||
|
||||
const (
|
||||
eggMetadataGlob = "**/*egg-info/PKG-INFO"
|
||||
eggFileMetadataGlob = "**/*.egg-info"
|
||||
wheelMetadataGlob = "**/*dist-info/METADATA"
|
||||
)
|
||||
const eggInfoGlob = "**/*.egg-info"
|
||||
|
||||
// NewPythonIndexCataloger returns a new cataloger for python packages referenced from poetry lock files, requirements.txt files, and setup.py files.
|
||||
func NewPythonIndexCataloger() *generic.Cataloger {
|
||||
@@ -22,5 +18,5 @@ func NewPythonIndexCataloger() *generic.Cataloger {
|
||||
// NewPythonPackageCataloger returns a new cataloger for python packages within egg or wheel installation directories.
|
||||
func NewPythonPackageCataloger() *generic.Cataloger {
|
||||
return generic.NewCataloger("python-package-cataloger").
|
||||
WithParserByGlobs(parseWheelOrEgg, eggMetadataGlob, eggFileMetadataGlob, wheelMetadataGlob)
|
||||
WithParserByGlobs(parseWheelOrEgg, eggInfoGlob, "**/*dist-info/METADATA", "**/*egg-info/PKG-INFO")
|
||||
}
|
||||
|
||||
@@ -3,6 +3,8 @@ package python
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"github.com/anchore/syft/syft/pkg"
|
||||
"github.com/anchore/syft/syft/pkg/cataloger/internal/pkgtest"
|
||||
"github.com/anchore/syft/syft/source"
|
||||
@@ -192,9 +194,7 @@ func Test_PackageCataloger(t *testing.T) {
|
||||
resolver := source.NewMockResolverForPaths(test.fixtures...)
|
||||
|
||||
locations, err := resolver.FilesByPath(test.fixtures...)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
require.NoError(t, err)
|
||||
|
||||
test.expectedPackage.Locations = source.NewLocationSet(locations...)
|
||||
|
||||
@@ -223,9 +223,7 @@ func Test_PackageCataloger_IgnorePackage(t *testing.T) {
|
||||
resolver := source.NewMockResolverForPaths(test.MetadataFixture)
|
||||
|
||||
actual, _, err := NewPythonPackageCataloger().Catalog(resolver)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to catalog python package: %+v", err)
|
||||
}
|
||||
require.NoError(t, err)
|
||||
|
||||
if len(actual) != 0 {
|
||||
t.Fatalf("Expected 0 packages but found: %d", len(actual))
|
||||
@@ -233,3 +231,61 @@ func Test_PackageCataloger_IgnorePackage(t *testing.T) {
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func Test_IndexCataloger_Globs(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
fixture string
|
||||
expected []string
|
||||
}{
|
||||
{
|
||||
name: "obtain index files",
|
||||
fixture: "test-fixtures/glob-paths",
|
||||
expected: []string{
|
||||
"src/requirements.txt",
|
||||
"src/extra-requirements.txt",
|
||||
"src/requirements-dev.txt",
|
||||
"src/1-requirements-dev.txt",
|
||||
"src/setup.py",
|
||||
"src/poetry.lock",
|
||||
"src/Pipfile.lock",
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
for _, test := range tests {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
pkgtest.NewCatalogTester().
|
||||
FromDirectory(t, test.fixture).
|
||||
ExpectsResolverContentQueries(test.expected).
|
||||
TestCataloger(t, NewPythonIndexCataloger())
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func Test_PackageCataloger_Globs(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
fixture string
|
||||
expected []string
|
||||
}{
|
||||
{
|
||||
name: "obtain index files",
|
||||
fixture: "test-fixtures/glob-paths",
|
||||
expected: []string{
|
||||
"site-packages/x.dist-info/METADATA",
|
||||
"site-packages/y.egg-info/PKG-INFO",
|
||||
"site-packages/z.egg-info",
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
for _, test := range tests {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
pkgtest.NewCatalogTester().
|
||||
FromDirectory(t, test.fixture).
|
||||
ExpectsResolverContentQueries(test.expected).
|
||||
TestCataloger(t, NewPythonPackageCataloger())
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
@@ -81,7 +81,7 @@ func parseWheelOrEggMetadata(path string, reader io.Reader) (pkg.PythonPackageMe
|
||||
// of egg metadata (as opposed to a directory that contains more metadata
|
||||
// files).
|
||||
func isEggRegularFile(path string) bool {
|
||||
return file.GlobMatch(eggFileMetadataGlob, path)
|
||||
return file.GlobMatch(eggInfoGlob, path)
|
||||
}
|
||||
|
||||
// determineSitePackagesRootPath returns the path of the site packages root,
|
||||
|
||||
@@ -0,0 +1 @@
|
||||
bogus
|
||||
@@ -0,0 +1 @@
|
||||
bogus
|
||||
@@ -0,0 +1 @@
|
||||
bogus
|
||||
@@ -0,0 +1 @@
|
||||
example archive
|
||||
@@ -0,0 +1 @@
|
||||
bogus
|
||||
@@ -0,0 +1 @@
|
||||
example archive
|
||||
@@ -0,0 +1 @@
|
||||
bogus
|
||||
@@ -0,0 +1 @@
|
||||
example archive
|
||||
@@ -0,0 +1 @@
|
||||
example archive
|
||||
@@ -0,0 +1 @@
|
||||
bogus
|
||||
Reference in New Issue
Block a user