Files
syft/syft/pkg/cataloger/cataloger.go
T
Alex Goodman 988041ba6d Speed up cataloging by replacing globs searching with index lookups (#1510)
* replace raw globs with index equivelent operations

Signed-off-by: Alex Goodman <alex.goodman@anchore.com>

* add cataloger test for alpm cataloger

Signed-off-by: Alex Goodman <alex.goodman@anchore.com>

* fix import sorting for binary cataloger

Signed-off-by: Alex Goodman <alex.goodman@anchore.com>

* fix linting for mock resolver

Signed-off-by: Alex Goodman <alex.goodman@anchore.com>

* separate portage cataloger parser impl from cataloger

Signed-off-by: Alex Goodman <alex.goodman@anchore.com>

* enhance cataloger pkgtest utils to account for resolver responses

Signed-off-by: Alex Goodman <alex.goodman@anchore.com>

* add glob-based cataloger tests for alpm cataloger

Signed-off-by: Alex Goodman <alex.goodman@anchore.com>

* add glob-based cataloger tests for apkdb cataloger

Signed-off-by: Alex Goodman <alex.goodman@anchore.com>

* add glob-based cataloger tests for dpkg cataloger

Signed-off-by: Alex Goodman <alex.goodman@anchore.com>

* add glob-based cataloger tests for cpp cataloger

Signed-off-by: Alex Goodman <alex.goodman@anchore.com>

* add glob-based cataloger tests for dart cataloger

Signed-off-by: Alex Goodman <alex.goodman@anchore.com>

* add glob-based cataloger tests for dotnet cataloger

Signed-off-by: Alex Goodman <alex.goodman@anchore.com>

* add glob-based cataloger tests for elixir cataloger

Signed-off-by: Alex Goodman <alex.goodman@anchore.com>

* add glob-based cataloger tests for erlang cataloger

Signed-off-by: Alex Goodman <alex.goodman@anchore.com>

* add glob-based cataloger tests for golang cataloger

Signed-off-by: Alex Goodman <alex.goodman@anchore.com>

* add glob-based cataloger tests for haskell cataloger

Signed-off-by: Alex Goodman <alex.goodman@anchore.com>

* add glob-based cataloger tests for java cataloger

Signed-off-by: Alex Goodman <alex.goodman@anchore.com>

* add glob-based cataloger tests for javascript cataloger

Signed-off-by: Alex Goodman <alex.goodman@anchore.com>

* add glob-based cataloger tests for php cataloger

Signed-off-by: Alex Goodman <alex.goodman@anchore.com>

* add glob-based cataloger tests for portage cataloger

Signed-off-by: Alex Goodman <alex.goodman@anchore.com>

* add glob-based cataloger tests for python cataloger

Signed-off-by: Alex Goodman <alex.goodman@anchore.com>

* add glob-based cataloger tests for rpm cataloger

Signed-off-by: Alex Goodman <alex.goodman@anchore.com>

* add glob-based cataloger tests for rust cataloger

Signed-off-by: Alex Goodman <alex.goodman@anchore.com>

* add glob-based cataloger tests for sbom cataloger

Signed-off-by: Alex Goodman <alex.goodman@anchore.com>

* add glob-based cataloger tests for swift cataloger

Signed-off-by: Alex Goodman <alex.goodman@anchore.com>

* allow generic catloger to run all mimetype searches at once

Signed-off-by: Alex Goodman <alex.goodman@anchore.com>

* remove stutter from php and javascript cataloger constructors

Signed-off-by: Alex Goodman <alex.goodman@anchore.com>

* bump stereoscope

Signed-off-by: Alex Goodman <alex.goodman@anchore.com>

* add tests for generic.Search

Signed-off-by: Alex Goodman <alex.goodman@anchore.com>

* add exceptions for java archive git ignore entries

Signed-off-by: Alex Goodman <alex.goodman@anchore.com>

* enhance basename and extension resolver methods to be variadic

Signed-off-by: Alex Goodman <alex.goodman@anchore.com>

* dont allow * prefix on extension searches

Signed-off-by: Alex Goodman <alex.goodman@anchore.com>

* add glob-based cataloger tests for ruby cataloger

Signed-off-by: Alex Goodman <alex.goodman@anchore.com>

* remove unnecessary string casting

Signed-off-by: Alex Goodman <alex.goodman@anchore.com>

* incorporate surfacing of leaf link resolitions from stereoscope results

Signed-off-by: Alex Goodman <alex.goodman@anchore.com>

* [wip] switch to stereoscope file metadata

Signed-off-by: Alex Goodman <alex.goodman@anchore.com>

* [wip + failing] revert to old globs but keep new resolvers

Signed-off-by: Alex Goodman <alex.goodman@anchore.com>

* index files, links, and dirs within the directory resolver

Signed-off-by: Alex Goodman <alex.goodman@anchore.com>

* fix several resolver bugs and inconsistencies

Signed-off-by: Alex Goodman <alex.goodman@anchore.com>

* move format testutils to internal package

Signed-off-by: Alex Goodman <alex.goodman@anchore.com>

* update syft json to account for file type string normalization

Signed-off-by: Alex Goodman <alex.goodman@anchore.com>

* split up directory resolver from indexing

Signed-off-by: Alex Goodman <alex.goodman@anchore.com>

* update docs to include details about searching

Signed-off-by: Alex Goodman <alex.goodman@anchore.com>

* [wip] bump stereoscope to development version

Signed-off-by: Alex Goodman <alex.goodman@anchore.com>

* fix linting

Signed-off-by: Alex Goodman <alex.goodman@anchore.com>

* adjust symlinks fixture to be fixed to digest

Signed-off-by: Alex Goodman <alex.goodman@anchore.com>

* fix all-locations resolver tests

Signed-off-by: Alex Goodman <alex.goodman@anchore.com>

* fix test fixture reference

Signed-off-by: Alex Goodman <alex.goodman@anchore.com>

* rename file.Type

Signed-off-by: Alex Goodman <alex.goodman@anchore.com>

* bump stereoscope

Signed-off-by: Alex Goodman <alex.goodman@anchore.com>

* fix PR comment to exclude extra *

Signed-off-by: Alex Goodman <alex.goodman@anchore.com>

* bump to dev version of stereoscope

Signed-off-by: Alex Goodman <alex.goodman@anchore.com>

* bump to final version of stereoscope

Signed-off-by: Alex Goodman <alex.goodman@anchore.com>

* move observing resolver to pkgtest

Signed-off-by: Alex Goodman <alex.goodman@anchore.com>

---------

Signed-off-by: Alex Goodman <alex.goodman@anchore.com>
2023-02-09 16:19:47 +00:00

170 lines
5.6 KiB
Go

/*
Package cataloger provides the ability to process files from a container image or file system and discover packages
(gems, wheels, jars, rpms, debs, etc). Specifically, this package contains both a catalog function to utilize all
catalogers defined in child packages as well as the interface definition to implement a cataloger.
*/
package cataloger
import (
"strings"
"github.com/anchore/syft/internal/log"
"github.com/anchore/syft/syft/pkg"
"github.com/anchore/syft/syft/pkg/cataloger/alpm"
"github.com/anchore/syft/syft/pkg/cataloger/apkdb"
"github.com/anchore/syft/syft/pkg/cataloger/binary"
"github.com/anchore/syft/syft/pkg/cataloger/cpp"
"github.com/anchore/syft/syft/pkg/cataloger/dart"
"github.com/anchore/syft/syft/pkg/cataloger/deb"
"github.com/anchore/syft/syft/pkg/cataloger/dotnet"
"github.com/anchore/syft/syft/pkg/cataloger/elixir"
"github.com/anchore/syft/syft/pkg/cataloger/erlang"
"github.com/anchore/syft/syft/pkg/cataloger/golang"
"github.com/anchore/syft/syft/pkg/cataloger/haskell"
"github.com/anchore/syft/syft/pkg/cataloger/java"
"github.com/anchore/syft/syft/pkg/cataloger/javascript"
"github.com/anchore/syft/syft/pkg/cataloger/php"
"github.com/anchore/syft/syft/pkg/cataloger/portage"
"github.com/anchore/syft/syft/pkg/cataloger/python"
"github.com/anchore/syft/syft/pkg/cataloger/rpm"
"github.com/anchore/syft/syft/pkg/cataloger/ruby"
"github.com/anchore/syft/syft/pkg/cataloger/rust"
"github.com/anchore/syft/syft/pkg/cataloger/sbom"
"github.com/anchore/syft/syft/pkg/cataloger/swift"
)
const AllCatalogersPattern = "all"
// ImageCatalogers returns a slice of locally implemented catalogers that are fit for detecting installations of packages.
func ImageCatalogers(cfg Config) []pkg.Cataloger {
return filterCatalogers([]pkg.Cataloger{
alpm.NewAlpmdbCataloger(),
ruby.NewGemSpecCataloger(),
python.NewPythonPackageCataloger(),
php.NewComposerInstalledCataloger(),
javascript.NewPackageCataloger(),
deb.NewDpkgdbCataloger(),
rpm.NewRpmDBCataloger(),
java.NewJavaCataloger(cfg.Java()),
java.NewNativeImageCataloger(),
apkdb.NewApkdbCataloger(),
golang.NewGoModuleBinaryCataloger(),
dotnet.NewDotnetDepsCataloger(),
portage.NewPortageCataloger(),
sbom.NewSBOMCataloger(),
binary.NewCataloger(),
}, cfg.Catalogers)
}
// DirectoryCatalogers returns a slice of locally implemented catalogers that are fit for detecting packages from index files (and select installations)
func DirectoryCatalogers(cfg Config) []pkg.Cataloger {
return filterCatalogers([]pkg.Cataloger{
alpm.NewAlpmdbCataloger(),
ruby.NewGemFileLockCataloger(),
python.NewPythonIndexCataloger(),
python.NewPythonPackageCataloger(),
php.NewComposerLockCataloger(),
javascript.NewLockCataloger(),
deb.NewDpkgdbCataloger(),
rpm.NewRpmDBCataloger(),
rpm.NewFileCataloger(),
java.NewJavaCataloger(cfg.Java()),
java.NewJavaPomCataloger(),
java.NewNativeImageCataloger(),
apkdb.NewApkdbCataloger(),
golang.NewGoModuleBinaryCataloger(),
golang.NewGoModFileCataloger(),
rust.NewCargoLockCataloger(),
dart.NewPubspecLockCataloger(),
dotnet.NewDotnetDepsCataloger(),
swift.NewCocoapodsCataloger(),
cpp.NewConanCataloger(),
portage.NewPortageCataloger(),
haskell.NewHackageCataloger(),
sbom.NewSBOMCataloger(),
binary.NewCataloger(),
elixir.NewMixLockCataloger(),
erlang.NewRebarLockCataloger(),
}, cfg.Catalogers)
}
// AllCatalogers returns all implemented catalogers
func AllCatalogers(cfg Config) []pkg.Cataloger {
return filterCatalogers([]pkg.Cataloger{
alpm.NewAlpmdbCataloger(),
ruby.NewGemFileLockCataloger(),
ruby.NewGemSpecCataloger(),
python.NewPythonIndexCataloger(),
python.NewPythonPackageCataloger(),
javascript.NewLockCataloger(),
javascript.NewPackageCataloger(),
deb.NewDpkgdbCataloger(),
rpm.NewRpmDBCataloger(),
rpm.NewFileCataloger(),
java.NewJavaCataloger(cfg.Java()),
java.NewJavaPomCataloger(),
java.NewNativeImageCataloger(),
apkdb.NewApkdbCataloger(),
golang.NewGoModuleBinaryCataloger(),
golang.NewGoModFileCataloger(),
rust.NewCargoLockCataloger(),
rust.NewAuditBinaryCataloger(),
dart.NewPubspecLockCataloger(),
dotnet.NewDotnetDepsCataloger(),
php.NewComposerInstalledCataloger(),
php.NewComposerLockCataloger(),
swift.NewCocoapodsCataloger(),
cpp.NewConanCataloger(),
portage.NewPortageCataloger(),
haskell.NewHackageCataloger(),
sbom.NewSBOMCataloger(),
binary.NewCataloger(),
elixir.NewMixLockCataloger(),
erlang.NewRebarLockCataloger(),
}, cfg.Catalogers)
}
func RequestedAllCatalogers(cfg Config) bool {
for _, enableCatalogerPattern := range cfg.Catalogers {
if enableCatalogerPattern == AllCatalogersPattern {
return true
}
}
return false
}
func filterCatalogers(catalogers []pkg.Cataloger, enabledCatalogerPatterns []string) []pkg.Cataloger {
// if cataloger is not set, all applicable catalogers are enabled by default
if len(enabledCatalogerPatterns) == 0 {
return catalogers
}
for _, enableCatalogerPattern := range enabledCatalogerPatterns {
if enableCatalogerPattern == AllCatalogersPattern {
return catalogers
}
}
var keepCatalogers []pkg.Cataloger
for _, cataloger := range catalogers {
if contains(enabledCatalogerPatterns, cataloger.Name()) {
keepCatalogers = append(keepCatalogers, cataloger)
continue
}
log.Infof("skipping cataloger %q", cataloger.Name())
}
return keepCatalogers
}
func contains(enabledPartial []string, catalogerName string) bool {
catalogerName = strings.TrimSuffix(catalogerName, "-cataloger")
for _, partial := range enabledPartial {
partial = strings.TrimSuffix(partial, "-cataloger")
if partial == "" {
continue
}
if strings.Contains(catalogerName, partial) {
return true
}
}
return false
}