oss/syft
2
0
Fork 0
mirror of https://github.com/anchore/syft.git synced 2026-11-02 07:26:34 +01:00
Code Issues Packages Projects Releases Wiki Activity

Add documentation around catalogers, UI elements, and the event bus (#143)

Browse Source 
* add basic documentation for catalogers (with refactoring for simplification)

Signed-off-by: Alex Goodman <alex.goodman@anchore.com>

* add docs for catalog parsers, UI, and event bus

Signed-off-by: Alex Goodman <alex.goodman@anchore.com>

* update bus phrasing

Signed-off-by: Alex Goodman <alex.goodman@anchore.com>
This commit is contained in:
Alex Goodman
2020-08-12 11:04:39 -04:00
committed by GitHub
parent 5042d371cf
commit 51b9c73c31
33 changed files with 250 additions and 86 deletions
Download Patch File Download Diff File Expand all files Collapse all files
syft/cataloger/apkdb/cataloger.go
+6 -1
View File
@@ -7,11 +7,13 @@ import (
"github.com/anchore/syft/syft/scope"
)
// Cataloger catalogs pkg.ApkPkg Package Types defined in Alpine DB files.
type Cataloger struct {
cataloger common.GenericCataloger
}
func NewCataloger() *Cataloger {
// New returns a new Alpine DB cataloger object.
func New() *Cataloger {
globParsers := map[string]common.ParserFn{
"**/lib/apk/db/installed": parseApkDB,
}
@@ -21,14 +23,17 @@ func NewCataloger() *Cataloger {
}
}
// Name returns a string that uniquely describes this cataloger.
func (a *Cataloger) Name() string {
return "apkdb-cataloger"
}
// SelectFiles returns a set of discovered Alpine DB files from the user content source.
func (a *Cataloger) SelectFiles(resolver scope.FileResolver) []file.Reference {
return a.cataloger.SelectFiles(resolver)
}
// Catalog returns the Packages indexed from all Alpine DB files discovered.
func (a *Cataloger) Catalog(contents map[file.Reference]string) ([]pkg.Package, error) {
return a.cataloger.Catalog(contents, a.Name())
}
syft/cataloger/apkdb/parse_apk_db.go
+7
View File
@@ -9,10 +9,16 @@ import (
"strings"
"github.com/anchore/syft/internal/log"
"github.com/anchore/syft/syft/cataloger/common"
"github.com/anchore/syft/syft/pkg"
"github.com/mitchellh/mapstructure"
)
// integrity check
var _ common.ParserFn = parseApkDB
// parseApkDb parses individual packages from a given Alpine DB file. For more information on specific fields
// see https://wiki.alpinelinux.org/wiki/Apk_spec .
func parseApkDB(_ string, reader io.Reader) ([]pkg.Package, error) {
packages := make([]pkg.Package, 0)
@@ -55,6 +61,7 @@ func parseApkDB(_ string, reader io.Reader) ([]pkg.Package, error) {
}
// nolint:funlen
// parseApkDBEntry reads and parses a single pkg.ApkMetadata element from the stream, returning nil if their are no more entries.
func parseApkDBEntry(reader io.Reader) (*pkg.ApkMetadata, error) {
var entry pkg.ApkMetadata
pkgFields := make(map[string]interface{})
syft/cataloger/bundler/cataloger.go
+6 -1
View File
@@ -7,11 +7,13 @@ import (
"github.com/anchore/syft/syft/scope"
)
// Cataloger catalogs pkg.GemPkg Package Types defined in Bundler Gemfile.lock files.
type Cataloger struct {
cataloger common.GenericCataloger
}
func NewCataloger() *Cataloger {
// New returns a new Bundler cataloger object.
func New() *Cataloger {
globParsers := map[string]common.ParserFn{
"**/Gemfile.lock": parseGemfileLockEntries,
}
@@ -21,14 +23,17 @@ func NewCataloger() *Cataloger {
}
}
// Name returns a string that uniquely describes this cataloger.
func (a *Cataloger) Name() string {
return "bundler-cataloger"
}
// SelectFiles returns a set of discovered Gemfile.lock files from the user content source.
func (a *Cataloger) SelectFiles(resolver scope.FileResolver) []file.Reference {
return a.cataloger.SelectFiles(resolver)
}
// Catalog returns the Packages indexed from all Gemfile.lock files discovered.
func (a *Cataloger) Catalog(contents map[file.Reference]string) ([]pkg.Package, error) {
return a.cataloger.Catalog(contents, a.Name())
}
syft/cataloger/bundler/parse_gemfile_lock.go
+5
View File
@@ -6,11 +6,16 @@ import (
"strings"
"github.com/anchore/syft/internal"
"github.com/anchore/syft/syft/cataloger/common"
"github.com/anchore/syft/syft/pkg"
)
// integrity check
var _ common.ParserFn = parseGemfileLockEntries
var sectionsOfInterest = internal.NewStringSetFromSlice([]string{"GEM"})
// parseGemfileLockEntries is a parser function for Gemfile.lock contents, returning all Gems discovered.
func parseGemfileLockEntries(_ string, reader io.Reader) ([]pkg.Package, error) {
pkgs := make([]pkg.Package, 0)
scanner := bufio.NewScanner(reader)
syft/cataloger/controller.go → syft/cataloger/catalog.go
+14 -57
View File
@@ -4,14 +4,6 @@ import (
"github.com/anchore/stereoscope/pkg/file"
"github.com/anchore/syft/internal/bus"
"github.com/anchore/syft/internal/log"
"github.com/anchore/syft/syft/cataloger/apkdb"
"github.com/anchore/syft/syft/cataloger/bundler"
"github.com/anchore/syft/syft/cataloger/dpkg"
golang "github.com/anchore/syft/syft/cataloger/golang"
"github.com/anchore/syft/syft/cataloger/java"
"github.com/anchore/syft/syft/cataloger/javascript"
"github.com/anchore/syft/syft/cataloger/python"
"github.com/anchore/syft/syft/cataloger/rpmdb"
"github.com/anchore/syft/syft/event"
"github.com/anchore/syft/syft/pkg"
"github.com/anchore/syft/syft/scope"
@@ -20,54 +12,14 @@ import (
"github.com/wagoodman/go-progress"
)
var controllerInstance controller
func init() {
controllerInstance = newController()
}
func Catalogers() []string {
c := make([]string, len(controllerInstance.catalogers))
for idx, catalog := range controllerInstance.catalogers {
c[idx] = catalog.Name()
}
return c
}
func Catalog(s scope.Resolver) (*pkg.Catalog, error) {
return controllerInstance.catalog(s)
}
type controller struct {
catalogers []Cataloger
}
func newController() controller {
ctrlr := controller{
catalogers: make([]Cataloger, 0),
}
ctrlr.add(dpkg.NewCataloger())
ctrlr.add(bundler.NewCataloger())
ctrlr.add(python.NewCataloger())
ctrlr.add(rpmdb.NewCataloger())
ctrlr.add(java.NewCataloger())
ctrlr.add(apkdb.NewCataloger())
ctrlr.add(golang.NewCataloger())
ctrlr.add(javascript.NewCataloger())
return ctrlr
}
func (c *controller) add(a Cataloger) {
log.Debugf("adding cataloger: %s", a.Name())
c.catalogers = append(c.catalogers, a)
}
// Monitor provides progress-related data for observing the progress of a Catalog() call (published on the event bus).
type Monitor struct {
FilesProcessed progress.Monitorable
PackagesDiscovered progress.Monitorable
FilesProcessed progress.Monitorable // the number of files selected and contents analyzed from all registered catalogers
PackagesDiscovered progress.Monitorable // the number of packages discovered from all registered catalogers
}
func (c *controller) trackCataloger() (*progress.Manual, *progress.Manual) {
// newMonitor creates a new Monitor object and publishes the object on the bus as a CatalogerStarted event.
func newMonitor() (*progress.Manual, *progress.Manual) {
filesProcessed := progress.Manual{}
packagesDiscovered := progress.Manual{}
@@ -81,20 +33,25 @@ func (c *controller) trackCataloger() (*progress.Manual, *progress.Manual) {
return &filesProcessed, &packagesDiscovered
}
func (c *controller) catalog(s scope.Resolver) (*pkg.Catalog, error) {
// Catalog a given scope (container image or filesystem) with the given catalogers, returning all discovered packages.
// In order to efficiently retrieve contents from a underlying container image the content fetch requests are
// done in bulk. Specifically, all files of interest are collected from each catalogers and accumulated into a single
// request.
func Catalog(s scope.Resolver, catalogers ...Cataloger) (*pkg.Catalog, error) {
catalog := pkg.NewCatalog()
fileSelection := make([]file.Reference, 0)
filesProcessed, packagesDiscovered := c.trackCataloger()
filesProcessed, packagesDiscovered := newMonitor()
// ask catalogers for files to extract from the image tar
for _, a := range c.catalogers {
for _, a := range catalogers {
fileSelection = append(fileSelection, a.SelectFiles(s)...)
log.Debugf("cataloger '%s' selected '%d' files", a.Name(), len(fileSelection))
filesProcessed.N += int64(len(fileSelection))
}
// fetch contents for requested selection by catalogers
// TODO: we should consider refactoring to return a set of io.Readers instead of the full contents themselves (allow for optional buffering).
contents, err := s.MultipleFileContentsByRef(fileSelection...)
if err != nil {
return nil, err
@@ -102,7 +59,7 @@ func (c *controller) catalog(s scope.Resolver) (*pkg.Catalog, error) {
// perform analysis, accumulating errors for each failed analysis
var errs error
for _, a := range c.catalogers {
for _, a := range catalogers {
// TODO: check for multiple rounds of analyses by Iterate error
packages, err := a.Catalog(contents)
if err != nil {
syft/cataloger/cataloger.go
+35 -3
View File
@@ -1,16 +1,48 @@
/*
Package cataloger provides the ability to process files from a container image or file system and discover packages
(e.g. gems, wheels, jars, rpms, debs, etc.). Specifically, this package contains both a catalog function to utilize all
catalogers defined in child packages as well as the interface definition to implement a cataloger.
*/
package cataloger
import (
"github.com/anchore/stereoscope/pkg/file"
"github.com/anchore/syft/syft/cataloger/apkdb"
"github.com/anchore/syft/syft/cataloger/bundler"
"github.com/anchore/syft/syft/cataloger/dpkg"
"github.com/anchore/syft/syft/cataloger/golang"
"github.com/anchore/syft/syft/cataloger/java"
"github.com/anchore/syft/syft/cataloger/javascript"
"github.com/anchore/syft/syft/cataloger/python"
"github.com/anchore/syft/syft/cataloger/rpmdb"
"github.com/anchore/syft/syft/pkg"
"github.com/anchore/syft/syft/scope"
)
// Cataloger describes behavior for an object to participate in parsing container image or file system
// contents for the purpose of discovering Packages. Each concrete implementation should focus on discovering Packages
// for a specific Package Type or ecosystem.
type Cataloger interface {
// Name returns a string that uniquely describes a cataloger
Name() string
// TODO: add ID / Name for catalog for uniquely identifying this cataloger type
// SelectFiles discovers and returns specific files that the cataloger would like to inspect the contents of.
SelectFiles(scope.FileResolver) []file.Reference
// NOTE: one of the errors which is returned is "IterationNeeded", which indicates to the driver to
// continue with another Select/Catalog pass
// Catalog is given the file contents and should return any discovered Packages after analyzing the contents.
Catalog(map[file.Reference]string) ([]pkg.Package, error)
// TODO: add "IterationNeeded" error to indicate to the driver to continue with another Select/Catalog pass
// TODO: we should consider refactoring to return a set of io.Readers instead of the full contents themselves (allow for optional buffering).
}
// All returns a slice of all locally defined catalogers (defined in child packages).
func All() []Cataloger {
return []Cataloger{
dpkg.New(),
bundler.New(),
python.New(),
rpmdb.New(),
java.New(),
apkdb.New(),
golang.New(),
javascript.New(),
}
}
syft/cataloger/dpkg/cataloger.go
+6 -1
View File
@@ -7,11 +7,13 @@ import (
"github.com/anchore/syft/syft/scope"
)
// Cataloger catalogs pkg.DebPkg Package Types defined in DPKG status files.
type Cataloger struct {
cataloger common.GenericCataloger
}
func NewCataloger() *Cataloger {
// New returns a new Deb package cataloger object.
func New() *Cataloger {
globParsers := map[string]common.ParserFn{
"**/var/lib/dpkg/status": parseDpkgStatus,
}
@@ -21,14 +23,17 @@ func NewCataloger() *Cataloger {
}
}
// Name returns a string that uniquely describes this cataloger.
func (a *Cataloger) Name() string {
return "dpkg-cataloger"
}
// SelectFiles returns a set of discovered DPKG status files from the user content source.
func (a *Cataloger) SelectFiles(resolver scope.FileResolver) []file.Reference {
return a.cataloger.SelectFiles(resolver)
}
// Catalog returns the Packages indexed from all DPKG status files discovered.
func (a *Cataloger) Catalog(contents map[file.Reference]string) ([]pkg.Package, error) {
return a.cataloger.Catalog(contents, a.Name())
}
syft/cataloger/dpkg/parse_dpkg_status.go
+6
View File
@@ -6,12 +6,17 @@ import (
"io"
"strings"
"github.com/anchore/syft/syft/cataloger/common"
"github.com/anchore/syft/syft/pkg"
"github.com/mitchellh/mapstructure"
)
// integrity check
var _ common.ParserFn = parseDpkgStatus
var errEndOfPackages = fmt.Errorf("no more packages to read")
// parseDpkgStatus is a parser function for Debian DB status contents, returning all Debian packages listed.
func parseDpkgStatus(_ string, reader io.Reader) ([]pkg.Package, error) {
buffedReader := bufio.NewReader(reader)
var packages = make([]pkg.Package, 0)
@@ -35,6 +40,7 @@ func parseDpkgStatus(_ string, reader io.Reader) ([]pkg.Package, error) {
return packages, nil
}
// parseDpkgStatusEntry returns an individual Dpkg entry, or returns errEndOfPackages if there are no more packages to parse from the reader.
func parseDpkgStatusEntry(reader *bufio.Reader) (entry pkg.DpkgMetadata, err error) {
dpkgFields := make(map[string]string)
var key string
syft/cataloger/golang/cataloger.go
+6 -1
View File
@@ -7,11 +7,13 @@ import (
"github.com/anchore/syft/syft/scope"
)
// Cataloger catalogs pkg.GoModulePkg Package Types defined in go.mod files.
type Cataloger struct {
cataloger common.GenericCataloger
}
func NewCataloger() *Cataloger {
// New returns a new Go module cataloger object.
func New() *Cataloger {
globParsers := map[string]common.ParserFn{
"**/go.mod": parseGoMod,
}
@@ -21,14 +23,17 @@ func NewCataloger() *Cataloger {
}
}
// Name returns a string that uniquely describes this cataloger.
func (a *Cataloger) Name() string {
return "go-cataloger"
}
// SelectFiles returns a set of discovered go.mod files from the user content source.
func (a *Cataloger) SelectFiles(resolver scope.FileResolver) []file.Reference {
return a.cataloger.SelectFiles(resolver)
}
// Catalog returns the Packages indexed from all go.mod files discovered.
func (a *Cataloger) Catalog(contents map[file.Reference]string) ([]pkg.Package, error) {
return a.cataloger.Catalog(contents, a.Name())
}
syft/cataloger/golang/parse_go_mod.go
+1
View File
@@ -11,6 +11,7 @@ import (
"github.com/anchore/syft/syft/pkg"
)
// parseGoMod takes a go.mod and lists all packages discovered.
func parseGoMod(path string, reader io.Reader) ([]pkg.Package, error) {
packages := make(map[string]pkg.Package)
syft/cataloger/java/archive_parser.go
+14
View File
@@ -7,9 +7,13 @@ import (
"github.com/anchore/syft/internal"
"github.com/anchore/syft/internal/file"
"github.com/anchore/syft/syft/cataloger/common"
"github.com/anchore/syft/syft/pkg"
)
// integrity check
var _ common.ParserFn = parseJavaArchive
var archiveFormatGlobs = []string{
"**/*.jar",
"**/*.war",
@@ -28,6 +32,7 @@ type archiveParser struct {
detectNested bool
}
// parseJavaArchive is a parser function for java archive contents, returning all Java libraries and nested archives.
func parseJavaArchive(virtualPath string, reader io.Reader) ([]pkg.Package, error) {
parser, cleanupFn, err := newJavaArchiveParser(virtualPath, reader, true)
// note: even on error, we should always run cleanup functions
@@ -38,6 +43,7 @@ func parseJavaArchive(virtualPath string, reader io.Reader) ([]pkg.Package, erro
return parser.parse()
}
// uniquePkgKey creates a unique string to identify the given package.
func uniquePkgKey(p *pkg.Package) string {
if p == nil {
return ""
@@ -45,6 +51,8 @@ func uniquePkgKey(p *pkg.Package) string {
return fmt.Sprintf("%s|%s", p.Name, p.Version)
}
// newJavaArchiveParser returns a new java archive parser object for the given archive. Can be configured to discover
// and parse nested archives or ignore them.
func newJavaArchiveParser(virtualPath string, reader io.Reader, detectNested bool) (*archiveParser, func(), error) {
contentPath, archivePath, cleanupFn, err := saveArchiveToTmp(reader)
if err != nil {
@@ -67,6 +75,7 @@ func newJavaArchiveParser(virtualPath string, reader io.Reader, detectNested boo
}, cleanupFn, nil
}
// parse the loaded archive and return all packages found.
func (j *archiveParser) parse() ([]pkg.Package, error) {
var pkgs = make([]pkg.Package, 0)
@@ -106,6 +115,7 @@ func (j *archiveParser) parse() ([]pkg.Package, error) {
return pkgs, nil
}
// discoverMainPackage parses the root Java manifest used as the parent package to all discovered nested packages.
func (j *archiveParser) discoverMainPackage() (*pkg.Package, error) {
// search and parse java manifest files
manifestMatches := j.fileManifest.GlobMatch(manifestPath)
@@ -140,6 +150,8 @@ func (j *archiveParser) discoverMainPackage() (*pkg.Package, error) {
}, nil
}
// discoverPkgsFromPomProperties parses Maven POM properties for a given parent package, returning all listed Java packages found and
// associating each discovered package to the given parent package.
func (j *archiveParser) discoverPkgsFromPomProperties(parentPkg *pkg.Package) ([]pkg.Package, error) {
var pkgs = make([]pkg.Package, 0)
parentKey := uniquePkgKey(parentPkg)
@@ -192,6 +204,8 @@ func (j *archiveParser) discoverPkgsFromPomProperties(parentPkg *pkg.Package) ([
return pkgs, nil
}
// discoverPkgsFromNestedArchives finds Java archives within Java archives, returning all listed Java packages found and
// associating each discovered package to the given parent package.
func (j *archiveParser) discoverPkgsFromNestedArchives(parentPkg *pkg.Package) ([]pkg.Package, error) {
var pkgs = make([]pkg.Package, 0)
syft/cataloger/java/cataloger.go
+6 -1
View File
@@ -7,11 +7,13 @@ import (
"github.com/anchore/syft/syft/scope"
)
// Cataloger catalogs pkg.JavaPkg and pkg.JenkinsPluginPkg Package Types defined in java archive files.
type Cataloger struct {
cataloger common.GenericCataloger
}
func NewCataloger() *Cataloger {
// New returns a new Java archive cataloger object.
func New() *Cataloger {
globParsers := make(map[string]common.ParserFn)
for _, pattern := range archiveFormatGlobs {
globParsers[pattern] = parseJavaArchive
@@ -22,14 +24,17 @@ func NewCataloger() *Cataloger {
}
}
// Name returns a string that uniquely describes this cataloger.
func (a *Cataloger) Name() string {
return "java-cataloger"
}
// SelectFiles returns a set of discovered Java archive files from the user content source.
func (a *Cataloger) SelectFiles(resolver scope.FileResolver) []file.Reference {
return a.cataloger.SelectFiles(resolver)
}
// Catalog returns the Packages indexed from all Java archive files discovered.
func (a *Cataloger) Catalog(contents map[file.Reference]string) ([]pkg.Package, error) {
return a.cataloger.Catalog(contents, a.Name())
}
syft/cataloger/javascript/cataloger.go
+7 -2
View File
@@ -7,11 +7,13 @@ import (
"github.com/anchore/syft/syft/scope"
)
// Cataloger catalogs pkg.YarnPkg and pkg.NpmPkg Package Types defined in package-lock.json and yarn.lock files.
type Cataloger struct {
cataloger common.GenericCataloger
}
func NewCataloger() *Cataloger {
// New returns a new JavaScript cataloger object.
func New() *Cataloger {
globParsers := map[string]common.ParserFn{
"**/package-lock.json": parsePackageLock,
"**/yarn.lock": parseYarnLock,
@@ -22,14 +24,17 @@ func NewCataloger() *Cataloger {
}
}
// Name returns a string that uniquely describes this cataloger.
func (a *Cataloger) Name() string {
return "npm-cataloger"
return "javascript-cataloger"
}
// SelectFiles returns a set of discovered Javascript ecosystem files from the user content source.
func (a *Cataloger) SelectFiles(resolver scope.FileResolver) []file.Reference {
return a.cataloger.SelectFiles(resolver)
}
// Catalog returns the Packages indexed from all Javascript ecosystem files discovered.
func (a *Cataloger) Catalog(contents map[file.Reference]string) ([]pkg.Package, error) {
return a.cataloger.Catalog(contents, a.Name())
}
syft/cataloger/javascript/parse_package_lock.go
+8 -3
View File
@@ -5,15 +5,21 @@ import (
"fmt"
"io"
"github.com/anchore/syft/syft/cataloger/common"
"github.com/anchore/syft/syft/pkg"
)
// integrity check
var _ common.ParserFn = parsePackageLock
// PackageLock represents a JavaScript package.lock json file
type PackageLock struct {
Requires bool `json:"requires"`
LockfileVersion int `json:"lockfileVersion"`
Dependencies Dependencies
Dependencies map[string]Dependency
}
// Dependency represents a single package dependency listed in the package.lock json file
type Dependency struct {
Version string `json:"version"`
Resolved string `json:"resolved"`
@@ -21,8 +27,7 @@ type Dependency struct {
Requires map[string]string
}
type Dependencies map[string]Dependency
// parsePackageLock parses a package.lock and returns the discovered JavaScript packages.
func parsePackageLock(_ string, reader io.Reader) ([]pkg.Package, error) {
packages := make([]pkg.Package, 0)
dec := json.NewDecoder(reader)
syft/cataloger/javascript/parse_yarn_lock.go
+4
View File
@@ -8,9 +8,13 @@ import (
"strings"
"github.com/anchore/syft/internal/log"
"github.com/anchore/syft/syft/cataloger/common"
"github.com/anchore/syft/syft/pkg"
)
// integrity check
var _ common.ParserFn = parseYarnLock
var composedNameExp = regexp.MustCompile("^\"(@{1}[^@]+)")
var simpleNameExp = regexp.MustCompile(`^[a-zA-Z\-]+@`)
var versionExp = regexp.MustCompile(`^\W+(version)\W+`)
syft/cataloger/python/cataloger.go
+6 -4
View File
@@ -7,14 +7,13 @@ import (
"github.com/anchore/syft/syft/scope"
)
// Cataloger catalogs pkg.WheelPkg, pkg.EggPkg, and pkg.PythonRequirementsPkg Package Types defined in Python ecosystem files.
type Cataloger struct {
cataloger common.GenericCataloger
}
func NewCataloger() *Cataloger {
// we want to match on partial dir names
// /home/user/requests-2.10.0.dist-info/METADATA
// /home/user/requests-2.10.0/dist-info/METADATA
// New returns a new Python cataloger object.
func New() *Cataloger {
globParsers := map[string]common.ParserFn{
"**/*egg-info/PKG-INFO": parseEggMetadata,
"**/*dist-info/METADATA": parseWheelMetadata,
@@ -27,14 +26,17 @@ func NewCataloger() *Cataloger {
}
}
// Name returns a string that uniquely describes this cataloger.
func (a *Cataloger) Name() string {
return "python-cataloger"
}
// SelectFiles returns a set of discovered Python ecosystem files from the user content source.
func (a *Cataloger) SelectFiles(resolver scope.FileResolver) []file.Reference {
return a.cataloger.SelectFiles(resolver)
}
// Catalog returns the Packages indexed from all Python ecosystem files discovered.
func (a *Cataloger) Catalog(contents map[file.Reference]string) ([]pkg.Package, error) {
return a.cataloger.Catalog(contents, a.Name())
}
syft/cataloger/python/parse_poetry_lock.go
+5
View File
@@ -4,10 +4,15 @@ import (
"fmt"
"io"
"github.com/anchore/syft/syft/cataloger/common"
"github.com/anchore/syft/syft/pkg"
"github.com/pelletier/go-toml"
)
// integrity check
var _ common.ParserFn = parsePoetryLock
// parsePoetryLock is a parser function for poetry.lock contents, returning all python packages discovered.
func parsePoetryLock(_ string, reader io.Reader) ([]pkg.Package, error) {
tree, err := toml.LoadReader(reader)
if err != nil {
syft/cataloger/python/parse_requirements.go
+7
View File
@@ -6,9 +6,15 @@ import (
"io"
"strings"
"github.com/anchore/syft/syft/cataloger/common"
"github.com/anchore/syft/syft/pkg"
)
// integrity check
var _ common.ParserFn = parseRequirementsTxt
// parseRequirementsTxt takes a Python requirements.txt file, returning all Python packages that are locked to a
// specific version.
func parseRequirementsTxt(_ string, reader io.Reader) ([]pkg.Package, error) {
packages := make([]pkg.Package, 0)
@@ -55,6 +61,7 @@ func parseRequirementsTxt(_ string, reader io.Reader) ([]pkg.Package, error) {
return packages, nil
}
// removeTrailingComment takes a requirements.txt line and strips off comment strings.
func removeTrailingComment(line string) string {
parts := strings.Split(line, "#")
switch len(parts) {
syft/cataloger/python/parse_wheel_egg.go
+11
View File
@@ -6,9 +6,16 @@ import (
"io"
"strings"
"github.com/anchore/syft/syft/cataloger/common"
"github.com/anchore/syft/syft/pkg"
)
// integrity check
var _ common.ParserFn = parseWheelMetadata
var _ common.ParserFn = parseEggMetadata
// parseWheelMetadata is a parser function for individual Python Wheel metadata file contents, returning all Python
// packages listed.
func parseWheelMetadata(_ string, reader io.Reader) ([]pkg.Package, error) {
packages, err := parseWheelOrEggMetadata(reader)
for idx := range packages {
@@ -17,6 +24,8 @@ func parseWheelMetadata(_ string, reader io.Reader) ([]pkg.Package, error) {
return packages, err
}
// parseEggMetadata is a parser function for individual Python Egg metadata file contents, returning all Python
// packages listed.
func parseEggMetadata(_ string, reader io.Reader) ([]pkg.Package, error) {
packages, err := parseWheelOrEggMetadata(reader)
for idx := range packages {
@@ -25,6 +34,8 @@ func parseEggMetadata(_ string, reader io.Reader) ([]pkg.Package, error) {
return packages, err
}
// parseWheelOrEggMetadata takes a Python Egg or Wheel (which share the same format and values for our purposes),
// returning all Python packages listed.
func parseWheelOrEggMetadata(reader io.Reader) ([]pkg.Package, error) {
fields := make(map[string]string)
var key string
syft/cataloger/rpmdb/cataloger.go
+6 -1
View File
@@ -7,11 +7,13 @@ import (
"github.com/anchore/syft/syft/scope"
)
// Cataloger catalogs pkg.RpmPkg Package Types defined in RPM DB files.
type Cataloger struct {
cataloger common.GenericCataloger
}
func NewCataloger() *Cataloger {
// New returns a new RPM DB cataloger object.
func New() *Cataloger {
globParsers := map[string]common.ParserFn{
"**/var/lib/rpm/Packages": parseRpmDB,
}
@@ -21,14 +23,17 @@ func NewCataloger() *Cataloger {
}
}
// Name returns a string that uniquely describes this cataloger.
func (a *Cataloger) Name() string {
return "rpmdb-cataloger"
}
// SelectFiles returns a set of discovered RPM DB files from the user content source.
func (a *Cataloger) SelectFiles(resolver scope.FileResolver) []file.Reference {
return a.cataloger.SelectFiles(resolver)
}
// Catalog returns the Packages indexed from all RPM DB files discovered.
func (a *Cataloger) Catalog(contents map[file.Reference]string) ([]pkg.Package, error) {
return a.cataloger.Catalog(contents, a.Name())
}
syft/cataloger/rpmdb/parse_rpmdb.go
+5
View File
@@ -9,9 +9,14 @@ import (
rpmdb "github.com/anchore/go-rpmdb/pkg"
"github.com/anchore/syft/internal"
"github.com/anchore/syft/internal/log"
"github.com/anchore/syft/syft/cataloger/common"
"github.com/anchore/syft/syft/pkg"
)
// integrity check
var _ common.ParserFn = parseRpmDB
// parseApkDb parses an "Packages" RPM DB and returns the Packages listed within it.
func parseRpmDB(_ string, reader io.Reader) ([]pkg.Package, error) {
f, err := ioutil.TempFile("", internal.ApplicationName+"-rpmdb")
if err != nil {
syft/lib.go
+1 -1
View File
@@ -41,7 +41,7 @@ func IdentifyDistro(s scope.Scope) distro.Distro {
func CatalogFromScope(s scope.Scope) (*pkg.Catalog, error) {
log.Info("building the catalog")
return cataloger.Catalog(s)
return cataloger.Catalog(s, cataloger.All()...)
}
func SetLogger(logger logger.Logger) {
syft/pkg/metadata.go
+3 -2
View File
@@ -1,7 +1,7 @@
package pkg
// Available fields are described at http://manpages.ubuntu.com/manpages/xenial/man1/dpkg-query.1.html
// in the --showformat section
// DpkgMetadata represents all captured data for a Debian package DB entry; available fields are described
// at http://manpages.ubuntu.com/manpages/xenial/man1/dpkg-query.1.html in the --showformat section.
type DpkgMetadata struct {
Package string `mapstructure:"Package" json:"package"`
Source string `mapstructure:"Source" json:"source"`
@@ -9,6 +9,7 @@ type DpkgMetadata struct {
// TODO: consider keeping the remaining values as an embedded map
}
// RpmMetadata represents all captured data for a RHEL package DB entry.
type RpmMetadata struct {
Version string `mapstructure:"Version" json:"version"`
Epoch int `mapstructure:"Epoch" json:"epoch"`