From 51b9c73c31516b63aa4f544d85efecd287e00593 Mon Sep 17 00:00:00 2001 From: Alex Goodman Date: Wed, 12 Aug 2020 11:04:39 -0400 Subject: [PATCH] Add documentation around catalogers, UI elements, and the event bus (#143) * add basic documentation for catalogers (with refactoring for simplification) Signed-off-by: Alex Goodman * add docs for catalog parsers, UI, and event bus Signed-off-by: Alex Goodman * update bus phrasing Signed-off-by: Alex Goodman --- internal/bus/bus.go | 16 +++++ internal/ui/common/event_handlers.go | 2 + internal/ui/etui/ephemeral_tui.go | 25 ++++++- internal/ui/etui/internal_event_handlers.go | 1 + internal/ui/logger_output.go | 1 + internal/ui/select.go | 2 + internal/ui/ui.go | 2 + internal/version/build.go | 17 +++-- syft/cataloger/apkdb/cataloger.go | 7 +- syft/cataloger/apkdb/parse_apk_db.go | 7 ++ syft/cataloger/bundler/cataloger.go | 7 +- syft/cataloger/bundler/parse_gemfile_lock.go | 5 ++ syft/cataloger/{controller.go => catalog.go} | 71 ++++--------------- syft/cataloger/cataloger.go | 38 +++++++++- syft/cataloger/dpkg/cataloger.go | 7 +- syft/cataloger/dpkg/parse_dpkg_status.go | 6 ++ syft/cataloger/golang/cataloger.go | 7 +- syft/cataloger/golang/parse_go_mod.go | 1 + syft/cataloger/java/archive_parser.go | 14 ++++ syft/cataloger/java/cataloger.go | 7 +- syft/cataloger/javascript/cataloger.go | 9 ++- .../javascript/parse_package_lock.go | 11 ++- syft/cataloger/javascript/parse_yarn_lock.go | 4 ++ syft/cataloger/python/cataloger.go | 10 +-- syft/cataloger/python/parse_poetry_lock.go | 5 ++ syft/cataloger/python/parse_requirements.go | 7 ++ syft/cataloger/python/parse_wheel_egg.go | 11 +++ syft/cataloger/rpmdb/cataloger.go | 7 +- syft/cataloger/rpmdb/parse_rpmdb.go | 5 ++ syft/lib.go | 2 +- syft/pkg/metadata.go | 5 +- ui/event_handlers.go | 8 +++ ui/handler.go | 9 +++ 33 files changed, 250 insertions(+), 86 deletions(-) rename syft/cataloger/{controller.go => catalog.go} (51%) diff --git a/internal/bus/bus.go b/internal/bus/bus.go index fdbca5062..0810c2fc6 100644 --- a/internal/bus/bus.go +++ b/internal/bus/bus.go @@ -1,3 +1,16 @@ +/* +Package bus provides access to a singleton instance of an event bus (provided by the calling application). The event bus +is intended to allow for the syft library to publish events which library consumers can subscribe to. These events +can provide static information, but also have an object as a payload for which the consumer can poll for updates. +This is akin to a logger, except instead of only allowing strings to be logged, rich objects that can be interacted with. + +Note that the singleton instance is only allowed to publish events and not subscribe to them --this is intentional. +Internal library interactions should continue to use traditional in-execution-path approaches for data sharing +(e.g. function returns and channels) and not depend on bus subscriptions for critical interactions (e.g. one part of the +lib publishes an event and another part of the lib subscribes and reacts to that event). The bus is provided only as a +means for consumers to observe events emitted from the library (such as to provide a rich UI) and not to allow +consumers to augment or otherwise change execution. +*/ package bus import "github.com/wagoodman/go-partybus" @@ -5,6 +18,8 @@ import "github.com/wagoodman/go-partybus" var publisher partybus.Publisher var active bool +// SetPublisher sets the singleton event bus publisher. This is optional; if no bus is provided, the library will +// behave no differently than if a bus had been provided. func SetPublisher(p partybus.Publisher) { publisher = p if p != nil { @@ -12,6 +27,7 @@ func SetPublisher(p partybus.Publisher) { } } +// Publish an event onto the bus. If there is no bus set by the calling application, this does nothing. func Publish(event partybus.Event) { if active { publisher.Publish(event) diff --git a/internal/ui/common/event_handlers.go b/internal/ui/common/event_handlers.go index 7cdc090b5..756f74012 100644 --- a/internal/ui/common/event_handlers.go +++ b/internal/ui/common/event_handlers.go @@ -8,6 +8,8 @@ import ( "github.com/wagoodman/go-partybus" ) +// CatalogerFinishedHandler is a UI function for processing the CatalogerFinished bus event, displaying the catalog +// via the given presenter to stdout. func CatalogerFinishedHandler(event partybus.Event) error { // show the report to stdout pres, err := syftEventParsers.ParseCatalogerFinished(event) diff --git a/internal/ui/etui/ephemeral_tui.go b/internal/ui/etui/ephemeral_tui.go index 4fa6c4b3b..8a7295583 100644 --- a/internal/ui/etui/ephemeral_tui.go +++ b/internal/ui/etui/ephemeral_tui.go @@ -1,3 +1,21 @@ +/* +Package etui provides an "ephemeral" terminal user interface to display the application state dynamically. +The terminal is placed into raw mode and the cursor is manipulated to allow for a dynamic, multi-line +UI (provided by the jotframe lib), for this reason all other application mechanisms that write to the screen +must be suppressed before starting (such as logs); since bytes in the device and in application memory combine to make +a shared state, bytes coming from elsewhere to the screen will disrupt this state. + +This UI is primarily driven off of events from the event bus, creating single-line terminal widgets to represent a +published element on the event bus, typically polling the element for the latest state. This allows for the UI to +control update frequency to the screen, provide "liveness" indications that are interpolated between bus events, +and overall loosely couple the bus events from screen interactions. + +By convention, all elements published on the bus should be treated as read-only, and publishers on the bus should +attempt to enforce this when possible by wrapping complex objects with interfaces to prescribe interactions. Also by +convention, each new event that the UI should respond to should be added either in this package as a handler function, +or in the shared ui package as a function on the main handler object. All handler functions should be completed +processing an event before the ETUI exits (coordinated with a sync.WaitGroup) +*/ package etui import ( @@ -18,8 +36,10 @@ import ( "github.com/wagoodman/jotframe/pkg/frame" ) -// TODO: specify per-platform implementations with build tags +// TODO: specify per-platform implementations with build tags (needed when windows is supported by syft) +// setupScreen creates a new jotframe object to manage specific screen lines dynamically, preparing the screen device +// as needed (i.e. setting the terminal to raw mode). func setupScreen(output *os.File) *frame.Frame { config := frame.Config{ PositionPolicy: frame.PolicyFloatForward, @@ -36,6 +56,8 @@ func setupScreen(output *os.File) *frame.Frame { } // nolint:funlen,gocognit +// OutputToEphemeralTUI is a UI function that provides a terminal UI experience without a separate, in-application +// screen buffer. All logging is suppressed, buffered to a string, and output after the ETUI has been torn down. func OutputToEphemeralTUI(workerErrs <-chan error, subscription *partybus.Subscription) error { output := os.Stderr @@ -84,6 +106,7 @@ eventLoop: for { select { case err := <-workerErrs: + // TODO: we should show errors more explicitly in the ETUI if err != nil { return err } diff --git a/internal/ui/etui/internal_event_handlers.go b/internal/ui/etui/internal_event_handlers.go index 50550eaab..3df361a66 100644 --- a/internal/ui/etui/internal_event_handlers.go +++ b/internal/ui/etui/internal_event_handlers.go @@ -13,6 +13,7 @@ import ( "github.com/wagoodman/jotframe/pkg/frame" ) +// appUpdateAvailableHandler is a UI handler function to display a new application version to the top of the screen. func appUpdateAvailableHandler(_ context.Context, fr *frame.Frame, event partybus.Event, _ *sync.WaitGroup) error { newVersion, err := syftEventParsers.ParseAppUpdateAvailable(event) if err != nil { diff --git a/internal/ui/logger_output.go b/internal/ui/logger_output.go index 960799b55..511300edf 100644 --- a/internal/ui/logger_output.go +++ b/internal/ui/logger_output.go @@ -7,6 +7,7 @@ import ( "github.com/wagoodman/go-partybus" ) +// LoggerUI is a UI function that leverages the displays all application logs to the screen. func LoggerUI(workerErrs <-chan error, subscription *partybus.Subscription) error { events := subscription.Events() eventLoop: diff --git a/internal/ui/select.go b/internal/ui/select.go index 7d4b3a82e..b0da99633 100644 --- a/internal/ui/select.go +++ b/internal/ui/select.go @@ -10,6 +10,8 @@ import ( // TODO: build tags to exclude options from windows +// Select is responsible for determining the specific UI function given select user option, the current platform +// config values, and environment status (such as a TTY being present). func Select(verbose, quiet bool) UI { var ui UI diff --git a/internal/ui/ui.go b/internal/ui/ui.go index dd06cadd0..c9b77af3e 100644 --- a/internal/ui/ui.go +++ b/internal/ui/ui.go @@ -4,4 +4,6 @@ import ( "github.com/wagoodman/go-partybus" ) +// UI is a function that takes a channel of errors from the main() worker and a event bus subscription and is +// responsible for displaying pertinent events to the user, on the screen or otherwise. type UI func(<-chan error, *partybus.Subscription) error diff --git a/internal/version/build.go b/internal/version/build.go index bd3e2dd35..139fa5539 100644 --- a/internal/version/build.go +++ b/internal/version/build.go @@ -7,22 +7,25 @@ import ( const valueNotProvided = "[not provided]" +// all variables here are provided as build-time arguments, with clear default values var version = valueNotProvided var gitCommit = valueNotProvided var gitTreeState = valueNotProvided var buildDate = valueNotProvided var platform = fmt.Sprintf("%s/%s", runtime.GOOS, runtime.GOARCH) +// Version defines the application version details (generally from build information) type Version struct { - Version string - GitCommit string - GitTreeState string - BuildDate string - GoVersion string - Compiler string - Platform string + Version string // application semantic version + GitCommit string // git SHA at build-time + GitTreeState string // indication of git tree (either "clean" or "dirty") at build-time + BuildDate string // date of the build + GoVersion string // go runtime version at build-time + Compiler string // compiler used at build-time + Platform string // GOOS and GOARCH at build-time } +// FromBuild provides all version details func FromBuild() Version { return Version{ Version: version, diff --git a/syft/cataloger/apkdb/cataloger.go b/syft/cataloger/apkdb/cataloger.go index 6170e497d..dbe809da0 100644 --- a/syft/cataloger/apkdb/cataloger.go +++ b/syft/cataloger/apkdb/cataloger.go @@ -7,11 +7,13 @@ import ( "github.com/anchore/syft/syft/scope" ) +// Cataloger catalogs pkg.ApkPkg Package Types defined in Alpine DB files. type Cataloger struct { cataloger common.GenericCataloger } -func NewCataloger() *Cataloger { +// New returns a new Alpine DB cataloger object. +func New() *Cataloger { globParsers := map[string]common.ParserFn{ "**/lib/apk/db/installed": parseApkDB, } @@ -21,14 +23,17 @@ func NewCataloger() *Cataloger { } } +// Name returns a string that uniquely describes this cataloger. func (a *Cataloger) Name() string { return "apkdb-cataloger" } +// SelectFiles returns a set of discovered Alpine DB files from the user content source. func (a *Cataloger) SelectFiles(resolver scope.FileResolver) []file.Reference { return a.cataloger.SelectFiles(resolver) } +// Catalog returns the Packages indexed from all Alpine DB files discovered. func (a *Cataloger) Catalog(contents map[file.Reference]string) ([]pkg.Package, error) { return a.cataloger.Catalog(contents, a.Name()) } diff --git a/syft/cataloger/apkdb/parse_apk_db.go b/syft/cataloger/apkdb/parse_apk_db.go index 98591f9f0..999cff739 100644 --- a/syft/cataloger/apkdb/parse_apk_db.go +++ b/syft/cataloger/apkdb/parse_apk_db.go @@ -9,10 +9,16 @@ import ( "strings" "github.com/anchore/syft/internal/log" + "github.com/anchore/syft/syft/cataloger/common" "github.com/anchore/syft/syft/pkg" "github.com/mitchellh/mapstructure" ) +// integrity check +var _ common.ParserFn = parseApkDB + +// parseApkDb parses individual packages from a given Alpine DB file. For more information on specific fields +// see https://wiki.alpinelinux.org/wiki/Apk_spec . func parseApkDB(_ string, reader io.Reader) ([]pkg.Package, error) { packages := make([]pkg.Package, 0) @@ -55,6 +61,7 @@ func parseApkDB(_ string, reader io.Reader) ([]pkg.Package, error) { } // nolint:funlen +// parseApkDBEntry reads and parses a single pkg.ApkMetadata element from the stream, returning nil if their are no more entries. func parseApkDBEntry(reader io.Reader) (*pkg.ApkMetadata, error) { var entry pkg.ApkMetadata pkgFields := make(map[string]interface{}) diff --git a/syft/cataloger/bundler/cataloger.go b/syft/cataloger/bundler/cataloger.go index af7c9f7c6..3f3a37b33 100644 --- a/syft/cataloger/bundler/cataloger.go +++ b/syft/cataloger/bundler/cataloger.go @@ -7,11 +7,13 @@ import ( "github.com/anchore/syft/syft/scope" ) +// Cataloger catalogs pkg.GemPkg Package Types defined in Bundler Gemfile.lock files. type Cataloger struct { cataloger common.GenericCataloger } -func NewCataloger() *Cataloger { +// New returns a new Bundler cataloger object. +func New() *Cataloger { globParsers := map[string]common.ParserFn{ "**/Gemfile.lock": parseGemfileLockEntries, } @@ -21,14 +23,17 @@ func NewCataloger() *Cataloger { } } +// Name returns a string that uniquely describes this cataloger. func (a *Cataloger) Name() string { return "bundler-cataloger" } +// SelectFiles returns a set of discovered Gemfile.lock files from the user content source. func (a *Cataloger) SelectFiles(resolver scope.FileResolver) []file.Reference { return a.cataloger.SelectFiles(resolver) } +// Catalog returns the Packages indexed from all Gemfile.lock files discovered. func (a *Cataloger) Catalog(contents map[file.Reference]string) ([]pkg.Package, error) { return a.cataloger.Catalog(contents, a.Name()) } diff --git a/syft/cataloger/bundler/parse_gemfile_lock.go b/syft/cataloger/bundler/parse_gemfile_lock.go index ae22565d9..d59274289 100644 --- a/syft/cataloger/bundler/parse_gemfile_lock.go +++ b/syft/cataloger/bundler/parse_gemfile_lock.go @@ -6,11 +6,16 @@ import ( "strings" "github.com/anchore/syft/internal" + "github.com/anchore/syft/syft/cataloger/common" "github.com/anchore/syft/syft/pkg" ) +// integrity check +var _ common.ParserFn = parseGemfileLockEntries + var sectionsOfInterest = internal.NewStringSetFromSlice([]string{"GEM"}) +// parseGemfileLockEntries is a parser function for Gemfile.lock contents, returning all Gems discovered. func parseGemfileLockEntries(_ string, reader io.Reader) ([]pkg.Package, error) { pkgs := make([]pkg.Package, 0) scanner := bufio.NewScanner(reader) diff --git a/syft/cataloger/controller.go b/syft/cataloger/catalog.go similarity index 51% rename from syft/cataloger/controller.go rename to syft/cataloger/catalog.go index b82318cb3..ae4e4239d 100644 --- a/syft/cataloger/controller.go +++ b/syft/cataloger/catalog.go @@ -4,14 +4,6 @@ import ( "github.com/anchore/stereoscope/pkg/file" "github.com/anchore/syft/internal/bus" "github.com/anchore/syft/internal/log" - "github.com/anchore/syft/syft/cataloger/apkdb" - "github.com/anchore/syft/syft/cataloger/bundler" - "github.com/anchore/syft/syft/cataloger/dpkg" - golang "github.com/anchore/syft/syft/cataloger/golang" - "github.com/anchore/syft/syft/cataloger/java" - "github.com/anchore/syft/syft/cataloger/javascript" - "github.com/anchore/syft/syft/cataloger/python" - "github.com/anchore/syft/syft/cataloger/rpmdb" "github.com/anchore/syft/syft/event" "github.com/anchore/syft/syft/pkg" "github.com/anchore/syft/syft/scope" @@ -20,54 +12,14 @@ import ( "github.com/wagoodman/go-progress" ) -var controllerInstance controller - -func init() { - controllerInstance = newController() -} - -func Catalogers() []string { - c := make([]string, len(controllerInstance.catalogers)) - for idx, catalog := range controllerInstance.catalogers { - c[idx] = catalog.Name() - } - return c -} - -func Catalog(s scope.Resolver) (*pkg.Catalog, error) { - return controllerInstance.catalog(s) -} - -type controller struct { - catalogers []Cataloger -} - -func newController() controller { - ctrlr := controller{ - catalogers: make([]Cataloger, 0), - } - ctrlr.add(dpkg.NewCataloger()) - ctrlr.add(bundler.NewCataloger()) - ctrlr.add(python.NewCataloger()) - ctrlr.add(rpmdb.NewCataloger()) - ctrlr.add(java.NewCataloger()) - ctrlr.add(apkdb.NewCataloger()) - ctrlr.add(golang.NewCataloger()) - ctrlr.add(javascript.NewCataloger()) - return ctrlr -} - -func (c *controller) add(a Cataloger) { - log.Debugf("adding cataloger: %s", a.Name()) - c.catalogers = append(c.catalogers, a) -} - +// Monitor provides progress-related data for observing the progress of a Catalog() call (published on the event bus). type Monitor struct { - FilesProcessed progress.Monitorable - PackagesDiscovered progress.Monitorable + FilesProcessed progress.Monitorable // the number of files selected and contents analyzed from all registered catalogers + PackagesDiscovered progress.Monitorable // the number of packages discovered from all registered catalogers } -func (c *controller) trackCataloger() (*progress.Manual, *progress.Manual) { +// newMonitor creates a new Monitor object and publishes the object on the bus as a CatalogerStarted event. +func newMonitor() (*progress.Manual, *progress.Manual) { filesProcessed := progress.Manual{} packagesDiscovered := progress.Manual{} @@ -81,20 +33,25 @@ func (c *controller) trackCataloger() (*progress.Manual, *progress.Manual) { return &filesProcessed, &packagesDiscovered } -func (c *controller) catalog(s scope.Resolver) (*pkg.Catalog, error) { +// Catalog a given scope (container image or filesystem) with the given catalogers, returning all discovered packages. +// In order to efficiently retrieve contents from a underlying container image the content fetch requests are +// done in bulk. Specifically, all files of interest are collected from each catalogers and accumulated into a single +// request. +func Catalog(s scope.Resolver, catalogers ...Cataloger) (*pkg.Catalog, error) { catalog := pkg.NewCatalog() fileSelection := make([]file.Reference, 0) - filesProcessed, packagesDiscovered := c.trackCataloger() + filesProcessed, packagesDiscovered := newMonitor() // ask catalogers for files to extract from the image tar - for _, a := range c.catalogers { + for _, a := range catalogers { fileSelection = append(fileSelection, a.SelectFiles(s)...) log.Debugf("cataloger '%s' selected '%d' files", a.Name(), len(fileSelection)) filesProcessed.N += int64(len(fileSelection)) } // fetch contents for requested selection by catalogers + // TODO: we should consider refactoring to return a set of io.Readers instead of the full contents themselves (allow for optional buffering). contents, err := s.MultipleFileContentsByRef(fileSelection...) if err != nil { return nil, err @@ -102,7 +59,7 @@ func (c *controller) catalog(s scope.Resolver) (*pkg.Catalog, error) { // perform analysis, accumulating errors for each failed analysis var errs error - for _, a := range c.catalogers { + for _, a := range catalogers { // TODO: check for multiple rounds of analyses by Iterate error packages, err := a.Catalog(contents) if err != nil { diff --git a/syft/cataloger/cataloger.go b/syft/cataloger/cataloger.go index 1304e1caa..14621c491 100644 --- a/syft/cataloger/cataloger.go +++ b/syft/cataloger/cataloger.go @@ -1,16 +1,48 @@ +/* +Package cataloger provides the ability to process files from a container image or file system and discover packages +(e.g. gems, wheels, jars, rpms, debs, etc.). Specifically, this package contains both a catalog function to utilize all +catalogers defined in child packages as well as the interface definition to implement a cataloger. +*/ package cataloger import ( "github.com/anchore/stereoscope/pkg/file" + "github.com/anchore/syft/syft/cataloger/apkdb" + "github.com/anchore/syft/syft/cataloger/bundler" + "github.com/anchore/syft/syft/cataloger/dpkg" + "github.com/anchore/syft/syft/cataloger/golang" + "github.com/anchore/syft/syft/cataloger/java" + "github.com/anchore/syft/syft/cataloger/javascript" + "github.com/anchore/syft/syft/cataloger/python" + "github.com/anchore/syft/syft/cataloger/rpmdb" "github.com/anchore/syft/syft/pkg" "github.com/anchore/syft/syft/scope" ) +// Cataloger describes behavior for an object to participate in parsing container image or file system +// contents for the purpose of discovering Packages. Each concrete implementation should focus on discovering Packages +// for a specific Package Type or ecosystem. type Cataloger interface { + // Name returns a string that uniquely describes a cataloger Name() string - // TODO: add ID / Name for catalog for uniquely identifying this cataloger type + // SelectFiles discovers and returns specific files that the cataloger would like to inspect the contents of. SelectFiles(scope.FileResolver) []file.Reference - // NOTE: one of the errors which is returned is "IterationNeeded", which indicates to the driver to - // continue with another Select/Catalog pass + // Catalog is given the file contents and should return any discovered Packages after analyzing the contents. Catalog(map[file.Reference]string) ([]pkg.Package, error) + // TODO: add "IterationNeeded" error to indicate to the driver to continue with another Select/Catalog pass + // TODO: we should consider refactoring to return a set of io.Readers instead of the full contents themselves (allow for optional buffering). +} + +// All returns a slice of all locally defined catalogers (defined in child packages). +func All() []Cataloger { + return []Cataloger{ + dpkg.New(), + bundler.New(), + python.New(), + rpmdb.New(), + java.New(), + apkdb.New(), + golang.New(), + javascript.New(), + } } diff --git a/syft/cataloger/dpkg/cataloger.go b/syft/cataloger/dpkg/cataloger.go index d397db830..3a053d566 100644 --- a/syft/cataloger/dpkg/cataloger.go +++ b/syft/cataloger/dpkg/cataloger.go @@ -7,11 +7,13 @@ import ( "github.com/anchore/syft/syft/scope" ) +// Cataloger catalogs pkg.DebPkg Package Types defined in DPKG status files. type Cataloger struct { cataloger common.GenericCataloger } -func NewCataloger() *Cataloger { +// New returns a new Deb package cataloger object. +func New() *Cataloger { globParsers := map[string]common.ParserFn{ "**/var/lib/dpkg/status": parseDpkgStatus, } @@ -21,14 +23,17 @@ func NewCataloger() *Cataloger { } } +// Name returns a string that uniquely describes this cataloger. func (a *Cataloger) Name() string { return "dpkg-cataloger" } +// SelectFiles returns a set of discovered DPKG status files from the user content source. func (a *Cataloger) SelectFiles(resolver scope.FileResolver) []file.Reference { return a.cataloger.SelectFiles(resolver) } +// Catalog returns the Packages indexed from all DPKG status files discovered. func (a *Cataloger) Catalog(contents map[file.Reference]string) ([]pkg.Package, error) { return a.cataloger.Catalog(contents, a.Name()) } diff --git a/syft/cataloger/dpkg/parse_dpkg_status.go b/syft/cataloger/dpkg/parse_dpkg_status.go index 92c40b8d0..ded470170 100644 --- a/syft/cataloger/dpkg/parse_dpkg_status.go +++ b/syft/cataloger/dpkg/parse_dpkg_status.go @@ -6,12 +6,17 @@ import ( "io" "strings" + "github.com/anchore/syft/syft/cataloger/common" "github.com/anchore/syft/syft/pkg" "github.com/mitchellh/mapstructure" ) +// integrity check +var _ common.ParserFn = parseDpkgStatus + var errEndOfPackages = fmt.Errorf("no more packages to read") +// parseDpkgStatus is a parser function for Debian DB status contents, returning all Debian packages listed. func parseDpkgStatus(_ string, reader io.Reader) ([]pkg.Package, error) { buffedReader := bufio.NewReader(reader) var packages = make([]pkg.Package, 0) @@ -35,6 +40,7 @@ func parseDpkgStatus(_ string, reader io.Reader) ([]pkg.Package, error) { return packages, nil } +// parseDpkgStatusEntry returns an individual Dpkg entry, or returns errEndOfPackages if there are no more packages to parse from the reader. func parseDpkgStatusEntry(reader *bufio.Reader) (entry pkg.DpkgMetadata, err error) { dpkgFields := make(map[string]string) var key string diff --git a/syft/cataloger/golang/cataloger.go b/syft/cataloger/golang/cataloger.go index 762663f5b..2b44da5ee 100644 --- a/syft/cataloger/golang/cataloger.go +++ b/syft/cataloger/golang/cataloger.go @@ -7,11 +7,13 @@ import ( "github.com/anchore/syft/syft/scope" ) +// Cataloger catalogs pkg.GoModulePkg Package Types defined in go.mod files. type Cataloger struct { cataloger common.GenericCataloger } -func NewCataloger() *Cataloger { +// New returns a new Go module cataloger object. +func New() *Cataloger { globParsers := map[string]common.ParserFn{ "**/go.mod": parseGoMod, } @@ -21,14 +23,17 @@ func NewCataloger() *Cataloger { } } +// Name returns a string that uniquely describes this cataloger. func (a *Cataloger) Name() string { return "go-cataloger" } +// SelectFiles returns a set of discovered go.mod files from the user content source. func (a *Cataloger) SelectFiles(resolver scope.FileResolver) []file.Reference { return a.cataloger.SelectFiles(resolver) } +// Catalog returns the Packages indexed from all go.mod files discovered. func (a *Cataloger) Catalog(contents map[file.Reference]string) ([]pkg.Package, error) { return a.cataloger.Catalog(contents, a.Name()) } diff --git a/syft/cataloger/golang/parse_go_mod.go b/syft/cataloger/golang/parse_go_mod.go index 690d5ae3b..96d904bc8 100644 --- a/syft/cataloger/golang/parse_go_mod.go +++ b/syft/cataloger/golang/parse_go_mod.go @@ -11,6 +11,7 @@ import ( "github.com/anchore/syft/syft/pkg" ) +// parseGoMod takes a go.mod and lists all packages discovered. func parseGoMod(path string, reader io.Reader) ([]pkg.Package, error) { packages := make(map[string]pkg.Package) diff --git a/syft/cataloger/java/archive_parser.go b/syft/cataloger/java/archive_parser.go index ab8edc8cf..adb699d8c 100644 --- a/syft/cataloger/java/archive_parser.go +++ b/syft/cataloger/java/archive_parser.go @@ -7,9 +7,13 @@ import ( "github.com/anchore/syft/internal" "github.com/anchore/syft/internal/file" + "github.com/anchore/syft/syft/cataloger/common" "github.com/anchore/syft/syft/pkg" ) +// integrity check +var _ common.ParserFn = parseJavaArchive + var archiveFormatGlobs = []string{ "**/*.jar", "**/*.war", @@ -28,6 +32,7 @@ type archiveParser struct { detectNested bool } +// parseJavaArchive is a parser function for java archive contents, returning all Java libraries and nested archives. func parseJavaArchive(virtualPath string, reader io.Reader) ([]pkg.Package, error) { parser, cleanupFn, err := newJavaArchiveParser(virtualPath, reader, true) // note: even on error, we should always run cleanup functions @@ -38,6 +43,7 @@ func parseJavaArchive(virtualPath string, reader io.Reader) ([]pkg.Package, erro return parser.parse() } +// uniquePkgKey creates a unique string to identify the given package. func uniquePkgKey(p *pkg.Package) string { if p == nil { return "" @@ -45,6 +51,8 @@ func uniquePkgKey(p *pkg.Package) string { return fmt.Sprintf("%s|%s", p.Name, p.Version) } +// newJavaArchiveParser returns a new java archive parser object for the given archive. Can be configured to discover +// and parse nested archives or ignore them. func newJavaArchiveParser(virtualPath string, reader io.Reader, detectNested bool) (*archiveParser, func(), error) { contentPath, archivePath, cleanupFn, err := saveArchiveToTmp(reader) if err != nil { @@ -67,6 +75,7 @@ func newJavaArchiveParser(virtualPath string, reader io.Reader, detectNested boo }, cleanupFn, nil } +// parse the loaded archive and return all packages found. func (j *archiveParser) parse() ([]pkg.Package, error) { var pkgs = make([]pkg.Package, 0) @@ -106,6 +115,7 @@ func (j *archiveParser) parse() ([]pkg.Package, error) { return pkgs, nil } +// discoverMainPackage parses the root Java manifest used as the parent package to all discovered nested packages. func (j *archiveParser) discoverMainPackage() (*pkg.Package, error) { // search and parse java manifest files manifestMatches := j.fileManifest.GlobMatch(manifestPath) @@ -140,6 +150,8 @@ func (j *archiveParser) discoverMainPackage() (*pkg.Package, error) { }, nil } +// discoverPkgsFromPomProperties parses Maven POM properties for a given parent package, returning all listed Java packages found and +// associating each discovered package to the given parent package. func (j *archiveParser) discoverPkgsFromPomProperties(parentPkg *pkg.Package) ([]pkg.Package, error) { var pkgs = make([]pkg.Package, 0) parentKey := uniquePkgKey(parentPkg) @@ -192,6 +204,8 @@ func (j *archiveParser) discoverPkgsFromPomProperties(parentPkg *pkg.Package) ([ return pkgs, nil } +// discoverPkgsFromNestedArchives finds Java archives within Java archives, returning all listed Java packages found and +// associating each discovered package to the given parent package. func (j *archiveParser) discoverPkgsFromNestedArchives(parentPkg *pkg.Package) ([]pkg.Package, error) { var pkgs = make([]pkg.Package, 0) diff --git a/syft/cataloger/java/cataloger.go b/syft/cataloger/java/cataloger.go index a7b3cb322..97da1e702 100644 --- a/syft/cataloger/java/cataloger.go +++ b/syft/cataloger/java/cataloger.go @@ -7,11 +7,13 @@ import ( "github.com/anchore/syft/syft/scope" ) +// Cataloger catalogs pkg.JavaPkg and pkg.JenkinsPluginPkg Package Types defined in java archive files. type Cataloger struct { cataloger common.GenericCataloger } -func NewCataloger() *Cataloger { +// New returns a new Java archive cataloger object. +func New() *Cataloger { globParsers := make(map[string]common.ParserFn) for _, pattern := range archiveFormatGlobs { globParsers[pattern] = parseJavaArchive @@ -22,14 +24,17 @@ func NewCataloger() *Cataloger { } } +// Name returns a string that uniquely describes this cataloger. func (a *Cataloger) Name() string { return "java-cataloger" } +// SelectFiles returns a set of discovered Java archive files from the user content source. func (a *Cataloger) SelectFiles(resolver scope.FileResolver) []file.Reference { return a.cataloger.SelectFiles(resolver) } +// Catalog returns the Packages indexed from all Java archive files discovered. func (a *Cataloger) Catalog(contents map[file.Reference]string) ([]pkg.Package, error) { return a.cataloger.Catalog(contents, a.Name()) } diff --git a/syft/cataloger/javascript/cataloger.go b/syft/cataloger/javascript/cataloger.go index cd926b1be..723a42a94 100644 --- a/syft/cataloger/javascript/cataloger.go +++ b/syft/cataloger/javascript/cataloger.go @@ -7,11 +7,13 @@ import ( "github.com/anchore/syft/syft/scope" ) +// Cataloger catalogs pkg.YarnPkg and pkg.NpmPkg Package Types defined in package-lock.json and yarn.lock files. type Cataloger struct { cataloger common.GenericCataloger } -func NewCataloger() *Cataloger { +// New returns a new JavaScript cataloger object. +func New() *Cataloger { globParsers := map[string]common.ParserFn{ "**/package-lock.json": parsePackageLock, "**/yarn.lock": parseYarnLock, @@ -22,14 +24,17 @@ func NewCataloger() *Cataloger { } } +// Name returns a string that uniquely describes this cataloger. func (a *Cataloger) Name() string { - return "npm-cataloger" + return "javascript-cataloger" } +// SelectFiles returns a set of discovered Javascript ecosystem files from the user content source. func (a *Cataloger) SelectFiles(resolver scope.FileResolver) []file.Reference { return a.cataloger.SelectFiles(resolver) } +// Catalog returns the Packages indexed from all Javascript ecosystem files discovered. func (a *Cataloger) Catalog(contents map[file.Reference]string) ([]pkg.Package, error) { return a.cataloger.Catalog(contents, a.Name()) } diff --git a/syft/cataloger/javascript/parse_package_lock.go b/syft/cataloger/javascript/parse_package_lock.go index 9027abd6f..4018cbf63 100644 --- a/syft/cataloger/javascript/parse_package_lock.go +++ b/syft/cataloger/javascript/parse_package_lock.go @@ -5,15 +5,21 @@ import ( "fmt" "io" + "github.com/anchore/syft/syft/cataloger/common" "github.com/anchore/syft/syft/pkg" ) +// integrity check +var _ common.ParserFn = parsePackageLock + +// PackageLock represents a JavaScript package.lock json file type PackageLock struct { Requires bool `json:"requires"` LockfileVersion int `json:"lockfileVersion"` - Dependencies Dependencies + Dependencies map[string]Dependency } +// Dependency represents a single package dependency listed in the package.lock json file type Dependency struct { Version string `json:"version"` Resolved string `json:"resolved"` @@ -21,8 +27,7 @@ type Dependency struct { Requires map[string]string } -type Dependencies map[string]Dependency - +// parsePackageLock parses a package.lock and returns the discovered JavaScript packages. func parsePackageLock(_ string, reader io.Reader) ([]pkg.Package, error) { packages := make([]pkg.Package, 0) dec := json.NewDecoder(reader) diff --git a/syft/cataloger/javascript/parse_yarn_lock.go b/syft/cataloger/javascript/parse_yarn_lock.go index 6e245c739..6ede34fa9 100644 --- a/syft/cataloger/javascript/parse_yarn_lock.go +++ b/syft/cataloger/javascript/parse_yarn_lock.go @@ -8,9 +8,13 @@ import ( "strings" "github.com/anchore/syft/internal/log" + "github.com/anchore/syft/syft/cataloger/common" "github.com/anchore/syft/syft/pkg" ) +// integrity check +var _ common.ParserFn = parseYarnLock + var composedNameExp = regexp.MustCompile("^\"(@{1}[^@]+)") var simpleNameExp = regexp.MustCompile(`^[a-zA-Z\-]+@`) var versionExp = regexp.MustCompile(`^\W+(version)\W+`) diff --git a/syft/cataloger/python/cataloger.go b/syft/cataloger/python/cataloger.go index 3cbcd5b50..e92716413 100644 --- a/syft/cataloger/python/cataloger.go +++ b/syft/cataloger/python/cataloger.go @@ -7,14 +7,13 @@ import ( "github.com/anchore/syft/syft/scope" ) +// Cataloger catalogs pkg.WheelPkg, pkg.EggPkg, and pkg.PythonRequirementsPkg Package Types defined in Python ecosystem files. type Cataloger struct { cataloger common.GenericCataloger } -func NewCataloger() *Cataloger { - // we want to match on partial dir names - // /home/user/requests-2.10.0.dist-info/METADATA - // /home/user/requests-2.10.0/dist-info/METADATA +// New returns a new Python cataloger object. +func New() *Cataloger { globParsers := map[string]common.ParserFn{ "**/*egg-info/PKG-INFO": parseEggMetadata, "**/*dist-info/METADATA": parseWheelMetadata, @@ -27,14 +26,17 @@ func NewCataloger() *Cataloger { } } +// Name returns a string that uniquely describes this cataloger. func (a *Cataloger) Name() string { return "python-cataloger" } +// SelectFiles returns a set of discovered Python ecosystem files from the user content source. func (a *Cataloger) SelectFiles(resolver scope.FileResolver) []file.Reference { return a.cataloger.SelectFiles(resolver) } +// Catalog returns the Packages indexed from all Python ecosystem files discovered. func (a *Cataloger) Catalog(contents map[file.Reference]string) ([]pkg.Package, error) { return a.cataloger.Catalog(contents, a.Name()) } diff --git a/syft/cataloger/python/parse_poetry_lock.go b/syft/cataloger/python/parse_poetry_lock.go index ba0cc5bf0..6b08098a4 100644 --- a/syft/cataloger/python/parse_poetry_lock.go +++ b/syft/cataloger/python/parse_poetry_lock.go @@ -4,10 +4,15 @@ import ( "fmt" "io" + "github.com/anchore/syft/syft/cataloger/common" "github.com/anchore/syft/syft/pkg" "github.com/pelletier/go-toml" ) +// integrity check +var _ common.ParserFn = parsePoetryLock + +// parsePoetryLock is a parser function for poetry.lock contents, returning all python packages discovered. func parsePoetryLock(_ string, reader io.Reader) ([]pkg.Package, error) { tree, err := toml.LoadReader(reader) if err != nil { diff --git a/syft/cataloger/python/parse_requirements.go b/syft/cataloger/python/parse_requirements.go index 60c38518d..80c95eecc 100644 --- a/syft/cataloger/python/parse_requirements.go +++ b/syft/cataloger/python/parse_requirements.go @@ -6,9 +6,15 @@ import ( "io" "strings" + "github.com/anchore/syft/syft/cataloger/common" "github.com/anchore/syft/syft/pkg" ) +// integrity check +var _ common.ParserFn = parseRequirementsTxt + +// parseRequirementsTxt takes a Python requirements.txt file, returning all Python packages that are locked to a +// specific version. func parseRequirementsTxt(_ string, reader io.Reader) ([]pkg.Package, error) { packages := make([]pkg.Package, 0) @@ -55,6 +61,7 @@ func parseRequirementsTxt(_ string, reader io.Reader) ([]pkg.Package, error) { return packages, nil } +// removeTrailingComment takes a requirements.txt line and strips off comment strings. func removeTrailingComment(line string) string { parts := strings.Split(line, "#") switch len(parts) { diff --git a/syft/cataloger/python/parse_wheel_egg.go b/syft/cataloger/python/parse_wheel_egg.go index 4cde82ebc..fc5100687 100644 --- a/syft/cataloger/python/parse_wheel_egg.go +++ b/syft/cataloger/python/parse_wheel_egg.go @@ -6,9 +6,16 @@ import ( "io" "strings" + "github.com/anchore/syft/syft/cataloger/common" "github.com/anchore/syft/syft/pkg" ) +// integrity check +var _ common.ParserFn = parseWheelMetadata +var _ common.ParserFn = parseEggMetadata + +// parseWheelMetadata is a parser function for individual Python Wheel metadata file contents, returning all Python +// packages listed. func parseWheelMetadata(_ string, reader io.Reader) ([]pkg.Package, error) { packages, err := parseWheelOrEggMetadata(reader) for idx := range packages { @@ -17,6 +24,8 @@ func parseWheelMetadata(_ string, reader io.Reader) ([]pkg.Package, error) { return packages, err } +// parseEggMetadata is a parser function for individual Python Egg metadata file contents, returning all Python +// packages listed. func parseEggMetadata(_ string, reader io.Reader) ([]pkg.Package, error) { packages, err := parseWheelOrEggMetadata(reader) for idx := range packages { @@ -25,6 +34,8 @@ func parseEggMetadata(_ string, reader io.Reader) ([]pkg.Package, error) { return packages, err } +// parseWheelOrEggMetadata takes a Python Egg or Wheel (which share the same format and values for our purposes), +// returning all Python packages listed. func parseWheelOrEggMetadata(reader io.Reader) ([]pkg.Package, error) { fields := make(map[string]string) var key string diff --git a/syft/cataloger/rpmdb/cataloger.go b/syft/cataloger/rpmdb/cataloger.go index 7083639c2..44b5c9758 100644 --- a/syft/cataloger/rpmdb/cataloger.go +++ b/syft/cataloger/rpmdb/cataloger.go @@ -7,11 +7,13 @@ import ( "github.com/anchore/syft/syft/scope" ) +// Cataloger catalogs pkg.RpmPkg Package Types defined in RPM DB files. type Cataloger struct { cataloger common.GenericCataloger } -func NewCataloger() *Cataloger { +// New returns a new RPM DB cataloger object. +func New() *Cataloger { globParsers := map[string]common.ParserFn{ "**/var/lib/rpm/Packages": parseRpmDB, } @@ -21,14 +23,17 @@ func NewCataloger() *Cataloger { } } +// Name returns a string that uniquely describes this cataloger. func (a *Cataloger) Name() string { return "rpmdb-cataloger" } +// SelectFiles returns a set of discovered RPM DB files from the user content source. func (a *Cataloger) SelectFiles(resolver scope.FileResolver) []file.Reference { return a.cataloger.SelectFiles(resolver) } +// Catalog returns the Packages indexed from all RPM DB files discovered. func (a *Cataloger) Catalog(contents map[file.Reference]string) ([]pkg.Package, error) { return a.cataloger.Catalog(contents, a.Name()) } diff --git a/syft/cataloger/rpmdb/parse_rpmdb.go b/syft/cataloger/rpmdb/parse_rpmdb.go index 169bced1d..c899910ae 100644 --- a/syft/cataloger/rpmdb/parse_rpmdb.go +++ b/syft/cataloger/rpmdb/parse_rpmdb.go @@ -9,9 +9,14 @@ import ( rpmdb "github.com/anchore/go-rpmdb/pkg" "github.com/anchore/syft/internal" "github.com/anchore/syft/internal/log" + "github.com/anchore/syft/syft/cataloger/common" "github.com/anchore/syft/syft/pkg" ) +// integrity check +var _ common.ParserFn = parseRpmDB + +// parseApkDb parses an "Packages" RPM DB and returns the Packages listed within it. func parseRpmDB(_ string, reader io.Reader) ([]pkg.Package, error) { f, err := ioutil.TempFile("", internal.ApplicationName+"-rpmdb") if err != nil { diff --git a/syft/lib.go b/syft/lib.go index adbd48e4b..3d2810815 100644 --- a/syft/lib.go +++ b/syft/lib.go @@ -41,7 +41,7 @@ func IdentifyDistro(s scope.Scope) distro.Distro { func CatalogFromScope(s scope.Scope) (*pkg.Catalog, error) { log.Info("building the catalog") - return cataloger.Catalog(s) + return cataloger.Catalog(s, cataloger.All()...) } func SetLogger(logger logger.Logger) { diff --git a/syft/pkg/metadata.go b/syft/pkg/metadata.go index c2bac29b1..697ef3b5f 100644 --- a/syft/pkg/metadata.go +++ b/syft/pkg/metadata.go @@ -1,7 +1,7 @@ package pkg -// Available fields are described at http://manpages.ubuntu.com/manpages/xenial/man1/dpkg-query.1.html -// in the --showformat section +// DpkgMetadata represents all captured data for a Debian package DB entry; available fields are described +// at http://manpages.ubuntu.com/manpages/xenial/man1/dpkg-query.1.html in the --showformat section. type DpkgMetadata struct { Package string `mapstructure:"Package" json:"package"` Source string `mapstructure:"Source" json:"source"` @@ -9,6 +9,7 @@ type DpkgMetadata struct { // TODO: consider keeping the remaining values as an embedded map } +// RpmMetadata represents all captured data for a RHEL package DB entry. type RpmMetadata struct { Version string `mapstructure:"Version" json:"version"` Epoch int `mapstructure:"Epoch" json:"epoch"` diff --git a/ui/event_handlers.go b/ui/event_handlers.go index 14bf13b6c..8d71e94db 100644 --- a/ui/event_handlers.go +++ b/ui/event_handlers.go @@ -36,6 +36,8 @@ var ( dockerPullStageChars = strings.Split("▁▃▄▅▆▇█", "") ) +// startProcess is a helper function for providing common elements for long-running UI elements (such as a +// progress bar formatter and status spinner) func startProcess() (format.Simple, *common.Spinner) { width, _ := frame.GetTerminalSize() barWidth := int(0.25 * float64(width)) @@ -48,6 +50,7 @@ func startProcess() (format.Simple, *common.Spinner) { return formatter, &spinner } +// formatDockerPullPhase returns a single character that represents the status of a layer pull. func formatDockerPullPhase(phase docker.PullPhase, inputStr string) string { switch phase { case docker.WaitingPhase: @@ -69,6 +72,7 @@ func formatDockerPullPhase(phase docker.PullPhase, inputStr string) string { } // nolint:funlen +// formatDockerImagePullStatus writes the docker image pull status summarized into a single line for the given state. func formatDockerImagePullStatus(pullStatus *docker.PullStatus, spinner *common.Spinner, line *frame.Line) { var size, current uint64 @@ -136,6 +140,7 @@ func formatDockerImagePullStatus(pullStatus *docker.PullStatus, spinner *common. _, _ = io.WriteString(line, fmt.Sprintf(statusTitleTemplate+"%s%s", spin, title, progStr, auxInfo)) } +// PullDockerImageHandler periodically writes a formatted line widget representing a docker image pull event. func PullDockerImageHandler(ctx context.Context, fr *frame.Frame, event partybus.Event, wg *sync.WaitGroup) error { _, pullStatus, err := stereoEventParsers.ParsePullDockerImage(event) if err != nil { @@ -175,6 +180,7 @@ func PullDockerImageHandler(ctx context.Context, fr *frame.Frame, event partybus return err } +// FetchImageHandler periodically writes a the image save and write-to-disk process in the form of a progress bar. func FetchImageHandler(ctx context.Context, fr *frame.Frame, event partybus.Event, wg *sync.WaitGroup) error { _, prog, err := stereoEventParsers.ParseFetchImage(event) if err != nil { @@ -217,6 +223,7 @@ func FetchImageHandler(ctx context.Context, fr *frame.Frame, event partybus.Even return err } +// ReadImageHandler periodically writes a the image read/parse/build-tree status in the form of a progress bar. func ReadImageHandler(ctx context.Context, fr *frame.Frame, event partybus.Event, wg *sync.WaitGroup) error { _, prog, err := stereoEventParsers.ParseReadImage(event) if err != nil { @@ -260,6 +267,7 @@ func ReadImageHandler(ctx context.Context, fr *frame.Frame, event partybus.Event return nil } +// CatalogerStartedHandler periodically writes catalog statistics to a single line. func CatalogerStartedHandler(ctx context.Context, fr *frame.Frame, event partybus.Event, wg *sync.WaitGroup) error { monitor, err := syftEventParsers.ParseCatalogerStarted(event) if err != nil { diff --git a/ui/handler.go b/ui/handler.go index 105b3b0ee..e17be449c 100644 --- a/ui/handler.go +++ b/ui/handler.go @@ -1,3 +1,8 @@ +/* +Package ui provides all public UI elements intended to be repurposed in other applications. Specifically, a single +Handler object is provided to allow consuming applications (such as grype) to check if there are UI elements the handler +can respond to (given a specific event type) and handle the event in context of the given screen frame object. +*/ package ui import ( @@ -10,13 +15,16 @@ import ( "github.com/wagoodman/jotframe/pkg/frame" ) +// Handler is an aggregated event handler for the set of supported events (PullDockerImage, ReadImage, FetchImage, CatalogerStarted) type Handler struct { } +// NewHandler returns an empty Handler func NewHandler() *Handler { return &Handler{} } +// RespondsTo indicates if the handler is capable of handling the given event. func (r *Handler) RespondsTo(event partybus.Event) bool { switch event.Type { case stereoscopeEvent.PullDockerImage, stereoscopeEvent.ReadImage, stereoscopeEvent.FetchImage, syftEvent.CatalogerStarted: @@ -26,6 +34,7 @@ func (r *Handler) RespondsTo(event partybus.Event) bool { } } +// Handle calls the specific event handler for the given event within the context of the screen frame. func (r *Handler) Handle(ctx context.Context, fr *frame.Frame, event partybus.Event, wg *sync.WaitGroup) error { switch event.Type { case stereoscopeEvent.PullDockerImage: