oss/syft
2
0
Fork 0
mirror of https://github.com/anchore/syft.git synced 2026-11-01 23:16:30 +01:00
Code Issues Packages Projects Releases Wiki Activity

Generalize UI events for cataloging tasks (#2369)

Browse Source 
* generalize ui events for cataloging tasks

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

* moderate review comments

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

* incorporate review comments

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

* rename cataloger task progress object

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

* migrate cataloger task fn to bus helper

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>

---------

Signed-off-by: Alex Goodman <wagoodman@users.noreply.github.com>
This commit is contained in:
Alex Goodman
2023-11-30 16:25:50 +00:00
committed by GitHub
parent b943da6433
commit 4adfbeb5f0
28 changed files with 427 additions and 807 deletions
Download Patch File Download Diff File Expand all files Collapse all files
syft/event/event.go
-9
View File
@@ -14,15 +14,6 @@ const (
// Events from the syft library
// PackageCatalogerStarted is a partybus event that occurs when the package cataloging has begun
PackageCatalogerStarted partybus.EventType = typePrefix + "-package-cataloger-started-event"
// FileMetadataCatalogerStarted is a partybus event that occurs when the file metadata cataloging has begun
FileMetadataCatalogerStarted partybus.EventType = typePrefix + "-file-metadata-cataloger-started-event"
// FileDigestsCatalogerStarted is a partybus event that occurs when the file digests cataloging has begun
FileDigestsCatalogerStarted partybus.EventType = typePrefix + "-file-digests-cataloger-started-event"
// FileIndexingStarted is a partybus event that occurs when the directory resolver begins indexing a filesystem
FileIndexingStarted partybus.EventType = typePrefix + "-file-indexing-started-event"
syft/event/monitor/cataloger_task.go
-55
View File
@@ -1,55 +0,0 @@
package monitor
import (
"github.com/wagoodman/go-partybus"
"github.com/wagoodman/go-progress"
"github.com/anchore/syft/internal/bus"
"github.com/anchore/syft/syft/event"
)
// TODO: this should be refactored to support read-only/write-only access using idioms of the progress lib
type CatalogerTask struct {
prog *progress.Manual
// Title
Title string
// TitleOnCompletion a string to use as title when completed
TitleOnCompletion string
// SubStatus indicates this progress should be rendered as a sub-item
SubStatus bool
// RemoveOnCompletion indicates this progress line will be removed when completed
RemoveOnCompletion bool
// value is the value to display -- not public as SetValue needs to be called to initialize this progress
value string
}
func (e *CatalogerTask) init() {
e.prog = progress.NewManual(-1)
bus.Publish(partybus.Event{
Type: event.CatalogerTaskStarted,
Source: e,
})
}
func (e *CatalogerTask) SetCompleted() {
if e.prog != nil {
e.prog.SetCompleted()
}
}
func (e *CatalogerTask) SetValue(value string) {
if e.prog == nil {
e.init()
}
e.value = value
}
func (e *CatalogerTask) GetValue() string {
return e.value
}
func (e *CatalogerTask) GetMonitor() *progress.Manual {
return e.prog
}
syft/event/monitor/cataloger_task_progress.go
+10
View File
@@ -0,0 +1,10 @@
package monitor
import (
"github.com/wagoodman/go-progress"
)
type CatalogerTaskProgress struct {
*progress.AtomicStage
*progress.Manual
}
syft/event/monitor/generic_task.go
+15 -2
View File
@@ -18,6 +18,19 @@ type Title struct {
}
type GenericTask struct {
Title Title
Context string
// required fields
Title Title
// optional format fields
HideOnSuccess bool
HideStageOnSuccess bool
// optional fields
ID string
ParentID string
Context string
}
syft/event/parsers/parsers.go
+12 -45
View File
@@ -12,7 +12,6 @@ import (
"github.com/anchore/syft/syft/event"
"github.com/anchore/syft/syft/event/monitor"
"github.com/anchore/syft/syft/pkg/cataloger"
)
type ErrBadPayload struct {
@@ -40,45 +39,6 @@ func checkEventType(actual, expected partybus.EventType) error {
return nil
}
func ParsePackageCatalogerStarted(e partybus.Event) (*cataloger.Monitor, error) {
if err := checkEventType(e.Type, event.PackageCatalogerStarted); err != nil {
return nil, err
}
monitor, ok := e.Value.(cataloger.Monitor)
if !ok {
return nil, newPayloadErr(e.Type, "Value", e.Value)
}
return &monitor, nil
}
func ParseFileMetadataCatalogingStarted(e partybus.Event) (progress.StagedProgressable, error) {
if err := checkEventType(e.Type, event.FileMetadataCatalogerStarted); err != nil {
return nil, err
}
prog, ok := e.Value.(progress.StagedProgressable)
if !ok {
return nil, newPayloadErr(e.Type, "Value", e.Value)
}
return prog, nil
}
func ParseFileDigestsCatalogingStarted(e partybus.Event) (progress.StagedProgressable, error) {
if err := checkEventType(e.Type, event.FileDigestsCatalogerStarted); err != nil {
return nil, err
}
prog, ok := e.Value.(progress.StagedProgressable)
if !ok {
return nil, newPayloadErr(e.Type, "Value", e.Value)
}
return prog, nil
}
func ParseFileIndexingStarted(e partybus.Event) (string, progress.StagedProgressable, error) {
if err := checkEventType(e.Type, event.FileIndexingStarted); err != nil {
return "", nil, err
@@ -97,17 +57,24 @@ func ParseFileIndexingStarted(e partybus.Event) (string, progress.StagedProgress
return path, prog, nil
}
func ParseCatalogerTaskStarted(e partybus.Event) (*monitor.CatalogerTask, error) {
func ParseCatalogerTaskStarted(e partybus.Event) (progress.StagedProgressable, *monitor.GenericTask, error) {
if err := checkEventType(e.Type, event.CatalogerTaskStarted); err != nil {
return nil, err
return nil, nil, err
}
source, ok := e.Source.(*monitor.CatalogerTask)
var mon progress.StagedProgressable
source, ok := e.Source.(monitor.GenericTask)
if !ok {
return nil, newPayloadErr(e.Type, "Source", e.Source)
return nil, nil, newPayloadErr(e.Type, "Source", e.Source)
}
return source, nil
mon, ok = e.Value.(progress.StagedProgressable)
if !ok {
mon = nil
}
return mon, &source, nil
}
func ParseAttestationStartedEvent(e partybus.Event) (io.Reader, progress.Progressable, *monitor.GenericTask, error) {
syft/file/cataloger/filedigest/cataloger.go
+19 -19
View File
@@ -3,16 +3,16 @@ package filedigest
import (
"crypto"
"errors"
"fmt"
"github.com/wagoodman/go-partybus"
"github.com/wagoodman/go-progress"
"github.com/dustin/go-humanize"
stereoscopeFile "github.com/anchore/stereoscope/pkg/file"
"github.com/anchore/syft/internal"
"github.com/anchore/syft/internal/bus"
intFile "github.com/anchore/syft/internal/file"
"github.com/anchore/syft/internal/log"
"github.com/anchore/syft/syft/event"
"github.com/anchore/syft/syft/event/monitor"
"github.com/anchore/syft/syft/file"
intCataloger "github.com/anchore/syft/syft/file/cataloger/internal"
)
@@ -41,9 +41,11 @@ func (i *Cataloger) Catalog(resolver file.Resolver, coordinates ...file.Coordina
}
}
stage, prog := digestsCatalogingProgress(int64(len(locations)))
prog := digestsCatalogingProgress(int64(len(locations)))
for _, location := range locations {
stage.Current = location.RealPath
prog.Increment()
prog.AtomicStage.Set(location.Path())
result, err := i.catalogLocation(resolver, location)
if errors.Is(err, ErrUndigestableFile) {
@@ -61,8 +63,12 @@ func (i *Cataloger) Catalog(resolver file.Resolver, coordinates ...file.Coordina
prog.Increment()
results[location.Coordinates] = result
}
log.Debugf("file digests cataloger processed %d files", prog.Current())
prog.AtomicStage.Set(fmt.Sprintf("%s digests", humanize.Comma(prog.Current())))
prog.SetCompleted()
return results, nil
}
@@ -91,20 +97,14 @@ func (i *Cataloger) catalogLocation(resolver file.Resolver, location file.Locati
return digests, nil
}
func digestsCatalogingProgress(locations int64) (*progress.Stage, *progress.Manual) {
stage := &progress.Stage{}
prog := progress.NewManual(locations)
bus.Publish(partybus.Event{
Type: event.FileDigestsCatalogerStarted,
Value: struct {
progress.Stager
progress.Progressable
}{
Stager: progress.Stager(stage),
Progressable: prog,
func digestsCatalogingProgress(locations int64) *monitor.CatalogerTaskProgress {
info := monitor.GenericTask{
Title: monitor.Title{
Default: "Catalog file digests",
WhileRunning: "Cataloging file digests",
OnSuccess: "Cataloged file digests",
},
})
}
return stage, prog
return bus.StartCatalogerTask(info, locations, "")
}
syft/file/cataloger/filemetadata/cataloger.go
+20 -21
View File
@@ -1,12 +1,13 @@
package filemetadata
import (
"github.com/wagoodman/go-partybus"
"github.com/wagoodman/go-progress"
"fmt"
"github.com/dustin/go-humanize"
"github.com/anchore/syft/internal/bus"
"github.com/anchore/syft/internal/log"
"github.com/anchore/syft/syft/event"
"github.com/anchore/syft/syft/event/monitor"
"github.com/anchore/syft/syft/file"
)
@@ -20,7 +21,6 @@ func NewCataloger() *Cataloger {
func (i *Cataloger) Catalog(resolver file.Resolver, coordinates ...file.Coordinates) (map[file.Coordinates]file.Metadata, error) {
results := make(map[file.Coordinates]file.Metadata)
var locations <-chan file.Location
if len(coordinates) == 0 {
locations = resolver.AllLocations()
} else {
@@ -43,36 +43,35 @@ func (i *Cataloger) Catalog(resolver file.Resolver, coordinates ...file.Coordina
}()
}
stage, prog := metadataCatalogingProgress(int64(len(locations)))
prog := metadataCatalogingProgress(int64(len(locations)))
for location := range locations {
stage.Current = location.RealPath
prog.Increment()
prog.AtomicStage.Set(location.Path())
metadata, err := resolver.FileMetadataByLocation(location)
if err != nil {
return nil, err
}
results[location.Coordinates] = metadata
prog.Increment()
}
log.Debugf("file metadata cataloger processed %d files", prog.Current())
prog.AtomicStage.Set(fmt.Sprintf("%s locations", humanize.Comma(prog.Current())))
prog.SetCompleted()
return results, nil
}
func metadataCatalogingProgress(locations int64) (*progress.Stage, *progress.Manual) {
stage := &progress.Stage{}
prog := progress.NewManual(locations)
bus.Publish(partybus.Event{
Type: event.FileMetadataCatalogerStarted,
Value: struct {
progress.Stager
progress.Progressable
}{
Stager: progress.Stager(stage),
Progressable: prog,
func metadataCatalogingProgress(locations int64) *monitor.CatalogerTaskProgress {
info := monitor.GenericTask{
Title: monitor.Title{
Default: "Catalog file metadata",
WhileRunning: "Cataloging file metadata",
OnSuccess: "Cataloged file metadata",
},
})
}
return stage, prog
return bus.StartCatalogerTask(info, locations, "")
}
syft/pkg/cataloger/catalog.go
+29 -30
View File
@@ -6,14 +6,14 @@ import (
"runtime/debug"
"sync"
"github.com/dustin/go-humanize"
"github.com/hashicorp/go-multierror"
"github.com/wagoodman/go-partybus"
"github.com/wagoodman/go-progress"
"github.com/anchore/syft/internal/bus"
"github.com/anchore/syft/internal/log"
"github.com/anchore/syft/syft/artifact"
"github.com/anchore/syft/syft/event"
"github.com/anchore/syft/syft/event/monitor"
"github.com/anchore/syft/syft/file"
"github.com/anchore/syft/syft/linux"
"github.com/anchore/syft/syft/pkg"
@@ -35,21 +35,6 @@ type catalogResult struct {
Error error
}
// newMonitor creates a new Monitor object and publishes the object on the bus as a PackageCatalogerStarted event.
func newMonitor() (*progress.Manual, *progress.Manual) {
filesProcessed := progress.Manual{}
packagesDiscovered := progress.Manual{}
bus.Publish(partybus.Event{
Type: event.PackageCatalogerStarted,
Value: Monitor{
FilesProcessed: progress.Monitorable(&filesProcessed),
PackagesDiscovered: progress.Monitorable(&packagesDiscovered),
},
})
return &filesProcessed, &packagesDiscovered
}
func runCataloger(cataloger pkg.Cataloger, resolver file.Resolver) (catalogerResult *catalogResult, err error) {
// handle individual cataloger panics
defer func() {
@@ -101,6 +86,7 @@ func runCataloger(cataloger pkg.Cataloger, resolver file.Resolver) (catalogerRes
}
catalogerResult.Packages = append(catalogerResult.Packages, p)
}
catalogerResult.Relationships = append(catalogerResult.Relationships, relationships...)
log.WithFields("cataloger", cataloger.Name()).Trace("cataloging complete")
return catalogerResult, err
@@ -116,9 +102,7 @@ func Catalog(resolver file.Resolver, _ *linux.Release, parallelism int, cataloge
catalog := pkg.NewCollection()
var allRelationships []artifact.Relationship
filesProcessed, packagesDiscovered := newMonitor()
defer filesProcessed.SetCompleted()
defer packagesDiscovered.SetCompleted()
prog := monitorPackageCatalogingTask()
// perform analysis, accumulating errors for each failed analysis
var errs error
@@ -131,10 +115,11 @@ func Catalog(resolver file.Resolver, _ *linux.Release, parallelism int, cataloge
jobs := make(chan pkg.Cataloger, nCatalogers)
results := make(chan *catalogResult, nCatalogers)
discoveredPackages := make(chan int64, nCatalogers)
waitGroup := sync.WaitGroup{}
var totalPackagesDiscovered int64
for i := 0; i < parallelism; i++ {
waitGroup.Add(1)
@@ -148,20 +133,16 @@ func Catalog(resolver file.Resolver, _ *linux.Release, parallelism int, cataloge
// ensure we set the error to be aggregated
result.Error = err
discoveredPackages <- result.Discovered
prog.Add(result.Discovered)
totalPackagesDiscovered += result.Discovered
count := humanize.Comma(totalPackagesDiscovered)
prog.AtomicStage.Set(fmt.Sprintf("%s packages", count))
results <- result
}
}()
}
// dynamically show updated discovered package status
go func() {
for discovered := range discoveredPackages {
packagesDiscovered.Add(discovered)
}
}()
// Enqueue the jobs
for _, cataloger := range catalogers {
jobs <- cataloger
@@ -171,7 +152,6 @@ func Catalog(resolver file.Resolver, _ *linux.Release, parallelism int, cataloge
// Wait for the jobs to finish
waitGroup.Wait()
close(results)
close(discoveredPackages)
// collect the results
for result := range results {
@@ -186,6 +166,12 @@ func Catalog(resolver file.Resolver, _ *linux.Release, parallelism int, cataloge
allRelationships = append(allRelationships, pkg.NewRelationships(catalog)...)
if errs != nil {
prog.SetError(errs)
} else {
prog.SetCompleted()
}
return catalog, allRelationships, errs
}
@@ -228,3 +214,16 @@ func packageFileOwnershipRelationships(p pkg.Package, resolver file.PathResolver
}
return relationships, nil
}
func monitorPackageCatalogingTask() *monitor.CatalogerTaskProgress {
info := monitor.GenericTask{
Title: monitor.Title{
Default: "Catalog packages",
WhileRunning: "Cataloging packages",
OnSuccess: "Cataloged packages",
},
HideOnSuccess: false,
}
return bus.StartCatalogerTask(info, -1, "")
}
syft/pkg/cataloger/golang/licenses.go
+18 -14
View File
@@ -20,6 +20,7 @@ import (
"github.com/go-git/go-git/v5/storage/memory"
"github.com/scylladb/go-set/strset"
"github.com/anchore/syft/internal/bus"
"github.com/anchore/syft/internal/licenses"
"github.com/anchore/syft/internal/log"
"github.com/anchore/syft/syft/event/monitor"
@@ -31,7 +32,6 @@ import (
type goLicenses struct {
opts CatalogerConfig
localModCacheResolver file.WritableResolver
progress *monitor.CatalogerTask
lowerLicenseFileNames *strset.Set
}
@@ -39,11 +39,6 @@ func newGoLicenses(opts CatalogerConfig) goLicenses {
return goLicenses{
opts: opts,
localModCacheResolver: modCacheResolver(opts.LocalModCacheDir),
progress: &monitor.CatalogerTask{
SubStatus: true,
RemoveOnCompletion: true,
Title: "Downloading go mod",
},
lowerLicenseFileNames: strset.New(lowercaseLicenseFiles()...),
}
}
@@ -123,7 +118,16 @@ func (c *goLicenses) getLicensesFromRemote(moduleName, moduleVersion string) ([]
proxies := remotesForModule(c.opts.Proxies, c.opts.NoProxy, moduleName)
fsys, err := getModule(c.progress, proxies, moduleName, moduleVersion)
prog := bus.StartCatalogerTask(monitor.GenericTask{
Title: monitor.Title{
Default: "Download go mod",
WhileRunning: "Downloading go mod",
OnSuccess: "Downloaded go mod",
},
HideOnSuccess: true,
}, -1, "")
fsys, err := getModule(prog, proxies, moduleName, moduleVersion)
if err != nil {
return nil, err
}
@@ -206,7 +210,7 @@ func processCaps(s string) string {
})
}
func getModule(progress *monitor.CatalogerTask, proxies []string, moduleName, moduleVersion string) (fsys fs.FS, err error) {
func getModule(progress *monitor.CatalogerTaskProgress, proxies []string, moduleName, moduleVersion string) (fsys fs.FS, err error) {
for _, proxy := range proxies {
u, _ := url.Parse(proxy)
if proxy == "direct" {
@@ -218,7 +222,7 @@ func getModule(progress *monitor.CatalogerTask, proxies []string, moduleName, mo
fsys, err = getModuleProxy(progress, proxy, moduleName, moduleVersion)
case "file":
p := filepath.Join(u.Path, moduleName, "@v", moduleVersion)
progress.SetValue(fmt.Sprintf("file: %s", p))
progress.AtomicStage.Set(fmt.Sprintf("file: %s", p))
fsys = os.DirFS(p)
}
if fsys != nil {
@@ -228,9 +232,9 @@ func getModule(progress *monitor.CatalogerTask, proxies []string, moduleName, mo
return
}
func getModuleProxy(progress *monitor.CatalogerTask, proxy string, moduleName string, moduleVersion string) (out fs.FS, _ error) {
func getModuleProxy(progress *monitor.CatalogerTaskProgress, proxy string, moduleName string, moduleVersion string) (out fs.FS, _ error) {
u := fmt.Sprintf("%s/%s/@v/%s.zip", proxy, moduleName, moduleVersion)
progress.SetValue(u)
progress.AtomicStage.Set(u)
// get the module zip
resp, err := http.Get(u) //nolint:gosec
@@ -241,7 +245,7 @@ func getModuleProxy(progress *monitor.CatalogerTask, proxy string, moduleName st
if resp.StatusCode != http.StatusOK {
u = fmt.Sprintf("%s/%s/@v/%s.zip", proxy, strings.ToLower(moduleName), moduleVersion)
progress.SetValue(u)
progress.AtomicStage.Set(u)
// try lowercasing it; some packages have mixed casing that really messes up the proxy
resp, err = http.Get(u) //nolint:gosec
@@ -284,14 +288,14 @@ func findVersionPath(f fs.FS, dir string) string {
return ""
}
func getModuleRepository(progress *monitor.CatalogerTask, moduleName string, moduleVersion string) (fs.FS, error) {
func getModuleRepository(progress *monitor.CatalogerTaskProgress, moduleName string, moduleVersion string) (fs.FS, error) {
repoName := moduleName
parts := strings.Split(moduleName, "/")
if len(parts) > 2 {
repoName = fmt.Sprintf("%s/%s/%s", parts[0], parts[1], parts[2])
}
progress.SetValue(fmt.Sprintf("git: %s", repoName))
progress.AtomicStage.Set(fmt.Sprintf("git: %s", repoName))
f := memfs.New()
buf := &bytes.Buffer{}