diff --git a/cmd/syft/cli/ui/__snapshots__/handle_cataloger_task_test.snap b/cmd/syft/cli/ui/__snapshots__/handle_cataloger_task_test.snap index aff1f474a..fc204c05a 100755 --- a/cmd/syft/cli/ui/__snapshots__/handle_cataloger_task_test.snap +++ b/cmd/syft/cli/ui/__snapshots__/handle_cataloger_task_test.snap @@ -1,16 +1,24 @@ [TestHandler_handleCatalogerTaskStarted/cataloging_task_in_progress - 1] - some task title [some value] + ⠙ Cataloging contents + ⠙ some task title ━━━━━━━━━━━━━━━━━━━━ [some stage] --- [TestHandler_handleCatalogerTaskStarted/cataloging_sub_task_in_progress - 1] - └── some task title [some value] + ⠙ Cataloging contents + └── ⠙ some task title ━━━━━━━━━━━━━━━━━━━━ [some stage] --- [TestHandler_handleCatalogerTaskStarted/cataloging_sub_task_complete - 1] - ✔ └── some task done [some value] + ⠙ Cataloging contents + └── ✔ some task done [some stage] +--- + +[TestHandler_handleCatalogerTaskStarted/cataloging_sub_task_complete_--_hide_stage - 1] + ⠙ Cataloging contents + └── ✔ some task done --- [TestHandler_handleCatalogerTaskStarted/cataloging_sub_task_complete_with_removal - 1] - + ⠙ Cataloging contents --- diff --git a/cmd/syft/cli/ui/handle_attestation_test.go b/cmd/syft/cli/ui/handle_attestation_test.go index c79d01d2c..76857ed65 100644 --- a/cmd/syft/cli/ui/handle_attestation_test.go +++ b/cmd/syft/cli/ui/handle_attestation_test.go @@ -100,18 +100,21 @@ func TestHandler_handleAttestationStarted(t *testing.T) { Height: 80, } - models := handler.Handle(event) + models, _ := handler.Handle(event) require.Len(t, models, 2) t.Run("task line", func(t *testing.T) { tsk, ok := models[0].(taskprogress.Model) require.True(t, ok) - got := runModel(t, tsk, tt.iterations, taskprogress.TickMsg{ + gotModel := runModel(t, tsk, tt.iterations, taskprogress.TickMsg{ Time: time.Now(), Sequence: tsk.Sequence(), ID: tsk.ID(), }) + + got := gotModel.View() + t.Log(got) snaps.MatchSnapshot(t, got) }) @@ -119,11 +122,15 @@ func TestHandler_handleAttestationStarted(t *testing.T) { t.Run("log", func(t *testing.T) { log, ok := models[1].(attestLogFrame) require.True(t, ok) - got := runModel(t, log, tt.iterations, attestLogFrameTickMsg{ + + gotModel := runModel(t, log, tt.iterations, attestLogFrameTickMsg{ Time: time.Now(), Sequence: log.sequence, ID: log.id, }, log.reader.running) + + got := gotModel.View() + t.Log(got) snaps.MatchSnapshot(t, got) }) diff --git a/cmd/syft/cli/ui/handle_cataloger_task.go b/cmd/syft/cli/ui/handle_cataloger_task.go index 393bd6e7a..0d4539ed9 100644 --- a/cmd/syft/cli/ui/handle_cataloger_task.go +++ b/cmd/syft/cli/ui/handle_cataloger_task.go @@ -3,70 +3,121 @@ package ui import ( tea "github.com/charmbracelet/bubbletea" "github.com/charmbracelet/lipgloss" + "github.com/google/uuid" "github.com/wagoodman/go-partybus" "github.com/wagoodman/go-progress" "github.com/anchore/bubbly/bubbles/taskprogress" + "github.com/anchore/bubbly/bubbles/tree" "github.com/anchore/syft/internal/log" "github.com/anchore/syft/syft/event/monitor" syftEventParsers "github.com/anchore/syft/syft/event/parsers" ) -var _ progress.Stager = (*catalogerTaskStageAdapter)(nil) +// we standardize how rows are instantiated to ensure consistency in the appearance across the UI +type taskModelFactory func(title taskprogress.Title, opts ...taskprogress.Option) taskprogress.Model -type catalogerTaskStageAdapter struct { - mon *monitor.CatalogerTask +var _ tea.Model = (*catalogerTaskModel)(nil) + +type catalogerTaskModel struct { + model tree.Model + modelFactory taskModelFactory } -func newCatalogerTaskStageAdapter(mon *monitor.CatalogerTask) *catalogerTaskStageAdapter { - return &catalogerTaskStageAdapter{ - mon: mon, +func newCatalogerTaskTreeModel(f taskModelFactory) *catalogerTaskModel { + t := tree.NewModel() + t.Padding = " " + t.RootsWithoutPrefix = true + return &catalogerTaskModel{ + modelFactory: f, + model: t, } } -func (c catalogerTaskStageAdapter) Stage() string { - return c.mon.GetValue() +type newCatalogerTaskRowEvent struct { + info monitor.GenericTask + prog progress.StagedProgressable } -func (m *Handler) handleCatalogerTaskStarted(e partybus.Event) []tea.Model { - mon, err := syftEventParsers.ParseCatalogerTaskStarted(e) - if err != nil { - log.WithFields("error", err).Warn("unable to parse event") - return nil +func (cts catalogerTaskModel) Init() tea.Cmd { + return cts.model.Init() +} + +func (cts catalogerTaskModel) Update(msg tea.Msg) (tea.Model, tea.Cmd) { + event, ok := msg.(newCatalogerTaskRowEvent) + if !ok { + model, cmd := cts.model.Update(msg) + cts.model = model.(tree.Model) + + return cts, cmd } - var prefix string - if mon.SubStatus { - // TODO: support list of sub-statuses, not just a single leaf - prefix = "└── " - } + info, prog := event.info, event.prog - tsk := m.newTaskProgress( + tsk := cts.modelFactory( taskprogress.Title{ - // TODO: prefix should not be part of the title, but instead a separate field that is aware of the tree structure - Default: prefix + mon.Title, - Running: prefix + mon.Title, - Success: prefix + mon.TitleOnCompletion, + Default: info.Title.Default, + Running: info.Title.WhileRunning, + Success: info.Title.OnSuccess, }, - taskprogress.WithStagedProgressable( - struct { - progress.Stager - progress.Progressable - }{ - Progressable: mon.GetMonitor(), - Stager: newCatalogerTaskStageAdapter(mon), - }, - ), + taskprogress.WithStagedProgressable(prog), ) - // TODO: this isn't ideal since the model stays around after it is no longer needed, but it works for now - tsk.HideOnSuccess = mon.RemoveOnCompletion - tsk.HideStageOnSuccess = false - tsk.HideProgressOnSuccess = false + if info.Context != "" { + tsk.Context = []string{info.Context} + } - tsk.TitleStyle = lipgloss.NewStyle() - // TODO: this is a hack to get the spinner to not show up, but ideally the component would support making the spinner optional - tsk.Spinner.Spinner.Frames = []string{" "} + tsk.HideOnSuccess = info.HideOnSuccess + tsk.HideStageOnSuccess = info.HideStageOnSuccess + tsk.HideProgressOnSuccess = true - return []tea.Model{tsk} + if info.ParentID != "" { + tsk.TitleStyle = lipgloss.NewStyle() + } + + if err := cts.model.Add(info.ParentID, info.ID, tsk); err != nil { + log.WithFields("error", err).Error("unable to add cataloger task to tree model") + } + + return cts, tsk.Init() +} + +func (cts catalogerTaskModel) View() string { + return cts.model.View() +} + +func (m *Handler) handleCatalogerTaskStarted(e partybus.Event) ([]tea.Model, tea.Cmd) { + mon, info, err := syftEventParsers.ParseCatalogerTaskStarted(e) + if err != nil { + log.WithFields("error", err).Warn("unable to parse event") + return nil, nil + } + + var models []tea.Model + + // only create the new cataloger task tree once to manage all cataloger task events + m.onNewCatalogerTask.Do(func() { + models = append(models, newCatalogerTaskTreeModel(m.newTaskProgress)) + }) + + // we need to update the cataloger task model with a new row. We should never update the model outside of the + // bubbletea update-render event loop. Instead, we return a command that will be executed by the bubbletea runtime, + // producing a message that is passed to the cataloger task model. This is the prescribed way to update models + // in bubbletea. + + if info.ID == "" { + // ID is optional from the consumer perspective, but required internally + info.ID = uuid.Must(uuid.NewRandom()).String() + } + + cmd := func() tea.Msg { + // this message will cause the cataloger task model to add a new row to the output based on the given task + // information and progress data. + return newCatalogerTaskRowEvent{ + info: *info, + prog: mon, + } + } + + return models, cmd } diff --git a/cmd/syft/cli/ui/handle_cataloger_task_test.go b/cmd/syft/cli/ui/handle_cataloger_task_test.go index 055694588..3efb0fdac 100644 --- a/cmd/syft/cli/ui/handle_cataloger_task_test.go +++ b/cmd/syft/cli/ui/handle_cataloger_task_test.go @@ -2,19 +2,22 @@ package ui import ( "testing" - "time" tea "github.com/charmbracelet/bubbletea" "github.com/gkampitakis/go-snaps/snaps" "github.com/stretchr/testify/require" "github.com/wagoodman/go-partybus" + "github.com/wagoodman/go-progress" - "github.com/anchore/bubbly/bubbles/taskprogress" syftEvent "github.com/anchore/syft/syft/event" "github.com/anchore/syft/syft/event/monitor" ) func TestHandler_handleCatalogerTaskStarted(t *testing.T) { + title := monitor.Title{ + Default: "some task title", + OnSuccess: "some task done", + } tests := []struct { name string eventFn func(*testing.T) partybus.Event @@ -23,99 +26,171 @@ func TestHandler_handleCatalogerTaskStarted(t *testing.T) { { name: "cataloging task in progress", eventFn: func(t *testing.T) partybus.Event { - src := &monitor.CatalogerTask{ - SubStatus: false, - RemoveOnCompletion: false, - Title: "some task title", - TitleOnCompletion: "some task done", + value := &monitor.CatalogerTaskProgress{ + AtomicStage: progress.NewAtomicStage("some stage"), + Manual: progress.NewManual(100), } - src.SetValue("some value") + value.Manual.Add(50) return partybus.Event{ - Type: syftEvent.CatalogerTaskStarted, - Source: src, + Type: syftEvent.CatalogerTaskStarted, + Source: monitor.GenericTask{ + Title: title, + HideOnSuccess: false, + HideStageOnSuccess: false, + ID: "my-id", + }, + Value: value, } }, }, { name: "cataloging sub task in progress", eventFn: func(t *testing.T) partybus.Event { - src := &monitor.CatalogerTask{ - SubStatus: true, - RemoveOnCompletion: false, - Title: "some task title", - TitleOnCompletion: "some task done", + value := &monitor.CatalogerTaskProgress{ + AtomicStage: progress.NewAtomicStage("some stage"), + Manual: progress.NewManual(100), } - src.SetValue("some value") + value.Manual.Add(50) return partybus.Event{ - Type: syftEvent.CatalogerTaskStarted, - Source: src, + Type: syftEvent.CatalogerTaskStarted, + Source: monitor.GenericTask{ + Title: title, + HideOnSuccess: false, + HideStageOnSuccess: false, + ID: "my-id", + ParentID: "top-level-task", + }, + Value: value, } }, }, { name: "cataloging sub task complete", eventFn: func(t *testing.T) partybus.Event { - src := &monitor.CatalogerTask{ - SubStatus: true, - RemoveOnCompletion: false, - Title: "some task title", - TitleOnCompletion: "some task done", + value := &monitor.CatalogerTaskProgress{ + AtomicStage: progress.NewAtomicStage("some stage"), + Manual: progress.NewManual(100), } - src.SetValue("some value") - src.SetCompleted() + value.SetCompleted() return partybus.Event{ - Type: syftEvent.CatalogerTaskStarted, - Source: src, + Type: syftEvent.CatalogerTaskStarted, + Source: monitor.GenericTask{ + Title: title, + HideOnSuccess: false, + HideStageOnSuccess: false, + ID: "my-id", + ParentID: "top-level-task", + }, + Value: value, + } + }, + }, + { + name: "cataloging sub task complete -- hide stage", + eventFn: func(t *testing.T) partybus.Event { + value := &monitor.CatalogerTaskProgress{ + AtomicStage: progress.NewAtomicStage("some stage"), + Manual: progress.NewManual(100), + } + + value.SetCompleted() + + return partybus.Event{ + Type: syftEvent.CatalogerTaskStarted, + Source: monitor.GenericTask{ + Title: title, + HideOnSuccess: false, + HideStageOnSuccess: true, + ID: "my-id", + ParentID: "top-level-task", + }, + Value: value, } }, }, { name: "cataloging sub task complete with removal", eventFn: func(t *testing.T) partybus.Event { - src := &monitor.CatalogerTask{ - SubStatus: true, - RemoveOnCompletion: true, - Title: "some task title", - TitleOnCompletion: "some task done", + value := &monitor.CatalogerTaskProgress{ + AtomicStage: progress.NewAtomicStage("some stage"), + Manual: progress.NewManual(100), } - src.SetValue("some value") - src.SetCompleted() + value.SetCompleted() return partybus.Event{ - Type: syftEvent.CatalogerTaskStarted, - Source: src, + Type: syftEvent.CatalogerTaskStarted, + Source: monitor.GenericTask{ + Title: title, + HideOnSuccess: true, + HideStageOnSuccess: false, + ID: "my-id", + ParentID: "top-level-task", + }, + Value: value, } }, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - event := tt.eventFn(t) + // need to be able to get the initial newCatalogerTaskRowEvent + initialize the nested taskprogress model + if tt.iterations == 0 { + tt.iterations = 2 + } + + e := tt.eventFn(t) handler := New(DefaultHandlerConfig()) handler.WindowSize = tea.WindowSizeMsg{ Width: 100, Height: 80, } - models := handler.Handle(event) + info := monitor.GenericTask{ + Title: monitor.Title{ + Default: "Catalog contents", + WhileRunning: "Cataloging contents", + OnSuccess: "Cataloged contents", + }, + ID: "top-level-task", + } + + // note: this line / event is not under test, only needed to show a sub status + kickoffEvent := &monitor.CatalogerTaskProgress{ + AtomicStage: progress.NewAtomicStage(""), + Manual: progress.NewManual(-1), + } + + models, cmd := handler.Handle( + partybus.Event{ + Type: syftEvent.CatalogerTaskStarted, + Source: info, + Value: progress.StagedProgressable(kickoffEvent), + }, + ) require.Len(t, models, 1) + require.NotNil(t, cmd) model := models[0] - tsk, ok := model.(taskprogress.Model) + tr, ok := model.(*catalogerTaskModel) require.True(t, ok) - got := runModel(t, tsk, tt.iterations, taskprogress.TickMsg{ - Time: time.Now(), - Sequence: tsk.Sequence(), - ID: tsk.ID(), - }) + gotModel := runModel(t, tr, tt.iterations, cmd()) + + models, cmd = handler.Handle(e) + require.Len(t, models, 0) + require.NotNil(t, cmd) + + gotModel = runModel(t, gotModel, tt.iterations, cmd()) + + got := gotModel.View() + t.Log(got) snaps.MatchSnapshot(t, got) }) diff --git a/cmd/syft/cli/ui/handle_fetch_image_test.go b/cmd/syft/cli/ui/handle_fetch_image_test.go index c514b9865..a2df4701c 100644 --- a/cmd/syft/cli/ui/handle_fetch_image_test.go +++ b/cmd/syft/cli/ui/handle_fetch_image_test.go @@ -80,18 +80,21 @@ func TestHandler_handleFetchImage(t *testing.T) { Height: 80, } - models := handler.Handle(event) + models, _ := handler.Handle(event) require.Len(t, models, 1) model := models[0] tsk, ok := model.(taskprogress.Model) require.True(t, ok) - got := runModel(t, tsk, tt.iterations, taskprogress.TickMsg{ + gotModel := runModel(t, tsk, tt.iterations, taskprogress.TickMsg{ Time: time.Now(), Sequence: tsk.Sequence(), ID: tsk.ID(), }) + + got := gotModel.View() + t.Log(got) snaps.MatchSnapshot(t, got) }) diff --git a/cmd/syft/cli/ui/handle_file_digests_cataloger.go b/cmd/syft/cli/ui/handle_file_digests_cataloger.go deleted file mode 100644 index 79550e9cf..000000000 --- a/cmd/syft/cli/ui/handle_file_digests_cataloger.go +++ /dev/null @@ -1,28 +0,0 @@ -package ui - -import ( - tea "github.com/charmbracelet/bubbletea" - "github.com/wagoodman/go-partybus" - - "github.com/anchore/bubbly/bubbles/taskprogress" - "github.com/anchore/syft/internal/log" - syftEventParsers "github.com/anchore/syft/syft/event/parsers" -) - -func (m *Handler) handleFileDigestsCatalogerStarted(e partybus.Event) []tea.Model { - prog, err := syftEventParsers.ParseFileDigestsCatalogingStarted(e) - if err != nil { - log.WithFields("error", err).Warn("unable to parse event") - return nil - } - - tsk := m.newTaskProgress( - taskprogress.Title{ - Default: "Catalog file digests", - Running: "Cataloging file digests", - Success: "Cataloged file digests", - }, taskprogress.WithStagedProgressable(prog), - ) - - return []tea.Model{tsk} -} diff --git a/cmd/syft/cli/ui/handle_file_digests_cataloger_test.go b/cmd/syft/cli/ui/handle_file_digests_cataloger_test.go deleted file mode 100644 index 2e74009a4..000000000 --- a/cmd/syft/cli/ui/handle_file_digests_cataloger_test.go +++ /dev/null @@ -1,97 +0,0 @@ -package ui - -import ( - "testing" - "time" - - tea "github.com/charmbracelet/bubbletea" - "github.com/gkampitakis/go-snaps/snaps" - "github.com/stretchr/testify/require" - "github.com/wagoodman/go-partybus" - "github.com/wagoodman/go-progress" - - "github.com/anchore/bubbly/bubbles/taskprogress" - syftEvent "github.com/anchore/syft/syft/event" -) - -func TestHandler_handleFileDigestsCatalogerStarted(t *testing.T) { - - tests := []struct { - name string - eventFn func(*testing.T) partybus.Event - iterations int - }{ - { - name: "cataloging in progress", - eventFn: func(t *testing.T) partybus.Event { - prog := &progress.Manual{} - prog.SetTotal(100) - prog.Set(50) - - mon := struct { - progress.Progressable - progress.Stager - }{ - Progressable: prog, - Stager: &progress.Stage{ - Current: "current", - }, - } - - return partybus.Event{ - Type: syftEvent.FileDigestsCatalogerStarted, - Value: mon, - } - }, - }, - { - name: "cataloging complete", - eventFn: func(t *testing.T) partybus.Event { - prog := &progress.Manual{} - prog.SetTotal(100) - prog.Set(100) - prog.SetCompleted() - - mon := struct { - progress.Progressable - progress.Stager - }{ - Progressable: prog, - Stager: &progress.Stage{ - Current: "current", - }, - } - - return partybus.Event{ - Type: syftEvent.FileDigestsCatalogerStarted, - Value: mon, - } - }, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - event := tt.eventFn(t) - handler := New(DefaultHandlerConfig()) - handler.WindowSize = tea.WindowSizeMsg{ - Width: 100, - Height: 80, - } - - models := handler.Handle(event) - require.Len(t, models, 1) - model := models[0] - - tsk, ok := model.(taskprogress.Model) - require.True(t, ok) - - got := runModel(t, tsk, tt.iterations, taskprogress.TickMsg{ - Time: time.Now(), - Sequence: tsk.Sequence(), - ID: tsk.ID(), - }) - t.Log(got) - snaps.MatchSnapshot(t, got) - }) - } -} diff --git a/cmd/syft/cli/ui/handle_file_indexing_test.go b/cmd/syft/cli/ui/handle_file_indexing_test.go index 86473c411..65615119d 100644 --- a/cmd/syft/cli/ui/handle_file_indexing_test.go +++ b/cmd/syft/cli/ui/handle_file_indexing_test.go @@ -80,18 +80,21 @@ func TestHandler_handleFileIndexingStarted(t *testing.T) { Height: 80, } - models := handler.Handle(event) + models, _ := handler.Handle(event) require.Len(t, models, 1) model := models[0] tsk, ok := model.(taskprogress.Model) require.True(t, ok) - got := runModel(t, tsk, tt.iterations, taskprogress.TickMsg{ + gotModel := runModel(t, tsk, tt.iterations, taskprogress.TickMsg{ Time: time.Now(), Sequence: tsk.Sequence(), ID: tsk.ID(), }) + + got := gotModel.View() + t.Log(got) snaps.MatchSnapshot(t, got) }) diff --git a/cmd/syft/cli/ui/handle_file_metadata_cataloger.go b/cmd/syft/cli/ui/handle_file_metadata_cataloger.go deleted file mode 100644 index 58535abc1..000000000 --- a/cmd/syft/cli/ui/handle_file_metadata_cataloger.go +++ /dev/null @@ -1,29 +0,0 @@ -package ui - -import ( - tea "github.com/charmbracelet/bubbletea" - "github.com/wagoodman/go-partybus" - - "github.com/anchore/bubbly/bubbles/taskprogress" - "github.com/anchore/syft/internal/log" - syftEventParsers "github.com/anchore/syft/syft/event/parsers" -) - -func (m *Handler) handleFileMetadataCatalogerStarted(e partybus.Event) []tea.Model { - prog, err := syftEventParsers.ParseFileMetadataCatalogingStarted(e) - if err != nil { - log.WithFields("error", err).Warn("unable to parse event") - return nil - } - - tsk := m.newTaskProgress( - taskprogress.Title{ - Default: "Catalog file metadata", - Running: "Cataloging file metadata", - Success: "Cataloged file metadata", - }, - taskprogress.WithStagedProgressable(prog), - ) - - return []tea.Model{tsk} -} diff --git a/cmd/syft/cli/ui/handle_file_metadata_cataloger_test.go b/cmd/syft/cli/ui/handle_file_metadata_cataloger_test.go deleted file mode 100644 index d247001c8..000000000 --- a/cmd/syft/cli/ui/handle_file_metadata_cataloger_test.go +++ /dev/null @@ -1,97 +0,0 @@ -package ui - -import ( - "testing" - "time" - - tea "github.com/charmbracelet/bubbletea" - "github.com/gkampitakis/go-snaps/snaps" - "github.com/stretchr/testify/require" - "github.com/wagoodman/go-partybus" - "github.com/wagoodman/go-progress" - - "github.com/anchore/bubbly/bubbles/taskprogress" - syftEvent "github.com/anchore/syft/syft/event" -) - -func TestHandler_handleFileMetadataCatalogerStarted(t *testing.T) { - - tests := []struct { - name string - eventFn func(*testing.T) partybus.Event - iterations int - }{ - { - name: "cataloging in progress", - eventFn: func(t *testing.T) partybus.Event { - prog := &progress.Manual{} - prog.SetTotal(100) - prog.Set(50) - - mon := struct { - progress.Progressable - progress.Stager - }{ - Progressable: prog, - Stager: &progress.Stage{ - Current: "current", - }, - } - - return partybus.Event{ - Type: syftEvent.FileMetadataCatalogerStarted, - Value: mon, - } - }, - }, - { - name: "cataloging complete", - eventFn: func(t *testing.T) partybus.Event { - prog := &progress.Manual{} - prog.SetTotal(100) - prog.Set(100) - prog.SetCompleted() - - mon := struct { - progress.Progressable - progress.Stager - }{ - Progressable: prog, - Stager: &progress.Stage{ - Current: "current", - }, - } - - return partybus.Event{ - Type: syftEvent.FileMetadataCatalogerStarted, - Value: mon, - } - }, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - event := tt.eventFn(t) - handler := New(DefaultHandlerConfig()) - handler.WindowSize = tea.WindowSizeMsg{ - Width: 100, - Height: 80, - } - - models := handler.Handle(event) - require.Len(t, models, 1) - model := models[0] - - tsk, ok := model.(taskprogress.Model) - require.True(t, ok) - - got := runModel(t, tsk, tt.iterations, taskprogress.TickMsg{ - Time: time.Now(), - Sequence: tsk.Sequence(), - ID: tsk.ID(), - }) - t.Log(got) - snaps.MatchSnapshot(t, got) - }) - } -} diff --git a/cmd/syft/cli/ui/handle_package_cataloger.go b/cmd/syft/cli/ui/handle_package_cataloger.go deleted file mode 100644 index 3aa2f9330..000000000 --- a/cmd/syft/cli/ui/handle_package_cataloger.go +++ /dev/null @@ -1,87 +0,0 @@ -package ui - -import ( - "fmt" - - tea "github.com/charmbracelet/bubbletea" - "github.com/wagoodman/go-partybus" - "github.com/wagoodman/go-progress" - - "github.com/anchore/bubbly/bubbles/taskprogress" - "github.com/anchore/syft/internal/log" - syftEventParsers "github.com/anchore/syft/syft/event/parsers" - "github.com/anchore/syft/syft/pkg/cataloger" -) - -var _ progress.StagedProgressable = (*packageCatalogerProgressAdapter)(nil) - -type packageCatalogerProgressAdapter struct { - monitor *cataloger.Monitor - monitors []progress.Monitorable -} - -func newPackageCatalogerProgressAdapter(monitor *cataloger.Monitor) packageCatalogerProgressAdapter { - return packageCatalogerProgressAdapter{ - monitor: monitor, - monitors: []progress.Monitorable{ - monitor.FilesProcessed, - monitor.PackagesDiscovered, - }, - } -} - -func (p packageCatalogerProgressAdapter) Stage() string { - return fmt.Sprintf("%d packages", p.monitor.PackagesDiscovered.Current()) -} - -func (p packageCatalogerProgressAdapter) Current() int64 { - return p.monitor.PackagesDiscovered.Current() -} - -func (p packageCatalogerProgressAdapter) Error() error { - completedMonitors := 0 - for _, monitor := range p.monitors { - err := monitor.Error() - if err == nil { - continue - } - if progress.IsErrCompleted(err) { - completedMonitors++ - continue - } - // something went wrong - return err - } - if completedMonitors == len(p.monitors) && len(p.monitors) > 0 { - return p.monitors[0].Error() - } - return nil -} - -func (p packageCatalogerProgressAdapter) Size() int64 { - // this is an inherently unknown value (indeterminate total number of packages to discover) - return -1 -} - -func (m *Handler) handlePackageCatalogerStarted(e partybus.Event) []tea.Model { - monitor, err := syftEventParsers.ParsePackageCatalogerStarted(e) - if err != nil { - log.WithFields("error", err).Warn("unable to parse event") - return nil - } - - tsk := m.newTaskProgress( - taskprogress.Title{ - Default: "Catalog packages", - Running: "Cataloging packages", - Success: "Cataloged packages", - }, - taskprogress.WithStagedProgressable( - newPackageCatalogerProgressAdapter(monitor), - ), - ) - - tsk.HideStageOnSuccess = false - - return []tea.Model{tsk} -} diff --git a/cmd/syft/cli/ui/handle_package_cataloger_test.go b/cmd/syft/cli/ui/handle_package_cataloger_test.go deleted file mode 100644 index a5a72d59d..000000000 --- a/cmd/syft/cli/ui/handle_package_cataloger_test.go +++ /dev/null @@ -1,133 +0,0 @@ -package ui - -import ( - "testing" - "time" - - tea "github.com/charmbracelet/bubbletea" - "github.com/gkampitakis/go-snaps/snaps" - "github.com/stretchr/testify/require" - "github.com/wagoodman/go-partybus" - "github.com/wagoodman/go-progress" - - "github.com/anchore/bubbly/bubbles/taskprogress" - syftEvent "github.com/anchore/syft/syft/event" - "github.com/anchore/syft/syft/pkg/cataloger" -) - -func TestHandler_handlePackageCatalogerStarted(t *testing.T) { - - tests := []struct { - name string - eventFn func(*testing.T) partybus.Event - iterations int - }{ - { - name: "cataloging in progress", - eventFn: func(t *testing.T) partybus.Event { - prog := &progress.Manual{} - prog.SetTotal(100) - prog.Set(50) - - mon := cataloger.Monitor{ - FilesProcessed: progress.NewManual(-1), - PackagesDiscovered: prog, - } - - return partybus.Event{ - Type: syftEvent.PackageCatalogerStarted, - Value: mon, - } - }, - }, - { - name: "cataloging only files complete", - eventFn: func(t *testing.T) partybus.Event { - prog := &progress.Manual{} - prog.SetTotal(100) - prog.Set(50) - - files := progress.NewManual(-1) - files.SetCompleted() - - mon := cataloger.Monitor{ - FilesProcessed: files, - PackagesDiscovered: prog, - } - - return partybus.Event{ - Type: syftEvent.PackageCatalogerStarted, - Value: mon, - } - }, - }, - { - name: "cataloging only packages complete", - eventFn: func(t *testing.T) partybus.Event { - prog := &progress.Manual{} - prog.SetTotal(100) - prog.Set(100) - prog.SetCompleted() - - files := progress.NewManual(-1) - - mon := cataloger.Monitor{ - FilesProcessed: files, - PackagesDiscovered: prog, - } - - return partybus.Event{ - Type: syftEvent.PackageCatalogerStarted, - Value: mon, - } - }, - }, - { - name: "cataloging complete", - eventFn: func(t *testing.T) partybus.Event { - prog := &progress.Manual{} - prog.SetTotal(100) - prog.Set(100) - prog.SetCompleted() - - files := progress.NewManual(-1) - files.SetCompleted() - - mon := cataloger.Monitor{ - FilesProcessed: files, - PackagesDiscovered: prog, - } - - return partybus.Event{ - Type: syftEvent.PackageCatalogerStarted, - Value: mon, - } - }, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - event := tt.eventFn(t) - handler := New(DefaultHandlerConfig()) - handler.WindowSize = tea.WindowSizeMsg{ - Width: 100, - Height: 80, - } - - models := handler.Handle(event) - require.Len(t, models, 1) - model := models[0] - - tsk, ok := model.(taskprogress.Model) - require.True(t, ok) - - got := runModel(t, tsk, tt.iterations, taskprogress.TickMsg{ - Time: time.Now(), - Sequence: tsk.Sequence(), - ID: tsk.ID(), - }) - t.Log(got) - snaps.MatchSnapshot(t, got) - }) - } -} diff --git a/cmd/syft/cli/ui/handle_read_image_test.go b/cmd/syft/cli/ui/handle_read_image_test.go index 864d1e782..d3354c04e 100644 --- a/cmd/syft/cli/ui/handle_read_image_test.go +++ b/cmd/syft/cli/ui/handle_read_image_test.go @@ -98,18 +98,21 @@ func TestHandler_handleReadImage(t *testing.T) { Height: 80, } - models := handler.Handle(event) + models, _ := handler.Handle(event) require.Len(t, models, 1) model := models[0] tsk, ok := model.(taskprogress.Model) require.True(t, ok) - got := runModel(t, tsk, tt.iterations, taskprogress.TickMsg{ + gotModel := runModel(t, tsk, tt.iterations, taskprogress.TickMsg{ Time: time.Now(), Sequence: tsk.Sequence(), ID: tsk.ID(), }) + + got := gotModel.View() + t.Log(got) snaps.MatchSnapshot(t, got) }) diff --git a/cmd/syft/cli/ui/handler.go b/cmd/syft/cli/ui/handler.go index 1a0e64d10..aefc471f3 100644 --- a/cmd/syft/cli/ui/handler.go +++ b/cmd/syft/cli/ui/handler.go @@ -29,6 +29,8 @@ type Handler struct { Config HandlerConfig bubbly.EventHandler + + onNewCatalogerTask *sync.Once } func DefaultHandlerConfig() HandlerConfig { @@ -41,28 +43,32 @@ func New(cfg HandlerConfig) *Handler { d := bubbly.NewEventDispatcher() h := &Handler{ - EventHandler: d, - Running: &sync.WaitGroup{}, - Config: cfg, + EventHandler: d, + Running: &sync.WaitGroup{}, + Config: cfg, + onNewCatalogerTask: &sync.Once{}, } // register all supported event types with the respective handler functions d.AddHandlers(map[partybus.EventType]bubbly.EventHandlerFn{ - stereoscopeEvent.PullDockerImage: h.handlePullDockerImage, - stereoscopeEvent.PullContainerdImage: h.handlePullContainerdImage, - stereoscopeEvent.ReadImage: h.handleReadImage, - stereoscopeEvent.FetchImage: h.handleFetchImage, - syftEvent.PackageCatalogerStarted: h.handlePackageCatalogerStarted, - syftEvent.FileDigestsCatalogerStarted: h.handleFileDigestsCatalogerStarted, - syftEvent.FileMetadataCatalogerStarted: h.handleFileMetadataCatalogerStarted, - syftEvent.FileIndexingStarted: h.handleFileIndexingStarted, - syftEvent.AttestationStarted: h.handleAttestationStarted, - syftEvent.CatalogerTaskStarted: h.handleCatalogerTaskStarted, + stereoscopeEvent.PullDockerImage: simpleHandler(h.handlePullDockerImage), + stereoscopeEvent.PullContainerdImage: simpleHandler(h.handlePullContainerdImage), + stereoscopeEvent.ReadImage: simpleHandler(h.handleReadImage), + stereoscopeEvent.FetchImage: simpleHandler(h.handleFetchImage), + syftEvent.FileIndexingStarted: simpleHandler(h.handleFileIndexingStarted), + syftEvent.AttestationStarted: simpleHandler(h.handleAttestationStarted), + syftEvent.CatalogerTaskStarted: h.handleCatalogerTaskStarted, }) return h } +func simpleHandler(fn func(partybus.Event) []tea.Model) bubbly.EventHandlerFn { + return func(e partybus.Event) ([]tea.Model, tea.Cmd) { + return fn(e), nil + } +} + func (m *Handler) OnMessage(msg tea.Msg) { if msg, ok := msg.(tea.WindowSizeMsg); ok { m.WindowSize = msg diff --git a/cmd/syft/cli/ui/util_test.go b/cmd/syft/cli/ui/util_test.go index 71cc8809a..c13aadae0 100644 --- a/cmd/syft/cli/ui/util_test.go +++ b/cmd/syft/cli/ui/util_test.go @@ -4,12 +4,11 @@ import ( "reflect" "sync" "testing" - "unsafe" tea "github.com/charmbracelet/bubbletea" ) -func runModel(t testing.TB, m tea.Model, iterations int, message tea.Msg, h ...*sync.WaitGroup) string { +func runModel(t testing.TB, m tea.Model, iterations int, message tea.Msg, h ...*sync.WaitGroup) tea.Model { t.Helper() if iterations == 0 { iterations = 1 @@ -37,34 +36,21 @@ func runModel(t testing.TB, m tea.Model, iterations int, message tea.Msg, h ...* cmd = tea.Batch(nextCmds...) } - return m.View() + return m } -func flatten(p tea.Msg) (msgs []tea.Msg) { - if reflect.TypeOf(p).Name() == "batchMsg" { - partials := extractBatchMessages(p) - for _, m := range partials { - msgs = append(msgs, flatten(m)...) +func flatten(ps ...tea.Msg) (msgs []tea.Msg) { + for _, p := range ps { + if bm, ok := p.(tea.BatchMsg); ok { + for _, m := range bm { + if m == nil { + continue + } + msgs = append(msgs, flatten(m())...) + } + } else { + msgs = []tea.Msg{p} } - } else { - msgs = []tea.Msg{p} } return msgs } - -func extractBatchMessages(m tea.Msg) (ret []tea.Msg) { - sliceMsgType := reflect.SliceOf(reflect.TypeOf(tea.Cmd(nil))) - value := reflect.ValueOf(m) // note: this is technically unaddressable - - // make our own instance that is addressable - valueCopy := reflect.New(value.Type()).Elem() - valueCopy.Set(value) - - cmds := reflect.NewAt(sliceMsgType, unsafe.Pointer(valueCopy.UnsafeAddr())).Elem() - for i := 0; i < cmds.Len(); i++ { - item := cmds.Index(i) - r := item.Call(nil) - ret = append(ret, r[0].Interface().(tea.Msg)) - } - return ret -} diff --git a/cmd/syft/internal/ui/ui.go b/cmd/syft/internal/ui/ui.go index 6161294a5..85fb137e9 100644 --- a/cmd/syft/internal/ui/ui.go +++ b/cmd/syft/internal/ui/ui.go @@ -155,7 +155,11 @@ func (m *UI) Update(msg tea.Msg) (tea.Model, tea.Cmd) { return m, nil } - for _, newModel := range m.handler.Handle(msg) { + models, cmd := m.handler.Handle(msg) + if cmd != nil { + cmds = append(cmds, cmd) + } + for _, newModel := range models { if newModel == nil { continue } diff --git a/go.mod b/go.mod index d212ab1c9..e136fc43c 100644 --- a/go.mod +++ b/go.mod @@ -8,7 +8,7 @@ require ( github.com/Masterminds/sprig/v3 v3.2.3 github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d github.com/acobaugh/osrelease v0.1.0 - github.com/anchore/bubbly v0.0.0-20230801194016-acdb4981b461 + github.com/anchore/bubbly v0.0.0-20231115134915-def0aba654a9 github.com/anchore/clio v0.0.0-20231016125544-c98a83e1c7fc github.com/anchore/fangs v0.0.0-20231103141714-84c94dc43a2e github.com/anchore/go-logger v0.0.0-20230725134548-c21dafa1ec5a diff --git a/go.sum b/go.sum index 93efec363..1fa094f7c 100644 --- a/go.sum +++ b/go.sum @@ -89,8 +89,8 @@ github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuy github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= -github.com/anchore/bubbly v0.0.0-20230801194016-acdb4981b461 h1:xGu4/uMWucwWV0YV3fpFIQZ6KVfS/Wfhmma8t0s0vRo= -github.com/anchore/bubbly v0.0.0-20230801194016-acdb4981b461/go.mod h1:Ger02eh5NpPm2IqkPAy396HU1KlK3BhOeCljDYXySSk= +github.com/anchore/bubbly v0.0.0-20231115134915-def0aba654a9 h1:p0ZIe0htYOX284Y4axJaGBvXHU0VCCzLN5Wf5XbKStU= +github.com/anchore/bubbly v0.0.0-20231115134915-def0aba654a9/go.mod h1:3ZsFB9tzW3vl4gEiUeuSOMDnwroWxIxJelOOHUp8dSw= github.com/anchore/clio v0.0.0-20231016125544-c98a83e1c7fc h1:A1KFO+zZZmbNlz1+WKsCF0RKVx6XRoxsAG3lrqH9hUQ= github.com/anchore/clio v0.0.0-20231016125544-c98a83e1c7fc/go.mod h1:QeWvNzxsrUNxcs6haQo3OtISfXUXW0qAuiG4EQiz0GU= github.com/anchore/fangs v0.0.0-20231103141714-84c94dc43a2e h1:O8ZubApaSl7dRzKNvyfGq9cLIPLQ5v3Iz0Y3huHKCgg= diff --git a/internal/bus/helpers.go b/internal/bus/helpers.go index b2c883ebd..363d9754d 100644 --- a/internal/bus/helpers.go +++ b/internal/bus/helpers.go @@ -2,10 +2,12 @@ package bus import ( "github.com/wagoodman/go-partybus" + "github.com/wagoodman/go-progress" "github.com/anchore/clio" "github.com/anchore/syft/internal/redact" "github.com/anchore/syft/syft/event" + "github.com/anchore/syft/syft/event/monitor" ) func Exit() { @@ -33,3 +35,18 @@ func Notify(message string) { Value: message, }) } + +func StartCatalogerTask(info monitor.GenericTask, size int64, initialStage string) *monitor.CatalogerTaskProgress { + t := &monitor.CatalogerTaskProgress{ + AtomicStage: progress.NewAtomicStage(initialStage), + Manual: progress.NewManual(size), + } + + Publish(partybus.Event{ + Type: event.CatalogerTaskStarted, + Source: info, + Value: progress.StagedProgressable(t), + }) + + return t +} diff --git a/syft/event/event.go b/syft/event/event.go index b18ff9dec..6c760af66 100644 --- a/syft/event/event.go +++ b/syft/event/event.go @@ -14,15 +14,6 @@ const ( // Events from the syft library - // PackageCatalogerStarted is a partybus event that occurs when the package cataloging has begun - PackageCatalogerStarted partybus.EventType = typePrefix + "-package-cataloger-started-event" - - // FileMetadataCatalogerStarted is a partybus event that occurs when the file metadata cataloging has begun - FileMetadataCatalogerStarted partybus.EventType = typePrefix + "-file-metadata-cataloger-started-event" - - // FileDigestsCatalogerStarted is a partybus event that occurs when the file digests cataloging has begun - FileDigestsCatalogerStarted partybus.EventType = typePrefix + "-file-digests-cataloger-started-event" - // FileIndexingStarted is a partybus event that occurs when the directory resolver begins indexing a filesystem FileIndexingStarted partybus.EventType = typePrefix + "-file-indexing-started-event" diff --git a/syft/event/monitor/cataloger_task.go b/syft/event/monitor/cataloger_task.go deleted file mode 100644 index 4a06132e7..000000000 --- a/syft/event/monitor/cataloger_task.go +++ /dev/null @@ -1,55 +0,0 @@ -package monitor - -import ( - "github.com/wagoodman/go-partybus" - "github.com/wagoodman/go-progress" - - "github.com/anchore/syft/internal/bus" - "github.com/anchore/syft/syft/event" -) - -// TODO: this should be refactored to support read-only/write-only access using idioms of the progress lib - -type CatalogerTask struct { - prog *progress.Manual - // Title - Title string - // TitleOnCompletion a string to use as title when completed - TitleOnCompletion string - // SubStatus indicates this progress should be rendered as a sub-item - SubStatus bool - // RemoveOnCompletion indicates this progress line will be removed when completed - RemoveOnCompletion bool - // value is the value to display -- not public as SetValue needs to be called to initialize this progress - value string -} - -func (e *CatalogerTask) init() { - e.prog = progress.NewManual(-1) - - bus.Publish(partybus.Event{ - Type: event.CatalogerTaskStarted, - Source: e, - }) -} - -func (e *CatalogerTask) SetCompleted() { - if e.prog != nil { - e.prog.SetCompleted() - } -} - -func (e *CatalogerTask) SetValue(value string) { - if e.prog == nil { - e.init() - } - e.value = value -} - -func (e *CatalogerTask) GetValue() string { - return e.value -} - -func (e *CatalogerTask) GetMonitor() *progress.Manual { - return e.prog -} diff --git a/syft/event/monitor/cataloger_task_progress.go b/syft/event/monitor/cataloger_task_progress.go new file mode 100644 index 000000000..6f459c8e2 --- /dev/null +++ b/syft/event/monitor/cataloger_task_progress.go @@ -0,0 +1,10 @@ +package monitor + +import ( + "github.com/wagoodman/go-progress" +) + +type CatalogerTaskProgress struct { + *progress.AtomicStage + *progress.Manual +} diff --git a/syft/event/monitor/generic_task.go b/syft/event/monitor/generic_task.go index cf5a6ea6d..b4ed48d2d 100644 --- a/syft/event/monitor/generic_task.go +++ b/syft/event/monitor/generic_task.go @@ -18,6 +18,19 @@ type Title struct { } type GenericTask struct { - Title Title - Context string + + // required fields + + Title Title + + // optional format fields + + HideOnSuccess bool + HideStageOnSuccess bool + + // optional fields + + ID string + ParentID string + Context string } diff --git a/syft/event/parsers/parsers.go b/syft/event/parsers/parsers.go index 5f7f2d7c8..52a11d309 100644 --- a/syft/event/parsers/parsers.go +++ b/syft/event/parsers/parsers.go @@ -12,7 +12,6 @@ import ( "github.com/anchore/syft/syft/event" "github.com/anchore/syft/syft/event/monitor" - "github.com/anchore/syft/syft/pkg/cataloger" ) type ErrBadPayload struct { @@ -40,45 +39,6 @@ func checkEventType(actual, expected partybus.EventType) error { return nil } -func ParsePackageCatalogerStarted(e partybus.Event) (*cataloger.Monitor, error) { - if err := checkEventType(e.Type, event.PackageCatalogerStarted); err != nil { - return nil, err - } - - monitor, ok := e.Value.(cataloger.Monitor) - if !ok { - return nil, newPayloadErr(e.Type, "Value", e.Value) - } - - return &monitor, nil -} - -func ParseFileMetadataCatalogingStarted(e partybus.Event) (progress.StagedProgressable, error) { - if err := checkEventType(e.Type, event.FileMetadataCatalogerStarted); err != nil { - return nil, err - } - - prog, ok := e.Value.(progress.StagedProgressable) - if !ok { - return nil, newPayloadErr(e.Type, "Value", e.Value) - } - - return prog, nil -} - -func ParseFileDigestsCatalogingStarted(e partybus.Event) (progress.StagedProgressable, error) { - if err := checkEventType(e.Type, event.FileDigestsCatalogerStarted); err != nil { - return nil, err - } - - prog, ok := e.Value.(progress.StagedProgressable) - if !ok { - return nil, newPayloadErr(e.Type, "Value", e.Value) - } - - return prog, nil -} - func ParseFileIndexingStarted(e partybus.Event) (string, progress.StagedProgressable, error) { if err := checkEventType(e.Type, event.FileIndexingStarted); err != nil { return "", nil, err @@ -97,17 +57,24 @@ func ParseFileIndexingStarted(e partybus.Event) (string, progress.StagedProgress return path, prog, nil } -func ParseCatalogerTaskStarted(e partybus.Event) (*monitor.CatalogerTask, error) { +func ParseCatalogerTaskStarted(e partybus.Event) (progress.StagedProgressable, *monitor.GenericTask, error) { if err := checkEventType(e.Type, event.CatalogerTaskStarted); err != nil { - return nil, err + return nil, nil, err } - source, ok := e.Source.(*monitor.CatalogerTask) + var mon progress.StagedProgressable + + source, ok := e.Source.(monitor.GenericTask) if !ok { - return nil, newPayloadErr(e.Type, "Source", e.Source) + return nil, nil, newPayloadErr(e.Type, "Source", e.Source) } - return source, nil + mon, ok = e.Value.(progress.StagedProgressable) + if !ok { + mon = nil + } + + return mon, &source, nil } func ParseAttestationStartedEvent(e partybus.Event) (io.Reader, progress.Progressable, *monitor.GenericTask, error) { diff --git a/syft/file/cataloger/filedigest/cataloger.go b/syft/file/cataloger/filedigest/cataloger.go index 64dbdc05e..91f719c01 100644 --- a/syft/file/cataloger/filedigest/cataloger.go +++ b/syft/file/cataloger/filedigest/cataloger.go @@ -3,16 +3,16 @@ package filedigest import ( "crypto" "errors" + "fmt" - "github.com/wagoodman/go-partybus" - "github.com/wagoodman/go-progress" + "github.com/dustin/go-humanize" stereoscopeFile "github.com/anchore/stereoscope/pkg/file" "github.com/anchore/syft/internal" "github.com/anchore/syft/internal/bus" intFile "github.com/anchore/syft/internal/file" "github.com/anchore/syft/internal/log" - "github.com/anchore/syft/syft/event" + "github.com/anchore/syft/syft/event/monitor" "github.com/anchore/syft/syft/file" intCataloger "github.com/anchore/syft/syft/file/cataloger/internal" ) @@ -41,9 +41,11 @@ func (i *Cataloger) Catalog(resolver file.Resolver, coordinates ...file.Coordina } } - stage, prog := digestsCatalogingProgress(int64(len(locations))) + prog := digestsCatalogingProgress(int64(len(locations))) for _, location := range locations { - stage.Current = location.RealPath + prog.Increment() + prog.AtomicStage.Set(location.Path()) + result, err := i.catalogLocation(resolver, location) if errors.Is(err, ErrUndigestableFile) { @@ -61,8 +63,12 @@ func (i *Cataloger) Catalog(resolver file.Resolver, coordinates ...file.Coordina prog.Increment() results[location.Coordinates] = result } + log.Debugf("file digests cataloger processed %d files", prog.Current()) + + prog.AtomicStage.Set(fmt.Sprintf("%s digests", humanize.Comma(prog.Current()))) prog.SetCompleted() + return results, nil } @@ -91,20 +97,14 @@ func (i *Cataloger) catalogLocation(resolver file.Resolver, location file.Locati return digests, nil } -func digestsCatalogingProgress(locations int64) (*progress.Stage, *progress.Manual) { - stage := &progress.Stage{} - prog := progress.NewManual(locations) - - bus.Publish(partybus.Event{ - Type: event.FileDigestsCatalogerStarted, - Value: struct { - progress.Stager - progress.Progressable - }{ - Stager: progress.Stager(stage), - Progressable: prog, +func digestsCatalogingProgress(locations int64) *monitor.CatalogerTaskProgress { + info := monitor.GenericTask{ + Title: monitor.Title{ + Default: "Catalog file digests", + WhileRunning: "Cataloging file digests", + OnSuccess: "Cataloged file digests", }, - }) + } - return stage, prog + return bus.StartCatalogerTask(info, locations, "") } diff --git a/syft/file/cataloger/filemetadata/cataloger.go b/syft/file/cataloger/filemetadata/cataloger.go index d0330cfe7..427444534 100644 --- a/syft/file/cataloger/filemetadata/cataloger.go +++ b/syft/file/cataloger/filemetadata/cataloger.go @@ -1,12 +1,13 @@ package filemetadata import ( - "github.com/wagoodman/go-partybus" - "github.com/wagoodman/go-progress" + "fmt" + + "github.com/dustin/go-humanize" "github.com/anchore/syft/internal/bus" "github.com/anchore/syft/internal/log" - "github.com/anchore/syft/syft/event" + "github.com/anchore/syft/syft/event/monitor" "github.com/anchore/syft/syft/file" ) @@ -20,7 +21,6 @@ func NewCataloger() *Cataloger { func (i *Cataloger) Catalog(resolver file.Resolver, coordinates ...file.Coordinates) (map[file.Coordinates]file.Metadata, error) { results := make(map[file.Coordinates]file.Metadata) var locations <-chan file.Location - if len(coordinates) == 0 { locations = resolver.AllLocations() } else { @@ -43,36 +43,35 @@ func (i *Cataloger) Catalog(resolver file.Resolver, coordinates ...file.Coordina }() } - stage, prog := metadataCatalogingProgress(int64(len(locations))) + prog := metadataCatalogingProgress(int64(len(locations))) for location := range locations { - stage.Current = location.RealPath + prog.Increment() + prog.AtomicStage.Set(location.Path()) + metadata, err := resolver.FileMetadataByLocation(location) if err != nil { return nil, err } results[location.Coordinates] = metadata - prog.Increment() } + log.Debugf("file metadata cataloger processed %d files", prog.Current()) + + prog.AtomicStage.Set(fmt.Sprintf("%s locations", humanize.Comma(prog.Current()))) prog.SetCompleted() + return results, nil } -func metadataCatalogingProgress(locations int64) (*progress.Stage, *progress.Manual) { - stage := &progress.Stage{} - prog := progress.NewManual(locations) - - bus.Publish(partybus.Event{ - Type: event.FileMetadataCatalogerStarted, - Value: struct { - progress.Stager - progress.Progressable - }{ - Stager: progress.Stager(stage), - Progressable: prog, +func metadataCatalogingProgress(locations int64) *monitor.CatalogerTaskProgress { + info := monitor.GenericTask{ + Title: monitor.Title{ + Default: "Catalog file metadata", + WhileRunning: "Cataloging file metadata", + OnSuccess: "Cataloged file metadata", }, - }) + } - return stage, prog + return bus.StartCatalogerTask(info, locations, "") } diff --git a/syft/pkg/cataloger/catalog.go b/syft/pkg/cataloger/catalog.go index 840ea72b7..e87840347 100644 --- a/syft/pkg/cataloger/catalog.go +++ b/syft/pkg/cataloger/catalog.go @@ -6,14 +6,14 @@ import ( "runtime/debug" "sync" + "github.com/dustin/go-humanize" "github.com/hashicorp/go-multierror" - "github.com/wagoodman/go-partybus" "github.com/wagoodman/go-progress" "github.com/anchore/syft/internal/bus" "github.com/anchore/syft/internal/log" "github.com/anchore/syft/syft/artifact" - "github.com/anchore/syft/syft/event" + "github.com/anchore/syft/syft/event/monitor" "github.com/anchore/syft/syft/file" "github.com/anchore/syft/syft/linux" "github.com/anchore/syft/syft/pkg" @@ -35,21 +35,6 @@ type catalogResult struct { Error error } -// newMonitor creates a new Monitor object and publishes the object on the bus as a PackageCatalogerStarted event. -func newMonitor() (*progress.Manual, *progress.Manual) { - filesProcessed := progress.Manual{} - packagesDiscovered := progress.Manual{} - - bus.Publish(partybus.Event{ - Type: event.PackageCatalogerStarted, - Value: Monitor{ - FilesProcessed: progress.Monitorable(&filesProcessed), - PackagesDiscovered: progress.Monitorable(&packagesDiscovered), - }, - }) - return &filesProcessed, &packagesDiscovered -} - func runCataloger(cataloger pkg.Cataloger, resolver file.Resolver) (catalogerResult *catalogResult, err error) { // handle individual cataloger panics defer func() { @@ -101,6 +86,7 @@ func runCataloger(cataloger pkg.Cataloger, resolver file.Resolver) (catalogerRes } catalogerResult.Packages = append(catalogerResult.Packages, p) } + catalogerResult.Relationships = append(catalogerResult.Relationships, relationships...) log.WithFields("cataloger", cataloger.Name()).Trace("cataloging complete") return catalogerResult, err @@ -116,9 +102,7 @@ func Catalog(resolver file.Resolver, _ *linux.Release, parallelism int, cataloge catalog := pkg.NewCollection() var allRelationships []artifact.Relationship - filesProcessed, packagesDiscovered := newMonitor() - defer filesProcessed.SetCompleted() - defer packagesDiscovered.SetCompleted() + prog := monitorPackageCatalogingTask() // perform analysis, accumulating errors for each failed analysis var errs error @@ -131,10 +115,11 @@ func Catalog(resolver file.Resolver, _ *linux.Release, parallelism int, cataloge jobs := make(chan pkg.Cataloger, nCatalogers) results := make(chan *catalogResult, nCatalogers) - discoveredPackages := make(chan int64, nCatalogers) waitGroup := sync.WaitGroup{} + var totalPackagesDiscovered int64 + for i := 0; i < parallelism; i++ { waitGroup.Add(1) @@ -148,20 +133,16 @@ func Catalog(resolver file.Resolver, _ *linux.Release, parallelism int, cataloge // ensure we set the error to be aggregated result.Error = err - discoveredPackages <- result.Discovered + prog.Add(result.Discovered) + totalPackagesDiscovered += result.Discovered + count := humanize.Comma(totalPackagesDiscovered) + prog.AtomicStage.Set(fmt.Sprintf("%s packages", count)) results <- result } }() } - // dynamically show updated discovered package status - go func() { - for discovered := range discoveredPackages { - packagesDiscovered.Add(discovered) - } - }() - // Enqueue the jobs for _, cataloger := range catalogers { jobs <- cataloger @@ -171,7 +152,6 @@ func Catalog(resolver file.Resolver, _ *linux.Release, parallelism int, cataloge // Wait for the jobs to finish waitGroup.Wait() close(results) - close(discoveredPackages) // collect the results for result := range results { @@ -186,6 +166,12 @@ func Catalog(resolver file.Resolver, _ *linux.Release, parallelism int, cataloge allRelationships = append(allRelationships, pkg.NewRelationships(catalog)...) + if errs != nil { + prog.SetError(errs) + } else { + prog.SetCompleted() + } + return catalog, allRelationships, errs } @@ -228,3 +214,16 @@ func packageFileOwnershipRelationships(p pkg.Package, resolver file.PathResolver } return relationships, nil } + +func monitorPackageCatalogingTask() *monitor.CatalogerTaskProgress { + info := monitor.GenericTask{ + Title: monitor.Title{ + Default: "Catalog packages", + WhileRunning: "Cataloging packages", + OnSuccess: "Cataloged packages", + }, + HideOnSuccess: false, + } + + return bus.StartCatalogerTask(info, -1, "") +} diff --git a/syft/pkg/cataloger/golang/licenses.go b/syft/pkg/cataloger/golang/licenses.go index 3787f4278..c21937bb9 100644 --- a/syft/pkg/cataloger/golang/licenses.go +++ b/syft/pkg/cataloger/golang/licenses.go @@ -20,6 +20,7 @@ import ( "github.com/go-git/go-git/v5/storage/memory" "github.com/scylladb/go-set/strset" + "github.com/anchore/syft/internal/bus" "github.com/anchore/syft/internal/licenses" "github.com/anchore/syft/internal/log" "github.com/anchore/syft/syft/event/monitor" @@ -31,7 +32,6 @@ import ( type goLicenses struct { opts CatalogerConfig localModCacheResolver file.WritableResolver - progress *monitor.CatalogerTask lowerLicenseFileNames *strset.Set } @@ -39,11 +39,6 @@ func newGoLicenses(opts CatalogerConfig) goLicenses { return goLicenses{ opts: opts, localModCacheResolver: modCacheResolver(opts.LocalModCacheDir), - progress: &monitor.CatalogerTask{ - SubStatus: true, - RemoveOnCompletion: true, - Title: "Downloading go mod", - }, lowerLicenseFileNames: strset.New(lowercaseLicenseFiles()...), } } @@ -123,7 +118,16 @@ func (c *goLicenses) getLicensesFromRemote(moduleName, moduleVersion string) ([] proxies := remotesForModule(c.opts.Proxies, c.opts.NoProxy, moduleName) - fsys, err := getModule(c.progress, proxies, moduleName, moduleVersion) + prog := bus.StartCatalogerTask(monitor.GenericTask{ + Title: monitor.Title{ + Default: "Download go mod", + WhileRunning: "Downloading go mod", + OnSuccess: "Downloaded go mod", + }, + HideOnSuccess: true, + }, -1, "") + + fsys, err := getModule(prog, proxies, moduleName, moduleVersion) if err != nil { return nil, err } @@ -206,7 +210,7 @@ func processCaps(s string) string { }) } -func getModule(progress *monitor.CatalogerTask, proxies []string, moduleName, moduleVersion string) (fsys fs.FS, err error) { +func getModule(progress *monitor.CatalogerTaskProgress, proxies []string, moduleName, moduleVersion string) (fsys fs.FS, err error) { for _, proxy := range proxies { u, _ := url.Parse(proxy) if proxy == "direct" { @@ -218,7 +222,7 @@ func getModule(progress *monitor.CatalogerTask, proxies []string, moduleName, mo fsys, err = getModuleProxy(progress, proxy, moduleName, moduleVersion) case "file": p := filepath.Join(u.Path, moduleName, "@v", moduleVersion) - progress.SetValue(fmt.Sprintf("file: %s", p)) + progress.AtomicStage.Set(fmt.Sprintf("file: %s", p)) fsys = os.DirFS(p) } if fsys != nil { @@ -228,9 +232,9 @@ func getModule(progress *monitor.CatalogerTask, proxies []string, moduleName, mo return } -func getModuleProxy(progress *monitor.CatalogerTask, proxy string, moduleName string, moduleVersion string) (out fs.FS, _ error) { +func getModuleProxy(progress *monitor.CatalogerTaskProgress, proxy string, moduleName string, moduleVersion string) (out fs.FS, _ error) { u := fmt.Sprintf("%s/%s/@v/%s.zip", proxy, moduleName, moduleVersion) - progress.SetValue(u) + progress.AtomicStage.Set(u) // get the module zip resp, err := http.Get(u) //nolint:gosec @@ -241,7 +245,7 @@ func getModuleProxy(progress *monitor.CatalogerTask, proxy string, moduleName st if resp.StatusCode != http.StatusOK { u = fmt.Sprintf("%s/%s/@v/%s.zip", proxy, strings.ToLower(moduleName), moduleVersion) - progress.SetValue(u) + progress.AtomicStage.Set(u) // try lowercasing it; some packages have mixed casing that really messes up the proxy resp, err = http.Get(u) //nolint:gosec @@ -284,14 +288,14 @@ func findVersionPath(f fs.FS, dir string) string { return "" } -func getModuleRepository(progress *monitor.CatalogerTask, moduleName string, moduleVersion string) (fs.FS, error) { +func getModuleRepository(progress *monitor.CatalogerTaskProgress, moduleName string, moduleVersion string) (fs.FS, error) { repoName := moduleName parts := strings.Split(moduleName, "/") if len(parts) > 2 { repoName = fmt.Sprintf("%s/%s/%s", parts[0], parts[1], parts[2]) } - progress.SetValue(fmt.Sprintf("git: %s", repoName)) + progress.AtomicStage.Set(fmt.Sprintf("git: %s", repoName)) f := memfs.New() buf := &bytes.Buffer{}