chore: revert custom index in favor of key_Split

Signed-off-by: Keith Zantow <keith.zantow@anchore.com>
2026-11-01 06:56:40 +01:00 · 2026-09-20 11:22:10 -04:00
parent 99615b7dfa
commit 53cd4254e4
81 changed files with 7461 additions and 6624 deletions
@@ -2,6 +2,7 @@ package bus

 import (
 	"context"
+	"sync"

 	"github.com/wagoodman/go-partybus"
 	"github.com/wagoodman/go-progress"
@@ -38,39 +39,96 @@ func Notify(message string) {
 	})
 }

-type catalogerTaskProgressKey struct{}
+type catalogerTaskRegistryKey struct{}

-// WithCatalogerTaskProgress returns a context whose cataloger tasks report into prog rather than
-// starting a row of their own.
+// catalogerTaskRegistry remembers the row each cataloger publishes, keyed by the publishing task's
+// identity.
 //
-// Nested cataloging runs under such a context: re-running the cataloger pipeline against the
-// contents of an archive is work that belongs to the progress of the task driving the recursion, not
-// a second start of every cataloger. Starting rows again per archive floods a consumer's display,
-// and where the new row carries an ID the consumer has already seen (every package cataloger does -
-// the ID is the cataloger name) it replaces the live row for that ID. The replaced row is then never
-// rendered again, its completion is never observed, and a UI that waits on every row it started - as
-// the syft CLI does - blocks forever on teardown.
-func WithCatalogerTaskProgress(ctx context.Context, prog *monitor.TaskProgress) context.Context {
-	return context.WithValue(ctx, catalogerTaskProgressKey{}, prog)
+// A cataloger runs more than once per scan: once over the scan root, then again inside every archive
+// the archive cataloger walks. A row per run floods the display, and reusing a published ID replaces
+// the live row - which is then never completed, hanging a UI (like the syft CLI) that waits on every
+// row it started. So a re-run reports into the row it already owns, and its count covers the whole
+// scan rather than the root alone.
+type catalogerTaskRegistry struct {
+	mu   sync.Mutex
+	rows map[string]*monitor.TaskProgress
 }

-// catalogerTaskProgressFromContext returns the progress that cataloger tasks started under this
-// context report into, or nil when they should start one of their own.
-func catalogerTaskProgressFromContext(ctx context.Context) *monitor.TaskProgress {
-	prog, _ := ctx.Value(catalogerTaskProgressKey{}).(*monitor.TaskProgress)
-	return prog
+// WithCatalogerTaskRegistry returns a context under which each cataloger publishes one row no matter
+// how many times it runs; without it every run publishes, which suits a single-pass caller.
+//
+// Install once per scan before any task runs, so a re-run finds the first run's row. Idempotent: a
+// second registry would hold none of the existing rows, so every re-run would publish again.
+func WithCatalogerTaskRegistry(ctx context.Context) context.Context {
+	if catalogerTaskRegistryFromContext(ctx) != nil {
+		return ctx
+	}
+	return context.WithValue(ctx, catalogerTaskRegistryKey{}, &catalogerTaskRegistry{
+		rows: make(map[string]*monitor.TaskProgress),
+	})
+}
+
+func catalogerTaskRegistryFromContext(ctx context.Context) *catalogerTaskRegistry {
+	reg, _ := ctx.Value(catalogerTaskRegistryKey{}).(*catalogerTaskRegistry)
+	return reg
+}
+
+// catalogerTaskKey identifies the row a task owns: its ID where it has one (a package cataloger's ID
+// is its name), otherwise its parent and title (as every file cataloger, which publishes no ID).
+func catalogerTaskKey(info monitor.GenericTask) string {
+	if info.ID != "" {
+		return "id:" + info.ID
+	}
+	return "title:" + info.ParentID + "/" + info.Title.Default
 }

 func StartCatalogerTask(ctx context.Context, info monitor.GenericTask, size int64, initialStage string) *monitor.TaskProgress {
-	if parent := catalogerTaskProgressFromContext(ctx); parent != nil {
-		// the stage is shared, so what this cataloger is working on shows up on the row already on
-		// screen; the count is not, so this task completing does not complete the row it reports into
-		return &monitor.TaskProgress{
-			AtomicStage: parent.AtomicStage,
-			Manual:      progress.NewManual(size),
-		}
+	reg := catalogerTaskRegistryFromContext(ctx)
+	if reg == nil {
+		return publishCatalogerTask(info, size, initialStage)
 	}

+	reg.mu.Lock()
+	defer reg.mu.Unlock()
+
+	key := catalogerTaskKey(info)
+	if row, ok := reg.rows[key]; ok {
+		resumeCatalogerTask(row, size, initialStage)
+		return row
+	}
+
+	t := publishCatalogerTask(info, size, initialStage)
+	reg.rows[key] = t
+
+	return t
+}
+
+// resumeCatalogerTask reopens a row an earlier run completed and grows its total by what this run
+// adds.
+//
+// Growing the total matters: a row completes once its count reaches its total, so incrementing a row
+// whose total covers only the first run makes it read as finished mid-walk. An indeterminate total on
+// either side stays indeterminate.
+//
+// Only a completed row is reopened; one carrying a real failure keeps it, since a later clean run
+// does not mean the earlier one succeeded.
+func resumeCatalogerTask(row *monitor.TaskProgress, size int64, initialStage string) {
+	if size < 0 || row.Size() < 0 {
+		row.SetTotal(-1)
+	} else {
+		row.SetTotal(row.Size() + size)
+	}
+
+	if progress.IsErrCompleted(row.Error()) {
+		row.SetError(nil)
+	}
+
+	if initialStage != "" {
+		row.AtomicStage.Set(initialStage)
+	}
+}
+
+func publishCatalogerTask(info monitor.GenericTask, size int64, initialStage string) *monitor.TaskProgress {
 	t := &monitor.TaskProgress{
 		AtomicStage: progress.NewAtomicStage(initialStage),
 		Manual:      progress.NewManual(size),
@@ -0,0 +1,152 @@
+package bus
+
+import (
+	"context"
+	"errors"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	"github.com/wagoodman/go-progress"
+
+	"github.com/anchore/syft/syft/event/monitor"
+)
+
+func digestsTask() monitor.GenericTask {
+	return monitor.GenericTask{
+		Title:    monitor.Title{Default: "File digests"},
+		ParentID: monitor.TopLevelCatalogingTaskID,
+	}
+}
+
+func TestStartCatalogerTask_withoutARegistryEveryRunPublishes(t *testing.T) {
+	// the right answer for a caller driving a single pass: nothing re-runs, so nothing should be reused
+	ctx := context.Background()
+
+	first := StartCatalogerTask(ctx, digestsTask(), 2, "")
+	second := StartCatalogerTask(ctx, digestsTask(), 3, "")
+
+	assert.NotSame(t, first, second)
+	assert.Equal(t, int64(2), first.Size())
+	assert.Equal(t, int64(3), second.Size())
+}
+
+func TestStartCatalogerTask_reRunReportsIntoTheRowItAlreadyOwns(t *testing.T) {
+	ctx := WithCatalogerTaskRegistry(context.Background())
+
+	first := StartCatalogerTask(ctx, digestsTask(), 2, "")
+	first.Increment()
+	first.Increment()
+	first.SetCompleted()
+
+	// the archive walk reaches the same cataloger again, over an archive's contents this time
+	second := StartCatalogerTask(ctx, digestsTask(), 3, "")
+	require.Same(t, first, second, "a re-run must find the row it already published")
+
+	assert.Equal(t, int64(5), second.Size(), "the total must cover both runs, not just the first")
+	assert.False(t, progress.IsCompleted(second), "a row with work still to do must not read as complete")
+
+	for i := 0; i < 3; i++ {
+		second.Increment()
+	}
+	second.SetCompleted()
+
+	assert.Equal(t, int64(5), second.Current())
+	assert.Equal(t, int64(5), second.Size(), "the count must not run past the total")
+	assert.True(t, progress.IsCompleted(second))
+}
+
+func TestStartCatalogerTask_aTotalResolvedOnCompletionIsAddable(t *testing.T) {
+	// a run that could not say how much it would do says so when it signs off (Manual.SetCompleted
+	// backfills the total from the count), so by the time a re-run arrives there is a real number to
+	// add to. This is the file metadata cataloger's shape: it publishes -1 every run.
+	ctx := WithCatalogerTaskRegistry(context.Background())
+
+	row := StartCatalogerTask(ctx, digestsTask(), -1, "")
+	row.Increment()
+	row.SetCompleted()
+	require.Equal(t, int64(1), row.Size(), "signing off must resolve the total to what the run did")
+
+	require.Same(t, row, StartCatalogerTask(ctx, digestsTask(), -1, ""))
+	assert.Equal(t, int64(-1), row.Size(), "a re-run that cannot say its own size makes the row indeterminate again")
+
+	row.Increment()
+	row.SetCompleted()
+	assert.Equal(t, int64(2), row.Current())
+	assert.Equal(t, int64(2), row.Size(), "and signing off resolves it again, over both runs")
+}
+
+func TestStartCatalogerTask_indeterminateTotalsStayIndeterminate(t *testing.T) {
+	// a run whose size is unknown cannot be added to one that is known
+	tests := []struct {
+		name        string
+		first, next int64
+		complete    func(row *monitor.TaskProgress)
+	}{
+		{
+			name:  "the re-run cannot say its own size",
+			first: 2, next: -1,
+			complete: func(row *monitor.TaskProgress) { row.Increment(); row.SetCompleted() },
+		},
+		{
+			name:  "the first run did nothing, so completing resolved nothing",
+			first: -1, next: 3,
+			complete: func(row *monitor.TaskProgress) { row.SetCompleted() },
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			ctx := WithCatalogerTaskRegistry(context.Background())
+
+			row := StartCatalogerTask(ctx, digestsTask(), tt.first, "")
+			tt.complete(row)
+
+			require.Same(t, row, StartCatalogerTask(ctx, digestsTask(), tt.next, ""))
+			assert.Equal(t, int64(-1), row.Size())
+			assert.False(t, progress.IsCompleted(row), "an indeterminate row is not complete until it says so")
+		})
+	}
+}
+
+func TestStartCatalogerTask_aFailedRowKeepsItsFailure(t *testing.T) {
+	// a later run finding nothing wrong does not mean the earlier one succeeded
+	ctx := WithCatalogerTaskRegistry(context.Background())
+	failure := errors.New("unable to read the thing")
+
+	row := StartCatalogerTask(ctx, digestsTask(), 2, "")
+	row.SetError(failure)
+
+	require.Same(t, row, StartCatalogerTask(ctx, digestsTask(), 3, ""))
+	assert.ErrorIs(t, row.Error(), failure)
+}
+
+func TestStartCatalogerTask_rowsAreKeyedByTaskIdentity(t *testing.T) {
+	ctx := WithCatalogerTaskRegistry(context.Background())
+
+	byID := func(id string) monitor.GenericTask {
+		return monitor.GenericTask{ID: id, Title: monitor.Title{Default: "shared title"}}
+	}
+
+	assert.Same(t, StartCatalogerTask(ctx, byID("a"), -1, ""), StartCatalogerTask(ctx, byID("a"), -1, ""))
+	assert.NotSame(t, StartCatalogerTask(ctx, byID("a"), -1, ""), StartCatalogerTask(ctx, byID("b"), -1, ""),
+		"an ID is the identity where there is one, even when two tasks share a title")
+
+	// a task publishing without an ID, as every file cataloger does, is keyed by where it sits and what
+	// it is called
+	underCataloging := monitor.GenericTask{Title: monitor.Title{Default: "Files"}, ParentID: "cataloging"}
+	underSomethingElse := monitor.GenericTask{Title: monitor.Title{Default: "Files"}, ParentID: "other"}
+
+	assert.Same(t, StartCatalogerTask(ctx, underCataloging, -1, ""), StartCatalogerTask(ctx, underCataloging, -1, ""))
+	assert.NotSame(t, StartCatalogerTask(ctx, underCataloging, -1, ""), StartCatalogerTask(ctx, underSomethingElse, -1, ""))
+}
+
+func TestWithCatalogerTaskRegistry_isIdempotent(t *testing.T) {
+	// anything that re-runs catalogers may ask for a registry; a second one would hold none of the rows
+	// already published and every re-run would publish again
+	ctx := WithCatalogerTaskRegistry(context.Background())
+	row := StartCatalogerTask(ctx, digestsTask(), 2, "")
+
+	nested := WithCatalogerTaskRegistry(ctx)
+	assert.Same(t, row, StartCatalogerTask(nested, digestsTask(), 2, ""))
+}