diff --git a/.github/workflows/proxied-local-smoke.yml b/.github/workflows/proxied-local-smoke.yml index 49d7a5128..d8d92221e 100644 --- a/.github/workflows/proxied-local-smoke.yml +++ b/.github/workflows/proxied-local-smoke.yml @@ -23,6 +23,7 @@ on: - 'cmd/bd/proxied_server*.go' - 'cmd/bd/dolt*.go' - 'cmd/bd/*_proxied_server.go' + - 'cmd/bd/purge*.go' - 'cmd/bd/*managed_local*' - 'cmd/bd/proxied_outage_reconnect_integration_test.go' - 'cmd/bd/db_proxy_child.go' @@ -49,6 +50,7 @@ on: - 'cmd/bd/proxied_server*.go' - 'cmd/bd/dolt*.go' - 'cmd/bd/*_proxied_server.go' + - 'cmd/bd/purge*.go' - 'cmd/bd/*managed_local*' - 'cmd/bd/proxied_outage_reconnect_integration_test.go' - 'cmd/bd/db_proxy_child.go' @@ -125,7 +127,8 @@ jobs: TestManagedLocalProxiedDoltStartRefusesOverLiveProxy \ TestManagedLocalProxiedDoltStartRefusesWithProxyDown \ TestManagedLocalProxiedDoltLifecycleLeavesOtherTopologiesAlone \ - TestManagedLocalProxiedTrackerUOWConformance + TestManagedLocalProxiedTrackerUOWConformance \ + TestManagedLocalProxiedPurgeWispsPlaneRetention do if ! grep -qx "$test_name" <<<"$listed"; then echo "managed-local lifecycle test $test_name is missing; got: '$listed'" >&2 diff --git a/CHANGELOG.md b/CHANGELOG.md index 5c1a06274..88af367f2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -190,6 +190,38 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added +- **`bd purge` covers an orchestrator's wisp retention sweep.** Three changes + that together let `bd purge --wisps-plane --older-than 168h --force` replace + a raw `DELETE FROM wisps` retention step: + - **Live-dependent protection, always on.** `bd purge` no longer deletes a + closed bead that a live bead depends on through a `parent-child`, + `tracks` or `blocks` edge — a closed molecule root whose step is still + open, a closed wisp a live convoy tracks, a closed blocker of live work. + "Live" is any status that is not done, including custom statuses. The + held-back count is reported as `live_dependent_skipped` in `--json` (and + on its own line in text output). The protection is on the role + (`issueops.SweepRequest.ProtectLiveDependents`); `bd prune` does not ask + for it. + - **`--wisps-plane`** selects every closed row stored in the wisps table, + including `--no-history` beads, which the default ephemeral selection + leaves to `bd prune`. It works on embedded, server and proxied + workspaces, and because it reaches durable-tier rows it requires + `--older-than` or `--pattern`, like `bd prune` + (`issueops.SweepWispsPlane`). + - **`--older-than` accepts hour and finer durations** (`36h`, `168h`, + `90m`, down to `1s`) on both `bd purge` and `bd prune`. Day values (`7`, + `7d`, `2w`) are unchanged. An `Nh` value used to be converted to whole + days: floored above a day (`36h` swept rows only 24 hours old) and + rounded UP to one day below it (`12h` kept everything younger than 24 + hours). Both are now taken exactly, so `bd prune --older-than 12h` now + deletes rows closed 12–24 hours ago that it used to keep. A value too + large to represent (e.g. `213504d`) is now refused instead of silently + wrapping to a tiny age. + - **`--limit N`** caps one `bd purge` run at N beads, oldest-closed first + (`issueops.SweepRequest.Limit`), so a large backlog drains in bounded + transactions; `--json` then adds `remaining` and `has_more`. Loop while + `has_more` is true. + - **`bd backup` works on a proxied-server workspace bd runs the Dolt server for.** `bd backup init`, `sync`, `remove`, `status` and `restore` are routed over the proxied provider; before this, a proxied workspace — the default @@ -310,6 +342,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Changed +- **`bd purge` keeps closed beads a live bead depends on.** A closed bead + that a not-done bead depends on through `parent-child`, `tracks` or + `blocks` is no longer purged (reported as `live_dependent_skipped`), so a + purge that used to delete a closed molecule root under a live step now + leaves it. See the `--wisps-plane` entry under Added. + - **Proxied-server refusals now say *why* they refuse.** The JSON a refused command prints gains a `reason` field next to the existing `code`, `error` and `mutates`: `design` for a refusal that is expected to stay (shared diff --git a/backend/conformance/role_bundle_cases.go b/backend/conformance/role_bundle_cases.go index 3cd4239d7..0b0725aa4 100644 --- a/backend/conformance/role_bundle_cases.go +++ b/backend/conformance/role_bundle_cases.go @@ -657,6 +657,11 @@ var roleContractCases = []roleContract{ RunSweeperEmptyMatchIsZeroAndNil, RunSweeperRecordsExactlyOneHistoryEntry, RunSweeperDoesNotMutateTheCallerRequest, + RunSweeperWispsPlaneClearsTheWholeWispsTable, + RunSweeperWispsPlaneRequiresAFilter, + RunSweeperProtectsLiveDependents, + RunSweeperProtectsLiveDependentsAcrossPlanes, + RunSweeperLimitTakesTheOldestClosedFirst, ), roleCases("TreeWalker", "TreeWalker()", oncePerRole, diff --git a/backend/conformance/sweeper_contract.go b/backend/conformance/sweeper_contract.go index db70d027b..bae89c58d 100644 --- a/backend/conformance/sweeper_contract.go +++ b/backend/conformance/sweeper_contract.go @@ -89,6 +89,11 @@ type SweeperFixture struct { // by hand. A nil Exec means "this backend cannot seed raw rows", and the // case that needs it SKIPS with that reason rather than passing quietly. Exec func(ctx context.Context, statements []SQLStatement) error + // AddDependencies wires edges between seeded rows, filled from the + // backend's DependencyEditor role so each adapter routes an edge to the + // dependency table of its source's plane itself. The live-dependent case + // needs it; a nil hook SKIPS that case with the reason. + AddDependencies func(ctx context.Context, req publicops.AddDependenciesRequest) error } // sweeperClosedAt is the stamp every seeded candidate carries, and @@ -345,7 +350,7 @@ func RunSweeperDryRunChangesNothing(t *testing.T, ctx context.Context, fixture S IDPattern: sweeperPattern(fixture, "dry"), } - preview := sweeperSweep(t, ctx, fixture, sweeperWithDryRun(request, true)) + preview := sweeperSweep(t, ctx, fixture, sweeperAsDryRun(request)) if !preview.DryRun { t.Error("preview.DryRun = false; the result must echo the request it answered") } @@ -356,7 +361,7 @@ func RunSweeperDryRunChangesNothing(t *testing.T, ctx context.Context, fixture S // A second preview: a dry run that had deleted anything reports a smaller // number here. - if again := sweeperSweep(t, ctx, fixture, sweeperWithDryRun(request, true)); again.Swept != preview.Swept { + if again := sweeperSweep(t, ctx, fixture, sweeperAsDryRun(request)); again.Swept != preview.Swept { t.Fatalf("second preview Swept = %d, first = %d: the first preview changed the store", again.Swept, preview.Swept) } @@ -556,7 +561,7 @@ func RunSweeperRecordsExactlyOneHistoryEntry(t *testing.T, ctx context.Context, } before := sweeperHistory(t, ctx, fixture) - sweeperSweep(t, ctx, fixture, sweeperWithDryRun(request, true)) + sweeperSweep(t, ctx, fixture, sweeperAsDryRun(request)) if after := sweeperHistory(t, ctx, fixture); after != before { t.Errorf("history went %d -> %d across a DRY RUN, want no entry at all", before, after) } @@ -609,6 +614,230 @@ func RunSweeperDoesNotMutateTheCallerRequest(t *testing.T, ctx context.Context, } } +// RunSweeperWispsPlaneClearsTheWholeWispsTable pins issueops.SweepWispsPlane: +// the selection is the wisps TABLE, so a closed no-history bead — durable-tier, +// and left alone by SweepEphemeral (RunSweeperLeavesNoHistoryBeadsToTheDurableTier) +// — goes with the closed ephemeral wisp, while a closed issues-plane row the +// same pattern admits stays. +func RunSweeperWispsPlaneClearsTheWholeWispsTable(t *testing.T, ctx context.Context, fixture SweeperFixture) { + t.Helper() + wisp := sweeperSeedClosedIssue(t, ctx, fixture, "plane", true) + durable := sweeperSeedClosedIssue(t, ctx, fixture, "plane", false) + noHistory := sweeperIssue(fixture, "plane", "nohist", false) + noHistory.NoHistory = true + if err := fixture.CreateWisp(ctx, noHistory, "sweeper-seed"); err != nil { + t.Fatalf("seeding %s: %v", noHistory.ID, err) + } + pinned := sweeperSeed(t, ctx, fixture, sweeperIssue(fixture, "plane", "pinned", true), func(issue *types.Issue) { + issue.Pinned = true + }) + + request := publicops.SweepRequest{ + Tier: publicops.SweepWispsPlane, + IDPattern: sweeperPattern(fixture, "plane"), + ClosedBefore: &sweeperCutoff, + } + preview := sweeperSweep(t, ctx, fixture, sweeperAsDryRun(request)) + if preview.Swept != 2 { + t.Fatalf("dry-run Swept = %d, want 2 (the ephemeral wisp and the no-history bead)", preview.Swept) + } + sweeperAssertWispRows(t, ctx, fixture, 3, wisp, noHistory.ID, pinned) + + result := sweeperSweep(t, ctx, fixture, request) + if result.Swept != 2 { + t.Errorf("Swept = %d, want 2 (the ephemeral wisp and the no-history bead)", result.Swept) + } + if result.Skipped.Pinned != 1 { + t.Errorf("Skipped.Pinned = %d, want 1 — the plane selector does not override pinning", result.Skipped.Pinned) + } + sweeperAssertWispRows(t, ctx, fixture, 0, wisp, noHistory.ID) + sweeperAssertWispRows(t, ctx, fixture, 1, pinned) + sweeperAssertIssueRows(t, ctx, fixture, 1, durable) +} + +// RunSweeperWispsPlaneRequiresAFilter pins that the wisps-plane tier carries +// the durable tier's require-a-filter gate: it reaches no-history beads, which +// are durable-tier. The refusal is asserted with its effect, as the durable +// case does. +func RunSweeperWispsPlaneRequiresAFilter(t *testing.T, ctx context.Context, fixture SweeperFixture) { + t.Helper() + id := sweeperSeedClosedIssue(t, ctx, fixture, "planegate", true) + + result, err := fixture.Sweeper.Sweep(ctx, publicops.SweepRequest{Tier: publicops.SweepWispsPlane}) + if !errors.Is(err, publicops.ErrValidation) { + t.Fatalf("Sweep(wisps-plane, no cutoff, no pattern) error = %v, want ErrValidation", err) + } + if result.Swept != 0 { + t.Errorf("refused sweep reported Swept = %d, want 0", result.Swept) + } + sweeperAssertWispRows(t, ctx, fixture, 1, id) +} + +// RunSweeperProtectsLiveDependents pins issueops.SweepRequest.ProtectLiveDependents: +// a closed row a LIVE row depends on through parent-child, tracks or blocks is +// held back and counted, while a closed row reached only by a non-structural +// edge, or only by a dependent that is itself closed, is swept. +// +// The live sources carry three different not-done statuses (open, +// in_progress, hooked), because "live" is the not-done set and not "open". +// It runs on the wisps-plane tier with a cutoff — the request an +// orchestrator's retention sweep sends — after proving on a dry run that the +// same rows ARE candidates without the protection. +func RunSweeperProtectsLiveDependents(t *testing.T, ctx context.Context, fixture SweeperFixture) { + t.Helper() + if fixture.AddDependencies == nil { + t.Skip("fixture has no AddDependencies hook; cannot wire the edges this case needs") + } + live := func(name string, status types.Status) string { + issue := sweeperIssue(fixture, "live", name, true) + issue.Status = status + issue.ClosedAt = nil + return sweeperSeed(t, ctx, fixture, issue, nil) + } + closed := func(name string) string { + return sweeperSeed(t, ctx, fixture, sweeperIssue(fixture, "live", name, true), nil) + } + + parent := closed("parent") + openChild := live("child-open", types.StatusOpen) + tracked := closed("tracked") + convoy := live("convoy", types.StatusInProgress) + blocker := closed("blocker") + hooked := live("hooked", types.StatusHooked) + related := closed("related") + relater := live("relater", types.StatusOpen) + doneParent := closed("done-parent") + doneChild := closed("done-child") + + if err := fixture.AddDependencies(ctx, publicops.AddDependenciesRequest{ + Actor: "sweeper-seed", + Edges: []publicops.DependencyEdge{ + {IssueID: openChild, DependsOnID: parent, Type: publicops.DepParentChild}, + {IssueID: convoy, DependsOnID: tracked, Type: publicops.DepTracks}, + {IssueID: hooked, DependsOnID: blocker, Type: publicops.DepBlocks}, + {IssueID: relater, DependsOnID: related, Type: publicops.DepRelated}, + {IssueID: doneChild, DependsOnID: doneParent, Type: publicops.DepParentChild}, + }, + }); err != nil { + t.Fatalf("wiring dependencies: %v", err) + } + + request := publicops.SweepRequest{ + Tier: publicops.SweepWispsPlane, + IDPattern: sweeperPattern(fixture, "live"), + ClosedBefore: &sweeperCutoff, + } + unprotected := sweeperSweep(t, ctx, fixture, sweeperAsDryRun(request)) + if unprotected.Swept != 6 || unprotected.Skipped.LiveDependent != 0 { + t.Fatalf("unprotected dry run: Swept = %d, LiveDependent = %d; want 6 and 0 — "+ + "the protection must be asked for", unprotected.Swept, unprotected.Skipped.LiveDependent) + } + + request.ProtectLiveDependents = true + result := sweeperSweep(t, ctx, fixture, request) + if result.Skipped.LiveDependent != 3 { + t.Errorf("Skipped.LiveDependent = %d, want 3 (the parent, the tracked bead and the blocker)", + result.Skipped.LiveDependent) + } + if result.Swept != 3 { + t.Errorf("Swept = %d, want 3 (the related-only bead and the closed parent/child pair)", result.Swept) + } + sweeperAssertWispRows(t, ctx, fixture, 3, parent, tracked, blocker) + sweeperAssertWispRows(t, ctx, fixture, 0, related, doneParent, doneChild) + sweeperAssertWispRows(t, ctx, fixture, 4, openChild, convoy, hooked, relater) +} + +// RunSweeperProtectsLiveDependentsAcrossPlanes pins that the live-dependent +// protection reads BOTH dependency tables: a closed wisp that a live DURABLE +// (issues-table) bead depends on through parent-child, tracks or blocks is +// kept, whatever not-done status the durable source carries. +func RunSweeperProtectsLiveDependentsAcrossPlanes(t *testing.T, ctx context.Context, fixture SweeperFixture) { + t.Helper() + if fixture.AddDependencies == nil { + t.Skip("fixture has no AddDependencies hook; cannot wire the edges this case needs") + } + durableLive := func(name string, status types.Status) string { + issue := sweeperIssue(fixture, "xplane", name, false) + issue.Status = status + issue.ClosedAt = nil + return sweeperSeed(t, ctx, fixture, issue, nil) + } + closedWisp := func(name string) string { + return sweeperSeed(t, ctx, fixture, sweeperIssue(fixture, "xplane", name, true), nil) + } + + parent := closedWisp("parent") + tracked := closedWisp("tracked") + blocker := closedWisp("blocker") + loose := closedWisp("loose") + child := durableLive("child", types.StatusOpen) + convoy := durableLive("convoy", types.StatusInProgress) + waiter := durableLive("waiter", types.StatusDeferred) + + if err := fixture.AddDependencies(ctx, publicops.AddDependenciesRequest{ + Actor: "sweeper-seed", + Edges: []publicops.DependencyEdge{ + {IssueID: child, DependsOnID: parent, Type: publicops.DepParentChild}, + {IssueID: convoy, DependsOnID: tracked, Type: publicops.DepTracks}, + {IssueID: waiter, DependsOnID: blocker, Type: publicops.DepBlocks}, + }, + }); err != nil { + t.Fatalf("wiring cross-plane dependencies: %v", err) + } + + result := sweeperSweep(t, ctx, fixture, publicops.SweepRequest{ + Tier: publicops.SweepWispsPlane, + IDPattern: sweeperPattern(fixture, "xplane"), + ClosedBefore: &sweeperCutoff, + ProtectLiveDependents: true, + }) + if result.Skipped.LiveDependent != 3 || result.Swept != 1 { + t.Errorf("Swept = %d, LiveDependent = %d; want 1 (the unlinked wisp) and 3", result.Swept, result.Skipped.LiveDependent) + } + sweeperAssertWispRows(t, ctx, fixture, 3, parent, tracked, blocker) + sweeperAssertWispRows(t, ctx, fixture, 0, loose) + sweeperAssertIssueRows(t, ctx, fixture, 3, child, convoy, waiter) +} + +// RunSweeperLimitTakesTheOldestClosedFirst pins issueops.SweepRequest.Limit: +// one call deletes at most Limit rows, oldest closed_at first, and reports the +// rest as Remaining; the next call picks up where it stopped. +func RunSweeperLimitTakesTheOldestClosedFirst(t *testing.T, ctx context.Context, fixture SweeperFixture) { + t.Helper() + seed := func(name string, closed time.Time) string { + return sweeperSeed(t, ctx, fixture, sweeperIssue(fixture, "limit", name, true), func(issue *types.Issue) { + issue.ClosedAt = &closed + }) + } + newest := seed("newest", sweeperClosedAt) + middle := seed("middle", sweeperClosedAt.Add(-24*time.Hour)) + oldest := seed("oldest", sweeperClosedAt.Add(-48*time.Hour)) + + request := publicops.SweepRequest{ + Tier: publicops.SweepEphemeral, + IDPattern: sweeperPattern(fixture, "limit"), + Limit: 2, + } + first := sweeperSweep(t, ctx, fixture, request) + if first.Swept != 2 || first.Remaining != 1 { + t.Fatalf("first limited sweep: Swept = %d, Remaining = %d; want 2 and 1", first.Swept, first.Remaining) + } + sweeperAssertWispRows(t, ctx, fixture, 0, oldest, middle) + sweeperAssertWispRows(t, ctx, fixture, 1, newest) + + second := sweeperSweep(t, ctx, fixture, request) + if second.Swept != 1 || second.Remaining != 0 { + t.Fatalf("second limited sweep: Swept = %d, Remaining = %d; want 1 and 0", second.Swept, second.Remaining) + } + sweeperAssertWispRows(t, ctx, fixture, 0, newest) + + if _, err := fixture.Sweeper.Sweep(ctx, publicops.SweepRequest{ + Tier: publicops.SweepEphemeral, IDPattern: sweeperPattern(fixture, "limit"), Limit: -1, + }); !errors.Is(err, publicops.ErrValidation) { + t.Fatalf("negative Limit error = %v, want ErrValidation", err) + } +} + // --- fixture helpers ------------------------------------------------------- // sweeperPattern is the glob that scopes one case's ids: a sweep is asked of a @@ -664,8 +893,8 @@ func sweeperSeedOpen(t *testing.T, ctx context.Context, fixture SweeperFixture, return sweeperSeed(t, ctx, fixture, issue, nil) } -func sweeperWithDryRun(request publicops.SweepRequest, dryRun bool) publicops.SweepRequest { - request.DryRun = dryRun +func sweeperAsDryRun(request publicops.SweepRequest) publicops.SweepRequest { + request.DryRun = true return request } diff --git a/cmd/bd/BUILD.bazel b/cmd/bd/BUILD.bazel index 1267ccb2f..177e87af0 100644 --- a/cmd/bd/BUILD.bazel +++ b/cmd/bd/BUILD.bazel @@ -921,6 +921,8 @@ go_test( "prune_proxied_integration_test.go", "purge_embedded_test.go", "purge_test.go", + "purge_wisps_plane_integration_test.go", + "purge_wisps_plane_managed_local_test.go", "query_embedded_test.go", "query_limit_policy_test.go", "query_proxied_integration_test.go", diff --git a/cmd/bd/prune.go b/cmd/bd/prune.go index 510a62101..e230e199c 100644 --- a/cmd/bd/prune.go +++ b/cmd/bd/prune.go @@ -73,7 +73,7 @@ func init() { pruneCmd.Flags().BoolP("force", "f", false, "Actually prune (without this, shows preview)") pruneCmd.Flags().Bool("ignore-references", false, "Delete closed beads even when referenced by open beads (use with care; see --help for details)") pruneCmd.Flags().Bool("dry-run", false, "Preview what would be pruned with stats") - pruneCmd.Flags().String("older-than", "", "Only prune beads closed more than N ago (e.g., 30d, 2w, 60)") + pruneCmd.Flags().String("older-than", "", "Only prune beads closed more than N ago (e.g., 30d, 2w, 60, 36h)") pruneCmd.Flags().String("pattern", "", "Only prune beads matching ID glob pattern (e.g., 'gm-old-*')") rootCmd.AddCommand(pruneCmd) } diff --git a/cmd/bd/purge.go b/cmd/bd/purge.go index 0445e5ee7..0337db793 100644 --- a/cmd/bd/purge.go +++ b/cmd/bd/purge.go @@ -2,6 +2,7 @@ package main import ( "fmt" + "math" "strconv" "strings" "time" @@ -36,6 +37,12 @@ type purgeScope struct { // is not done. `bd prune` asks unless --ignore-references; `bd purge` // never does, because a wisp's citations are as transient as the wisp. protectReferenced bool + // protectLiveDependents asks the role to skip candidates a live bead + // depends on through a parent-child, tracks or blocks edge. `bd purge` + // always asks: deleting a closed wisp whose child, tracker or blocked bead + // is still live cuts that live work loose from its root, and there is no + // retention reason to want that — a deliberate removal is `bd delete`. + protectLiveDependents bool // reportsReferences publishes the reference-skip keys under --json. It is // separate from protectReferenced because `bd prune --ignore-references` // still publishes them, as zeroes, which is the shipped shape. @@ -52,7 +59,21 @@ Closed ephemeral beads (wisps, transient molecules) accumulate rapidly and have no value once closed. This command removes them to reclaim storage. Deletes: issues, dependencies, labels, events, and comments for matching beads. -Skips: pinned beads (protected). +Skips: pinned beads, and closed beads a live bead still depends on through a +parent-child, tracks or blocks edge (a closed molecule root whose step is open, +a closed bead a live convoy tracks). Live means any status that is not done. + +--wisps-plane selects by storage plane instead: every closed row stored in the +wisps table, including --no-history beads, which the default ephemeral +selection leaves to ` + "`bd prune`" + `. It reaches durable-tier rows, so it requires +--older-than or --pattern, like ` + "`bd prune`" + `. + +--limit N caps one run at N beads, oldest-closed first, so a large backlog can +be drained in bounded transactions; --json then reports "remaining" and +"has_more". Loop while has_more is true. + +--older-than takes days (7, 7d), weeks (2w) or a duration with hour precision +(36h, 168h, 90m). To delete closed non-ephemeral beads (regular tasks, features, bugs, etc.) use ` + "`bd prune`" + ` instead. @@ -65,6 +86,9 @@ EXAMPLES: bd purge --force # Delete all closed ephemeral beads bd purge --older-than 7d --force # Only purge items closed 7+ days ago bd purge --pattern "*-wisp-*" # Only purge matching ID pattern + bd purge --older-than 36h --force # Hour precision: closed 36+ hours ago + bd purge --wisps-plane --older-than 168h --force + # Every closed wisps-table row, incl. no-history bd purge --dry-run # Detailed preview with stats`, SilenceUsage: true, SilenceErrors: true, @@ -76,14 +100,20 @@ EXAMPLES: } }() - return runPurgeOrPrune(cmd, purgeScope{ - cmdName: "purge", - pastTense: "purged", - countKey: "purged_count", - dryRunCountKey: "purge_count", - subjectNoun: "closed ephemeral bead", - tier: issueops.SweepEphemeral, - }) + scope := purgeScope{ + cmdName: "purge", + pastTense: "purged", + countKey: "purged_count", + dryRunCountKey: "purge_count", + subjectNoun: "closed ephemeral bead", + tier: issueops.SweepEphemeral, + protectLiveDependents: true, + } + if wispsPlane, _ := cmd.Flags().GetBool("wisps-plane"); wispsPlane { + scope.subjectNoun = "closed wisps-plane bead" + scope.tier = issueops.SweepWispsPlane + } + return runPurgeOrPrune(cmd, scope) }, } @@ -110,6 +140,11 @@ func runPurgeOrPrune(cmd *cobra.Command, scope purgeScope) error { dryRun, _ := cmd.Flags().GetBool("dry-run") olderThan, _ := cmd.Flags().GetString("older-than") pattern, _ := cmd.Flags().GetString("pattern") + // Only `bd purge` defines --limit; GetInt on an undefined flag is 0, "no cap". + limit, _ := cmd.Flags().GetInt("limit") + if limit < 0 { + return HandleErrorRespectJSON("invalid --limit value %d: must be 0 (no limit) or positive", limit) + } // The ROLE refuses an unfiltered durable sweep — that guard is // workapi.ValidateSweepRequest, below every front door. This branch is here @@ -117,30 +152,36 @@ func runPurgeOrPrune(cmd *cobra.Command, scope purgeScope) error { // flags a person who typed `bd prune --force` has to reach for. The contract // case RunSweeperRefusesAnUnfilteredDurableSweep proves the guard survives // this branch being deleted. - if scope.tier == issueops.SweepDurable && olderThan == "" && pattern == "" { + if scope.tier != issueops.SweepEphemeral && olderThan == "" && pattern == "" { + command := scope.cmdName + if scope.tier == issueops.SweepWispsPlane { + command += " --wisps-plane" + } return HandleErrorWithHint( - fmt.Sprintf("bd %s requires --older-than or --pattern", scope.cmdName), + fmt.Sprintf("bd %s requires --older-than or --pattern", command), "Protects against accidental bulk deletion. Use `--pattern '*'` to\n"+ " include all closed beads in this scope, or `--older-than 1d`\n"+ " / `--pattern ''` to narrow the deletion.") } request := issueops.SweepRequest{ - Actor: actor, - Tier: scope.tier, - IDPattern: pattern, - ProtectReferenced: scope.protectReferenced, + Actor: actor, + Tier: scope.tier, + IDPattern: pattern, + ProtectReferenced: scope.protectReferenced, + ProtectLiveDependents: scope.protectLiveDependents, + Limit: limit, // A --dry-run and an UNCONFIRMED run ask the role the same question — // "what would this do" — so both send DryRun. --force is this // command's confirmation, not a request field. DryRun: dryRun || !force, } if olderThan != "" { - days, err := parseHumanDuration(olderThan) + age, err := parseOlderThan(olderThan) if err != nil { return HandleErrorRespectJSON("invalid --older-than value %q: %v", olderThan, err) } - cutoff := time.Now().UTC().AddDate(0, 0, -days) + cutoff := time.Now().UTC().Add(-age) request.ClosedBefore = &cutoff } @@ -165,16 +206,16 @@ func runPurgeOrPrune(cmd *cobra.Command, scope purgeScope) error { switch { case result.Swept == 0: - return emitSweepEmpty(scope, olderThan, pattern, result) + return emitSweepEmpty(scope, olderThan, pattern, limit, result) case dryRun: - return emitSweepDryRun(scope, result) + return emitSweepDryRun(scope, limit, result) case !force: return emitSweepConfirm(scope, olderThan, pattern, result) } commandDidWrite.Store(true) commandMayEmptyJSONLExport.Store(true) - return emitSweepResult(scope, result) + return emitSweepResult(scope, limit, result) } // warnSweepDefenseSkips reports the candidates the role's own recheck threw @@ -208,12 +249,40 @@ func addReferenceStats(scope purgeScope, stats map[string]interface{}, result is } } -func emitSweepEmpty(scope purgeScope, olderThan, pattern string, result issueops.SweepResult) error { +// addLimitStats reports what --limit left for the next run. It is attached +// only when the caller asked for a limit, so the unlimited shape is unchanged. +func addLimitStats(stats map[string]interface{}, limit int, result issueops.SweepResult) { + if limit <= 0 { + return + } + stats["remaining"] = result.Remaining + stats["has_more"] = result.Remaining > 0 +} + +// printLimitNote tells a text reader that --limit left rows behind. +func printLimitNote(result issueops.SweepResult) { + if result.Remaining > 0 { + fmt.Printf(" Remaining (over --limit): %d — run again to continue\n", result.Remaining) + } +} + +// addLiveDependentStats attaches the live-dependent skip count to a --json +// payload when the protection held anything back, the way pinned_skipped is +// reported. +func addLiveDependentStats(stats map[string]interface{}, result issueops.SweepResult) { + if result.Skipped.LiveDependent > 0 { + stats["live_dependent_skipped"] = result.Skipped.LiveDependent + } +} + +func emitSweepEmpty(scope purgeScope, olderThan, pattern string, limit int, result issueops.SweepResult) error { if jsonOutput { stats := map[string]interface{}{ scope.countKey: 0, "message": fmt.Sprintf("No %ss to %s", scope.subjectNoun, scope.cmdName), } + addLiveDependentStats(stats, result) + addLimitStats(stats, limit, result) addReferenceStats(scope, stats, result) return outputJSON(stats) } @@ -225,6 +294,10 @@ func emitSweepEmpty(scope purgeScope, olderThan, pattern string, result issueops msg += fmt.Sprintf(" (matching %q)", pattern) } fmt.Println(msg) + if result.Skipped.LiveDependent > 0 { + fmt.Println(ui.MutedStyle.Render(fmt.Sprintf( + " (%d closed bead(s) protected by live dependents)", result.Skipped.LiveDependent))) + } if result.Skipped.Referenced > 0 { fmt.Println(ui.MutedStyle.Render(fmt.Sprintf( " (%d closed bead(s) protected by open-bead references — use --ignore-references to override)", @@ -233,7 +306,7 @@ func emitSweepEmpty(scope purgeScope, olderThan, pattern string, result issueops return nil } -func emitSweepDryRun(scope purgeScope, result issueops.SweepResult) error { +func emitSweepDryRun(scope purgeScope, limit int, result issueops.SweepResult) error { if jsonOutput { stats := map[string]interface{}{ "dry_run": true, @@ -245,6 +318,8 @@ func emitSweepDryRun(scope purgeScope, result issueops.SweepResult) error { if result.Skipped.Pinned > 0 { stats["pinned_skipped"] = result.Skipped.Pinned } + addLiveDependentStats(stats, result) + addLimitStats(stats, limit, result) addReferenceStats(scope, stats, result) return outputJSON(stats) } @@ -255,6 +330,9 @@ func emitSweepDryRun(scope purgeScope, result issueops.SweepResult) error { if result.Skipped.Pinned > 0 { fmt.Printf(" Pinned (skipped): %d\n", result.Skipped.Pinned) } + if result.Skipped.LiveDependent > 0 { + fmt.Printf(" Live dependent (skipped): %d\n", result.Skipped.LiveDependent) + } if result.Skipped.Referenced > 0 { fmt.Printf(" %s %d\n", ui.MutedStyle.Render("Referenced (skipped):"), result.Skipped.Referenced) sample := result.ReferencedIDs @@ -271,6 +349,7 @@ func emitSweepDryRun(scope purgeScope, result issueops.SweepResult) error { } fmt.Printf(" %s %s%s\n", ui.MutedStyle.Render("Referenced IDs (sample):"), strings.Join(idStrs, ", "), suffix) } + printLimitNote(result) fmt.Printf("\n(Dry-run mode — no changes made)\n") return nil } @@ -280,10 +359,16 @@ func emitSweepConfirm(scope purgeScope, olderThan, pattern string, result issueo if result.Skipped.Pinned > 0 { fmt.Printf("Skipping %d pinned bead(s)\n", result.Skipped.Pinned) } + if result.Skipped.LiveDependent > 0 { + fmt.Printf("Skipping %d bead(s) a live bead depends on\n", result.Skipped.LiveDependent) + } if result.Skipped.Referenced > 0 { fmt.Println(ui.MutedStyle.Render(fmt.Sprintf("Skipping %d referenced bead(s)", result.Skipped.Referenced))) } hint := fmt.Sprintf("bd %s --force", scope.cmdName) + if scope.tier == issueops.SweepWispsPlane { + hint += " --wisps-plane" + } if olderThan != "" { hint += " --older-than " + olderThan } @@ -295,7 +380,7 @@ func emitSweepConfirm(scope purgeScope, olderThan, pattern string, result issueo fmt.Sprintf("Use --force to confirm or --dry-run to preview.\n %s", hint)) } -func emitSweepResult(scope purgeScope, result issueops.SweepResult) error { +func emitSweepResult(scope purgeScope, limit int, result issueops.SweepResult) error { if jsonOutput { stats := map[string]interface{}{ scope.countKey: result.Swept, @@ -306,6 +391,8 @@ func emitSweepResult(scope purgeScope, result issueops.SweepResult) error { if result.Skipped.Pinned > 0 { stats["pinned_skipped"] = result.Skipped.Pinned } + addLiveDependentStats(stats, result) + addLimitStats(stats, limit, result) addReferenceStats(scope, stats, result) return outputJSON(stats) } @@ -316,9 +403,13 @@ func emitSweepResult(scope purgeScope, result issueops.SweepResult) error { if result.Skipped.Pinned > 0 { fmt.Printf(" Pinned (skipped): %d\n", result.Skipped.Pinned) } + if result.Skipped.LiveDependent > 0 { + fmt.Printf(" Live dependent (skipped): %d\n", result.Skipped.LiveDependent) + } if result.Skipped.Referenced > 0 { fmt.Printf(" %s %d\n", ui.MutedStyle.Render("Referenced (skipped):"), result.Skipped.Referenced) } + printLimitNote(result) return nil } @@ -329,53 +420,77 @@ func capitalize(s string) string { return strings.ToUpper(s[:1]) + s[1:] } -// parseHumanDuration parses a human-friendly duration string into days. -// Accepts: "7d", "30d", "24h", "2w", or just a number (treated as days). -func parseHumanDuration(s string) (int, error) { +// parseOlderThan parses an --older-than value into an age. +// +// Day-denominated values keep the meaning they always had: a bare number is +// days ("7"), and "7d" / "2w" are days and weeks. Anything else is a Go +// duration with hour precision or finer — "36h", "168h", "90m", "1h30m" — and +// is taken exactly; "Nh" used to be floored to whole days, which made "36h" +// sweep rows only 24 hours old. The suffix letters are case-insensitive, as +// they always were. The age must be positive. +func parseOlderThan(s string) (time.Duration, error) { s = strings.TrimSpace(s) if s == "" { return 0, fmt.Errorf("empty duration") } - // Plain number = days if days, err := strconv.Atoi(s); err == nil { - if days <= 0 { - return 0, fmt.Errorf("duration must be positive") - } - return days, nil + return daysAge(int64(days)) } - // Parse suffix unit := s[len(s)-1] - numStr := s[:len(s)-1] - num, err := strconv.Atoi(numStr) - if err != nil { - return 0, fmt.Errorf("invalid number %q", numStr) + switch unit { + case 'd', 'D', 'w', 'W': + num, err := strconv.Atoi(s[:len(s)-1]) + if err != nil { + return 0, fmt.Errorf("invalid number %q", s[:len(s)-1]) + } + days := int64(num) + if unit == 'w' || unit == 'W' { + if days > maxOlderThanDays/7 || days < -maxOlderThanDays/7 { + return 0, errOlderThanRange + } + days *= 7 + } + return daysAge(days) } - if num <= 0 { + + age, err := time.ParseDuration(strings.ToLower(s)) + if err != nil { + // time.ParseDuration refuses an out-of-range value ("9999999h") the + // same way it refuses a malformed one, so neither can wrap around. + return 0, fmt.Errorf("invalid or out-of-range duration %q (use a number of days, Nd, Nw, or a duration such as 36h or 90m)", s) + } + return positiveAge(age) +} + +// maxOlderThanDays is the largest day count a time.Duration can hold. A larger +// value used to overflow silently — 213504d wrapped to 25 minutes — which on a +// destructive command selected nearly everything instead of nothing. +const maxOlderThanDays = math.MaxInt64 / int64(24*time.Hour) + +var errOlderThanRange = fmt.Errorf("duration out of range (at most %d days)", maxOlderThanDays) + +func daysAge(days int64) (time.Duration, error) { + if days > maxOlderThanDays { + return 0, errOlderThanRange + } + return positiveAge(time.Duration(days) * 24 * time.Hour) +} + +func positiveAge(age time.Duration) (time.Duration, error) { + if age <= 0 { return 0, fmt.Errorf("duration must be positive") } - - switch unit { - case 'h', 'H': - days := num / 24 - if days == 0 { - days = 1 // minimum 1 day - } - return days, nil - case 'd', 'D': - return num, nil - case 'w', 'W': - return num * 7, nil - default: - return 0, fmt.Errorf("unknown unit %q (use h, d, or w)", string(unit)) - } + return age, nil } func init() { purgeCmd.Flags().BoolP("force", "f", false, "Actually purge (without this, shows preview)") purgeCmd.Flags().Bool("dry-run", false, "Preview what would be purged with stats") - purgeCmd.Flags().String("older-than", "", "Only purge beads closed more than N ago (e.g., 7d, 2w, 30)") + purgeCmd.Flags().String("older-than", "", "Only purge beads closed more than N ago (e.g., 7d, 2w, 30, 36h)") + purgeCmd.Flags().Int("limit", 0, "Purge at most N beads this run, oldest-closed first (0 = no limit); --json reports remaining/has_more") + purgeCmd.Flags().Bool("wisps-plane", false, "Select every closed row in the wisps table, including --no-history beads (requires --older-than or --pattern)") purgeCmd.Flags().String("pattern", "", "Only purge beads matching ID glob pattern (e.g., *-wisp-*)") rootCmd.AddCommand(purgeCmd) } diff --git a/cmd/bd/purge_test.go b/cmd/bd/purge_test.go index 89d84bb1b..d44cf8221 100644 --- a/cmd/bd/purge_test.go +++ b/cmd/bd/purge_test.go @@ -1,40 +1,68 @@ package main -import "testing" +import ( + "testing" + "time" +) -func TestParseHumanDuration(t *testing.T) { +// TestParseOlderThan pins --older-than on both `bd purge` and `bd prune`. +// Day-denominated values keep their meaning; hour (and finer) values are taken +// exactly rather than floored to whole days — "36h" used to become 24h and +// sweep rows younger than the caller asked for. +func TestParseOlderThan(t *testing.T) { + const day = 24 * time.Hour tests := []struct { input string - want int + want time.Duration wantErr bool }{ - {"7", 7, false}, - {"30", 30, false}, - {"7d", 7, false}, - {"30d", 30, false}, - {"2w", 14, false}, - {"1w", 7, false}, - {"48h", 2, false}, - {"12h", 1, false}, // rounds up to 1 day minimum - {"7D", 7, false}, - {"2W", 14, false}, + // Unchanged day semantics. + {"7", 7 * day, false}, + {"30", 30 * day, false}, + {"7d", 7 * day, false}, + {"30d", 30 * day, false}, + {"2w", 14 * day, false}, + {"1w", 7 * day, false}, + {"7D", 7 * day, false}, + {"2W", 14 * day, false}, + // Hour precision, no longer floored to days. + {"48h", 48 * time.Hour, false}, + {"168h", 7 * day, false}, + {"36h", 36 * time.Hour, false}, + {"12h", 12 * time.Hour, false}, + {"36H", 36 * time.Hour, false}, + {"90m", 90 * time.Minute, false}, + {"1h30m", 90 * time.Minute, false}, + // Refusals. {"", 0, true}, {"0", 0, true}, {"-1", 0, true}, {"0d", 0, true}, + {"0h", 0, true}, + {"-36h", 0, true}, {"abc", 0, true}, {"7x", 0, true}, + {"d", 0, true}, + // Out of range must be refused, never wrapped: 213504d used to + // overflow to 25m26s and select almost every closed row. + {"213504d", 0, true}, + {"213504", 0, true}, + {"30501w", 0, true}, + {"106752d", 0, true}, + {"9223372036854775807d", 0, true}, + {"99999999999h", 0, true}, + // The largest representable day count still parses. + {"106751d", 106751 * day, false}, } for _, tt := range tests { t.Run(tt.input, func(t *testing.T) { - got, err := parseHumanDuration(tt.input) + got, err := parseOlderThan(tt.input) if (err != nil) != tt.wantErr { - t.Errorf("parseHumanDuration(%q) error = %v, wantErr %v", tt.input, err, tt.wantErr) - return + t.Fatalf("parseOlderThan(%q) error = %v, wantErr %v", tt.input, err, tt.wantErr) } if got != tt.want { - t.Errorf("parseHumanDuration(%q) = %d, want %d", tt.input, got, tt.want) + t.Errorf("parseOlderThan(%q) = %s, want %s", tt.input, got, tt.want) } }) } diff --git a/cmd/bd/purge_wisps_plane_integration_test.go b/cmd/bd/purge_wisps_plane_integration_test.go new file mode 100644 index 000000000..c7147f3ee --- /dev/null +++ b/cmd/bd/purge_wisps_plane_integration_test.go @@ -0,0 +1,207 @@ +//go:build cgo + +package main + +import ( + "encoding/json" + "os" + "os/exec" + "strings" + "testing" + "time" +) + +// purgeScenarioRunner drives one storage mode for the purge retention +// scenario. run never fatals: several steps assert a refusal. +type purgeScenarioRunner struct { + mode string + run func(t *testing.T, args ...string) (stdout, stderr string, err error) +} + +func (r purgeScenarioRunner) must(t *testing.T, args ...string) string { + t.Helper() + stdout, stderr, err := r.run(t, args...) + if err != nil { + t.Fatalf("%s: bd %s failed: %v\nstdout:\n%s\nstderr:\n%s", + r.mode, strings.Join(args, " "), err, stdout, stderr) + } + return stdout +} + +func (r purgeScenarioRunner) create(t *testing.T, title string, args ...string) string { + t.Helper() + id := strings.TrimSpace(r.must(t, append([]string{"create", title, "--silent"}, args...)...)) + if id == "" { + t.Fatalf("%s: bd create %q printed no id", r.mode, title) + } + return id +} + +func (r purgeScenarioRunner) exists(t *testing.T, id string) bool { + t.Helper() + _, _, err := r.run(t, "show", id) + return err == nil +} + +// purgeJSON runs a purge that must succeed and decodes its --json object. +func (r purgeScenarioRunner) purgeJSON(t *testing.T, args ...string) map[string]any { + t.Helper() + out := r.must(t, append(append([]string{"purge"}, args...), "--json")...) + start := strings.Index(out, "{") + if start < 0 { + t.Fatalf("%s: bd purge %s --json printed no object:\n%s", r.mode, strings.Join(args, " "), out) + } + var result map[string]any + if err := json.Unmarshal([]byte(out[start:]), &result); err != nil { + t.Fatalf("%s: parse purge JSON: %v\n%s", r.mode, err, out) + } + return result +} + +func jsonCount(result map[string]any, key string) int { + n, _ := result[key].(float64) + return int(n) +} + +// runPurgeRetentionScenario is the retention sweep an orchestrator runs over +// the wisps plane — `bd purge --wisps-plane --older-than --force` — and +// the three things it has to get right: +// +// - the wisps PLANE is the selection, so a closed --no-history bead goes +// with the closed ephemeral wisp while a closed durable bead stays; +// - a closed wisp a live wisp depends on (parent-child, tracks) is kept and +// counted, on the default purge as much as the plane one; +// - --older-than has sub-day precision: rows closed seconds ago are swept +// by "1s" and kept by "36h" (which used to be floored to 24h), and an +// out-of-range value is refused rather than wrapped; +// - --limit drains a backlog in bounded runs, reporting remaining/has_more. +func runPurgeRetentionScenario(t *testing.T, r purgeScenarioRunner) { + t.Helper() + + wisp := r.create(t, "closed wisp", "--ephemeral") + noHistory := r.create(t, "closed no-history bead", "--no-history") + durable := r.create(t, "closed durable bead") + + parent := r.create(t, "closed molecule root", "--ephemeral") + child := r.create(t, "open step", "--ephemeral") + r.must(t, "dep", "add", child, parent, "--type", "parent-child") + + tracked := r.create(t, "closed tracked wisp", "--ephemeral") + convoy := r.create(t, "live convoy", "--ephemeral") + r.must(t, "dep", "add", convoy, tracked, "--type", "tracks") + r.must(t, "update", convoy, "--status", "in_progress") + + r.must(t, "close", wisp, noHistory, durable, tracked) + r.must(t, "close", parent, "--force") + + // Everything above closed within the last few seconds. closed_at is + // stored at second precision, so wait comfortably past one second before + // asking "--older-than 1s" for it. + time.Sleep(2500 * time.Millisecond) + + // The plane selector reaches durable-tier rows, so it takes the durable + // tier's require-a-filter gate. + if stdout, stderr, err := r.run(t, "purge", "--wisps-plane", "--force"); err == nil || + !strings.Contains(stdout+stderr, "requires --older-than or --pattern") { + t.Fatalf("%s: unfiltered purge --wisps-plane: err=%v\n%s%s", r.mode, err, stdout, stderr) + } + + // Hour precision: nothing here is 36 hours old. + if got := r.purgeJSON(t, "--wisps-plane", "--older-than", "36h", "--dry-run"); jsonCount(got, "purge_count") != 0 { + t.Fatalf("%s: --older-than 36h selected rows closed seconds ago: %v", r.mode, got) + } + + preview := r.purgeJSON(t, "--wisps-plane", "--older-than", "1s", "--dry-run") + if jsonCount(preview, "purge_count") != 2 || jsonCount(preview, "live_dependent_skipped") != 2 { + t.Fatalf("%s: dry run = %v; want purge_count 2 (wisp + no-history) and live_dependent_skipped 2 (root + tracked)", + r.mode, preview) + } + + // --limit drains in bounded runs and says whether more remain. + first := r.purgeJSON(t, "--wisps-plane", "--older-than", "1s", "--limit", "1", "--force") + if jsonCount(first, "purged_count") != 1 || jsonCount(first, "remaining") != 1 || first["has_more"] != true { + t.Fatalf("%s: purge --limit 1 = %v; want purged_count 1, remaining 1, has_more true", r.mode, first) + } + result := r.purgeJSON(t, "--wisps-plane", "--older-than", "1s", "--limit", "1", "--force") + if jsonCount(result, "purged_count") != 1 || jsonCount(result, "live_dependent_skipped") != 2 || + jsonCount(result, "remaining") != 0 || result["has_more"] != false { + t.Fatalf("%s: second purge --limit 1 = %v; want purged_count 1, live_dependent_skipped 2, remaining 0, has_more false", + r.mode, result) + } + if stdout, stderr, err := r.run(t, "purge", "--limit", "-1", "--force"); err == nil { + t.Fatalf("%s: purge --limit -1 succeeded:\n%s%s", r.mode, stdout, stderr) + } + if stdout, stderr, err := r.run(t, "purge", "--older-than", "213504d", "--force"); err == nil || + !strings.Contains(stdout+stderr, "out of range") { + t.Fatalf("%s: purge --older-than 213504d must refuse as out of range: err=%v\n%s%s", r.mode, err, stdout, stderr) + } + for id, want := range map[string]bool{ + wisp: false, noHistory: false, + parent: true, child: true, tracked: true, convoy: true, durable: true, + } { + if got := r.exists(t, id); got != want { + t.Errorf("%s: after purge --wisps-plane, %s exists = %v, want %v", r.mode, id, got, want) + } + } + + // The default (ephemeral-tier) purge protects live dependents too, and + // still leaves no-history beads alone. + root2 := r.create(t, "second closed root", "--ephemeral") + step2 := r.create(t, "second open step", "--ephemeral") + r.must(t, "dep", "add", step2, root2, "--type", "parent-child") + r.must(t, "close", root2, "--force") + noHistory2 := r.create(t, "second no-history bead", "--no-history") + r.must(t, "close", noHistory2) + plain := r.purgeJSON(t, "--force") + if jsonCount(plain, "live_dependent_skipped") < 1 { + t.Errorf("%s: default purge reported no live-dependent skip: %v", r.mode, plain) + } + if !r.exists(t, root2) { + t.Errorf("%s: default purge deleted %s, whose step %s is still open", r.mode, root2, step2) + } + if !r.exists(t, noHistory2) { + t.Errorf("%s: default purge deleted no-history bead %s; only --wisps-plane selects it", r.mode, noHistory2) + } +} + +func embeddedPurgeScenarioRunner(bd, dir string) purgeScenarioRunner { + return purgeScenarioRunner{ + mode: "embedded", + run: func(t *testing.T, args ...string) (string, string, error) { + t.Helper() + cmd := exec.Command(bd, args...) + cmd.Dir = dir + cmd.Env = bdEnv(dir) + stdout, stderr, err := runCommandBuffers(t, cmd) + return stdout.String(), stderr.String(), err + }, + } +} + +func proxiedPurgeScenarioRunner(bd string, p proxiedProject) purgeScenarioRunner { + return purgeScenarioRunner{ + mode: "proxied", + run: func(t *testing.T, args ...string) (string, string, error) { + t.Helper() + return bdProxiedRunBuffers(t, bd, p.dir, args...) + }, + } +} + +func TestEmbeddedPurgeWispsPlaneRetention(t *testing.T) { + if os.Getenv("BEADS_TEST_EMBEDDED_DOLT") != "1" { + t.Skip("set BEADS_TEST_EMBEDDED_DOLT=1 to run embedded dolt integration tests") + } + t.Parallel() + bd := buildEmbeddedBD(t) + dir, _, _ := bdInit(t, bd, "--prefix", "pwp") + runPurgeRetentionScenario(t, embeddedPurgeScenarioRunner(bd, dir)) +} + +func TestProxiedServerPurgeWispsPlaneRetention(t *testing.T) { + requireSharedProxiedServer(t) + t.Parallel() + bd := buildEmbeddedBD(t) + p := newSharedProxiedProject(t, bd, "pwx") + runPurgeRetentionScenario(t, proxiedPurgeScenarioRunner(bd, p)) +} diff --git a/cmd/bd/purge_wisps_plane_managed_local_test.go b/cmd/bd/purge_wisps_plane_managed_local_test.go new file mode 100644 index 000000000..0a4d6a741 --- /dev/null +++ b/cmd/bd/purge_wisps_plane_managed_local_test.go @@ -0,0 +1,19 @@ +//go:build cgo && unix + +package main + +import ( + "testing" + "time" +) + +// TestManagedLocalProxiedPurgeWispsPlaneRetention runs the purge retention +// scenario on the managed-local proxied topology, the default for a proxied +// workspace bd starts the Dolt server for. Named TestManagedLocalProxied* so +// the proxied-local smoke lane runs it. +func TestManagedLocalProxiedPurgeWispsPlaneRetention(t *testing.T) { + requireManagedLocalProxiedEnv(t) + bd := buildEmbeddedBD(t) + p := bdManagedLocalInit(t, bd, "pwl", 5*time.Minute) + runPurgeRetentionScenario(t, proxiedPurgeScenarioRunner(bd, p)) +} diff --git a/internal/storage/dolt/sweeper_contract_test.go b/internal/storage/dolt/sweeper_contract_test.go index 9c025b9b0..d01d2e96b 100644 --- a/internal/storage/dolt/sweeper_contract_test.go +++ b/internal/storage/dolt/sweeper_contract_test.go @@ -35,6 +35,21 @@ func TestSweeperContract(t *testing.T) { t.Run("TreatsALegacyTypedWispAsEphemeralTier", func(t *testing.T) { conformance.RunSweeperTreatsALegacyTypedWispAsEphemeralTier(t, ctx, fixture) }) + t.Run("WispsPlaneClearsTheWholeWispsTable", func(t *testing.T) { + conformance.RunSweeperWispsPlaneClearsTheWholeWispsTable(t, ctx, fixture) + }) + t.Run("WispsPlaneRequiresAFilter", func(t *testing.T) { + conformance.RunSweeperWispsPlaneRequiresAFilter(t, ctx, fixture) + }) + t.Run("ProtectsLiveDependents", func(t *testing.T) { + conformance.RunSweeperProtectsLiveDependents(t, ctx, fixture) + }) + t.Run("ProtectsLiveDependentsAcrossPlanes", func(t *testing.T) { + conformance.RunSweeperProtectsLiveDependentsAcrossPlanes(t, ctx, fixture) + }) + t.Run("LimitTakesTheOldestClosedFirst", func(t *testing.T) { + conformance.RunSweeperLimitTakesTheOldestClosedFirst(t, ctx, fixture) + }) t.Run("LeavesNoHistoryBeadsToTheDurableTier", func(t *testing.T) { conformance.RunSweeperLeavesNoHistoryBeadsToTheDurableTier(t, ctx, fixture) }) @@ -98,6 +113,16 @@ func newDoltSweeperFixture(t *testing.T, prefix string) (conformance.SweeperFixt } return nil }, + AddDependencies: func(ctx context.Context, req issueops.AddDependenciesRequest) error { + // Through the DependencyEditor ROLE, which routes each edge to its + // source plane's dependency table itself. + editor, err := store.DependencyEditor() + if err != nil { + return err + } + _, err = editor.AddDependencies(ctx, req) + return err + }, AddComment: func(ctx context.Context, issueID, author, text string) error { // Through the Commenter ROLE, which resolves the plane itself, so // the case can cite from a wisp's comment without knowing how this diff --git a/internal/storage/domain/db/issue.go b/internal/storage/domain/db/issue.go index 61e383520..c490a29b6 100644 --- a/internal/storage/domain/db/issue.go +++ b/internal/storage/domain/db/issue.go @@ -944,6 +944,13 @@ func (r *issueSQLRepositoryImpl) SearchAcrossIssuesAndWispsWithCounts(ctx contex return r.searchAcrossIssuesAndWispsWithCounts(ctx, query, filter) } +// SearchWispsPlane runs the shared wisps-plane search body on this +// repository's transaction, so the unit-of-work provider and the two store +// backends read the plane through one function. +func (r *issueSQLRepositoryImpl) SearchWispsPlane(ctx context.Context, query string, filter types.IssueFilter) ([]*types.Issue, error) { + return issueops.SearchWispsPlaneInTx(ctx, r.runner, query, filter) +} + func (r *issueSQLRepositoryImpl) SearchIssueIDs(ctx context.Context, query string, filter types.IssueFilter) ([]string, error) { return issueops.SearchIssueIDsInTx(ctx, r.runner, query, filter) } diff --git a/internal/storage/domain/issue.go b/internal/storage/domain/issue.go index cadb0fd22..a8137338a 100644 --- a/internal/storage/domain/issue.go +++ b/internal/storage/domain/issue.go @@ -72,6 +72,10 @@ type IssueSQLRepository interface { SearchAcrossIssuesAndWisps(ctx context.Context, query string, filter types.IssueFilter) (SearchPage, error) SearchAcrossIssuesAndWispsWithCounts(ctx context.Context, query string, filter types.IssueFilter) (SearchCountsPage, error) SearchIssueIDs(ctx context.Context, query string, filter types.IssueFilter) ([]string, error) + // SearchWispsPlane searches the wisps table ALONE, whatever each row's + // ephemeral, no_history or wisp_type values, and never the issues table + // (issueops.SearchWispsPlaneInTx). + SearchWispsPlane(ctx context.Context, query string, filter types.IssueFilter) ([]*types.Issue, error) GetReadyWork(ctx context.Context, filter types.WorkFilter) (SearchPage, error) GetReadyWorkWithCounts(ctx context.Context, filter types.WorkFilter) (SearchCountsPage, error) GetDescendants(ctx context.Context, rootID string, filter types.IssueFilter) ([]*types.Issue, error) @@ -308,6 +312,10 @@ type IssueUseCase interface { SearchIssues(ctx context.Context, query string, filter types.IssueFilter) (SearchPage, error) SearchIssuesWithCounts(ctx context.Context, query string, filter types.IssueFilter) (SearchCountsPage, error) SearchIssueIDs(ctx context.Context, query string, filter types.IssueFilter) ([]string, error) + // SearchWispsPlane searches the wisps table ALONE, whatever each row's + // ephemeral, no_history or wisp_type values, and never the issues table + // (issueops.SearchWispsPlaneInTx). + SearchWispsPlane(ctx context.Context, query string, filter types.IssueFilter) ([]*types.Issue, error) GetReadyWork(ctx context.Context, filter types.WorkFilter) (SearchPage, error) GetReadyWorkWithCounts(ctx context.Context, filter types.WorkFilter) (SearchCountsPage, error) GetDescendants(ctx context.Context, rootID string, filter types.IssueFilter) ([]*types.Issue, error) @@ -843,6 +851,14 @@ func (u *issueUseCaseImpl) SearchIssues(ctx context.Context, query string, filte return out, nil } +func (u *issueUseCaseImpl) SearchWispsPlane(ctx context.Context, query string, filter types.IssueFilter) ([]*types.Issue, error) { + out, err := u.issueRepo.SearchWispsPlane(ctx, query, filter) + if err != nil { + return nil, fmt.Errorf("SearchWispsPlane: %w", err) + } + return out, nil +} + func (u *issueUseCaseImpl) SearchIssueIDs(ctx context.Context, query string, filter types.IssueFilter) ([]string, error) { out, err := u.issueRepo.SearchIssueIDs(ctx, query, filter) if err != nil { diff --git a/internal/storage/embeddeddolt/sweeper_contract_test.go b/internal/storage/embeddeddolt/sweeper_contract_test.go index 2d221219c..7a2e550e0 100644 --- a/internal/storage/embeddeddolt/sweeper_contract_test.go +++ b/internal/storage/embeddeddolt/sweeper_contract_test.go @@ -40,6 +40,21 @@ func TestSweeperContract(t *testing.T) { t.Run("TreatsALegacyTypedWispAsEphemeralTier", func(t *testing.T) { conformance.RunSweeperTreatsALegacyTypedWispAsEphemeralTier(t, ctx, fixture) }) + t.Run("WispsPlaneClearsTheWholeWispsTable", func(t *testing.T) { + conformance.RunSweeperWispsPlaneClearsTheWholeWispsTable(t, ctx, fixture) + }) + t.Run("WispsPlaneRequiresAFilter", func(t *testing.T) { + conformance.RunSweeperWispsPlaneRequiresAFilter(t, ctx, fixture) + }) + t.Run("ProtectsLiveDependents", func(t *testing.T) { + conformance.RunSweeperProtectsLiveDependents(t, ctx, fixture) + }) + t.Run("ProtectsLiveDependentsAcrossPlanes", func(t *testing.T) { + conformance.RunSweeperProtectsLiveDependentsAcrossPlanes(t, ctx, fixture) + }) + t.Run("LimitTakesTheOldestClosedFirst", func(t *testing.T) { + conformance.RunSweeperLimitTakesTheOldestClosedFirst(t, ctx, fixture) + }) t.Run("LeavesNoHistoryBeadsToTheDurableTier", func(t *testing.T) { conformance.RunSweeperLeavesNoHistoryBeadsToTheDurableTier(t, ctx, fixture) }) @@ -99,6 +114,16 @@ func newEmbeddedSweeperFixture(t *testing.T, te *testEnv, prefix string) conform } return nil }, + AddDependencies: func(ctx context.Context, req issueops.AddDependenciesRequest) error { + // Through the DependencyEditor ROLE, which routes each edge to its + // source plane's dependency table itself. + editor, err := te.store.DependencyEditor() + if err != nil { + return err + } + _, err = editor.AddDependencies(ctx, req) + return err + }, AddComment: func(ctx context.Context, issueID, author, text string) error { // Through the Commenter ROLE, which resolves the plane itself, so // the case can cite from a wisp's comment without knowing how this diff --git a/internal/storage/issueops/search.go b/internal/storage/issueops/search.go index bd19def52..df0ff5b95 100644 --- a/internal/storage/issueops/search.go +++ b/internal/storage/issueops/search.go @@ -32,6 +32,35 @@ func SearchIssueIDsInTx(ctx context.Context, tx DBTX, query string, filter types return searchInTx(ctx, tx, query, filter, idProjection) } +// SearchWispsPlaneInTx searches the wisps plane ALONE: every row stored in +// the wisps table, whatever its ephemeral, no_history or wisp_type values, and +// never the issues table. +// +// It is not SearchIssuesInTx with Ephemeral=true. That filter adds an +// "ephemeral = 1" clause, which drops the no-history rows stored beside the +// wisps, and it falls back to the issues table when the wisps plane is empty. +// This answers "what is in the wisps table" — the unit +// issueops.SweepWispsPlane selects by — and a database with no wisps table +// answers with nothing. +func SearchWispsPlaneInTx(ctx context.Context, tx DBTX, query string, filter types.IssueFilter) ([]*types.Issue, error) { + proj := issueProjection + if filter.Lite { + proj = issueLiteProjection + } + results, err := searchTableInTxT(ctx, tx, query, filter, WispsFilterTables, proj) + if err != nil { + if missingOptionalWispTable(err) { + return nil, nil + } + return nil, fmt.Errorf("search wisps plane: %w", err) + } + results = trimToSearchLimit(results, filter.Limit) + if err := EnforceMaxRowsCap(len(results), filter.MaxRows, filter.MaxRowsSource); err != nil { + return nil, err + } + return results, nil +} + // searchProjection describes how to project, scan, and dedup search results. // Adding a narrow-projection variant means adding a new projection literal — // not a parallel top-level function or wisp-merge wrapper, which is how the diff --git a/internal/storage/issueops/sweep.go b/internal/storage/issueops/sweep.go index a609579e6..e27a3b671 100644 --- a/internal/storage/issueops/sweep.go +++ b/internal/storage/issueops/sweep.go @@ -31,7 +31,11 @@ import ( func SweepInTx(ctx context.Context, tx *sql.Tx, req publicops.SweepRequest) (publicops.SweepResult, error) { result := publicops.SweepResult{DryRun: req.DryRun} - candidates, err := SearchIssuesInTx(ctx, tx, "", workapi.BuildSweepCandidateFilter(req)) + search := SearchIssuesInTx + if workapi.SweepSearchesWispsPlaneOnly(req) { + search = SearchWispsPlaneInTx + } + candidates, err := search(ctx, tx, "", workapi.BuildSweepCandidateFilter(req)) if err != nil { return publicops.SweepResult{}, fmt.Errorf("listing sweep candidates: %w", err) } @@ -39,6 +43,14 @@ func SweepInTx(ctx context.Context, tx *sql.Tx, req publicops.SweepRequest) (pub kept, skips := workapi.FilterSweepCandidates(candidates, req.IDPattern, req.ClosedBefore) result.Skipped = skips + if req.ProtectLiveDependents { + protected, err := sweepLiveDependentsInTx(ctx, tx, kept) + if err != nil { + return publicops.SweepResult{}, err + } + kept, result.Skipped.LiveDependent = workapi.PartitionSweepLiveDependents(kept, protected) + } + if req.ProtectReferenced { referenced, err := sweepReferencedInTx(ctx, tx, kept) if err != nil { @@ -49,6 +61,8 @@ func SweepInTx(ctx context.Context, tx *sql.Tx, req publicops.SweepRequest) (pub result.Skipped.Referenced = count } + kept, result.Remaining = workapi.LimitSweepCandidates(kept, req.Limit) + if len(kept) == 0 { return result, nil } @@ -73,6 +87,46 @@ func SweepInTx(ctx context.Context, tx *sql.Tx, req publicops.SweepRequest) (pub return result, nil } +// sweepLiveDependentsInTx returns which of the candidates a live row depends +// on through a protecting edge (issueops.SweepRequest.ProtectLiveDependents), +// on the SAME transaction the candidates came off. Edges are read from both +// dependency tables, and the sources' statuses from both planes. +// +// Reading the custom statuses is required for the reason sweepReferencedInTx +// gives: a missed custom active status would under-protect. +func sweepLiveDependentsInTx(ctx context.Context, tx *sql.Tx, candidates []*types.Issue) (map[string]bool, error) { + if len(candidates) == 0 { + return nil, nil + } + ids := make([]string, len(candidates)) + for i, issue := range candidates { + ids[i] = issue.ID + } + incoming, err := GetDependentRecordsForIssuesInTx(ctx, tx, ids) + if err != nil { + return nil, fmt.Errorf("reading dependents for live-dependent protection: %w", err) + } + sources := workapi.SweepLiveDependentSources(incoming) + if len(sources) == 0 { + return nil, nil + } + custom, err := ResolveCustomStatusesDetailedInTx(ctx, tx) + if err != nil { + return nil, fmt.Errorf("reading custom statuses for live-dependent protection: %w", err) + } + liveRows, err := SearchIssuesInTx(ctx, tx, "", workapi.BuildSweepLiveDependentScanFilter(sources, custom)) + if err != nil { + return nil, fmt.Errorf("reading dependent statuses for live-dependent protection: %w", err) + } + live := make(map[string]bool, len(liveRows)) + for _, issue := range liveRows { + if issue != nil { + live[issue.ID] = true + } + } + return workapi.SweepLiveDependentTargets(incoming, live), nil +} + // sweepReferencedInTx returns which of the candidates are cited by a row that // is not done, reading the not-done set and its comments on the SAME // transaction the candidates came off. diff --git a/internal/storage/uow/notifying_parity_test.go b/internal/storage/uow/notifying_parity_test.go index 4e527c71d..5bccc063e 100644 --- a/internal/storage/uow/notifying_parity_test.go +++ b/internal/storage/uow/notifying_parity_test.go @@ -159,6 +159,7 @@ func TestRecordingIssueUseCaseCoversItsSurface(t *testing.T) { "PreviewDeleteWisp": reads, "SearchIssueIDs": reads, "SearchIssues": reads, + "SearchWispsPlane": reads, "SearchIssuesWithCounts": reads, "DeleteIssue": deletes, "DeleteIssues": deletes, diff --git a/internal/storage/uow/sweeper.go b/internal/storage/uow/sweeper.go index 3ce242c43..415833178 100644 --- a/internal/storage/uow/sweeper.go +++ b/internal/storage/uow/sweeper.go @@ -4,6 +4,7 @@ import ( "context" "fmt" + "github.com/steveyegge/beads/internal/storage/dberrors" "github.com/steveyegge/beads/internal/storage/domain" "github.com/steveyegge/beads/internal/types" "github.com/steveyegge/beads/internal/workapi" @@ -76,14 +77,22 @@ func (s *sweeper) Sweep(ctx context.Context, req publicops.SweepRequest) (public func sweepInUOW(ctx context.Context, uw UnitOfWork, req publicops.SweepRequest) (publicops.SweepResult, error) { result := publicops.SweepResult{DryRun: req.DryRun} - page, err := uw.IssueUseCase().SearchIssues(ctx, "", workapi.BuildSweepCandidateFilter(req)) + candidates, err := sweepCandidatesInUOW(ctx, uw, req) if err != nil { return publicops.SweepResult{}, fmt.Errorf("listing sweep candidates: %w", err) } - kept, skips := workapi.FilterSweepCandidates(page.Items, req.IDPattern, req.ClosedBefore) + kept, skips := workapi.FilterSweepCandidates(candidates, req.IDPattern, req.ClosedBefore) result.Skipped = skips + if req.ProtectLiveDependents { + protected, err := sweepLiveDependentsInUOW(ctx, uw, kept) + if err != nil { + return publicops.SweepResult{}, err + } + kept, result.Skipped.LiveDependent = workapi.PartitionSweepLiveDependents(kept, protected) + } + if req.ProtectReferenced { referenced, err := sweepReferencedInUOW(ctx, uw, kept) if err != nil { @@ -94,6 +103,8 @@ func sweepInUOW(ctx context.Context, uw UnitOfWork, req publicops.SweepRequest) result.Skipped.Referenced = count } + kept, result.Remaining = workapi.LimitSweepCandidates(kept, req.Limit) + if len(kept) == 0 { return result, nil } @@ -116,6 +127,73 @@ func sweepInUOW(ctx context.Context, uw UnitOfWork, req publicops.SweepRequest) return result, nil } +// sweepCandidatesInUOW reads the request's candidate rows: the wisps table +// alone for the wisps-plane tier, a tier query across both planes otherwise. +func sweepCandidatesInUOW(ctx context.Context, uw UnitOfWork, req publicops.SweepRequest) ([]*types.Issue, error) { + filter := workapi.BuildSweepCandidateFilter(req) + if workapi.SweepSearchesWispsPlaneOnly(req) { + return uw.IssueUseCase().SearchWispsPlane(ctx, "", filter) + } + page, err := uw.IssueUseCase().SearchIssues(ctx, "", filter) + if err != nil { + return nil, err + } + return page.Items, nil +} + +// sweepLiveDependentsInUOW returns which of the candidates a live row depends +// on through a protecting edge, on the unit of work the candidates came off. +// +// Edges are read from BOTH dependency tables: a wisp's edges live in +// wisp_dependencies and an issue's in dependencies, and the source of an edge +// pointing at a candidate can be either. The sources' statuses are then read +// across both planes by SearchIssues. +func sweepLiveDependentsInUOW(ctx context.Context, uw UnitOfWork, candidates []*types.Issue) (map[string]bool, error) { + if len(candidates) == 0 { + return nil, nil + } + ids := make([]string, len(candidates)) + for i, issue := range candidates { + ids[i] = issue.ID + } + edges := domain.DepListFilter{Types: publicops.SweepLiveDependentEdgeTypes, Direction: domain.DepDirectionIn} + durable, err := uw.DependencyUseCase().ListByIssueIDs(ctx, ids, edges) + if err != nil { + return nil, fmt.Errorf("reading dependents for live-dependent protection: %w", err) + } + wisp, err := uw.DependencyUseCase().ListByWispIDs(ctx, ids, edges) + if err != nil && !dberrors.IsTableNotExist(err) { + return nil, fmt.Errorf("reading wisp dependents for live-dependent protection: %w", err) + } + incoming := make(map[string][]*types.Dependency, len(durable.Incoming)+len(wisp.Incoming)) + for target, deps := range durable.Incoming { + incoming[target] = append(incoming[target], deps...) + } + for target, deps := range wisp.Incoming { + incoming[target] = append(incoming[target], deps...) + } + + sources := workapi.SweepLiveDependentSources(incoming) + if len(sources) == 0 { + return nil, nil + } + custom, err := uw.ConfigUseCase().GetCustomStatuses(ctx) + if err != nil { + return nil, fmt.Errorf("reading custom statuses for live-dependent protection: %w", err) + } + page, err := uw.IssueUseCase().SearchIssues(ctx, "", workapi.BuildSweepLiveDependentScanFilter(sources, custom)) + if err != nil { + return nil, fmt.Errorf("reading dependent statuses for live-dependent protection: %w", err) + } + live := make(map[string]bool, len(page.Items)) + for _, issue := range page.Items { + if issue != nil { + live[issue.ID] = true + } + } + return workapi.SweepLiveDependentTargets(incoming, live), nil +} + // sweepReferencedInUOW returns which of the candidates a not-done row cites, // reading the not-done set and its comments on the same unit of work the // candidates came off. Comments are read for the whole not-done set in ONE diff --git a/internal/storage/uow/sweeper_contract_test.go b/internal/storage/uow/sweeper_contract_test.go index 4ccec5a9a..e054b5226 100644 --- a/internal/storage/uow/sweeper_contract_test.go +++ b/internal/storage/uow/sweeper_contract_test.go @@ -35,6 +35,21 @@ func TestSweeperContract(t *testing.T) { t.Run("TreatsALegacyTypedWispAsEphemeralTier", func(t *testing.T) { conformance.RunSweeperTreatsALegacyTypedWispAsEphemeralTier(t, ctx, fixture) }) + t.Run("WispsPlaneClearsTheWholeWispsTable", func(t *testing.T) { + conformance.RunSweeperWispsPlaneClearsTheWholeWispsTable(t, ctx, fixture) + }) + t.Run("WispsPlaneRequiresAFilter", func(t *testing.T) { + conformance.RunSweeperWispsPlaneRequiresAFilter(t, ctx, fixture) + }) + t.Run("ProtectsLiveDependents", func(t *testing.T) { + conformance.RunSweeperProtectsLiveDependents(t, ctx, fixture) + }) + t.Run("ProtectsLiveDependentsAcrossPlanes", func(t *testing.T) { + conformance.RunSweeperProtectsLiveDependentsAcrossPlanes(t, ctx, fixture) + }) + t.Run("LimitTakesTheOldestClosedFirst", func(t *testing.T) { + conformance.RunSweeperLimitTakesTheOldestClosedFirst(t, ctx, fixture) + }) t.Run("LeavesNoHistoryBeadsToTheDurableTier", func(t *testing.T) { conformance.RunSweeperLeavesNoHistoryBeadsToTheDurableTier(t, ctx, fixture) }) @@ -98,6 +113,20 @@ func newUOWSweeperFixture(t *testing.T, ctx context.Context, prefix string) conf return "seed legacy sweep rows", nil }) }, + AddDependencies: func(ctx context.Context, req issueops.AddDependenciesRequest) error { + // Through the DependencyEditor ROLE, which routes each edge to its + // source plane's dependency table itself. + ds, ok := provider.(DependencyEditorSource) + if !ok { + return fmt.Errorf("provider %T does not offer the DependencyEditor accessor", provider) + } + editor, err := ds.DependencyEditor() + if err != nil { + return err + } + _, err = editor.AddDependencies(ctx, req) + return err + }, AddComment: func(ctx context.Context, issueID, author, text string) error { // Through the Commenter ROLE, which resolves the plane itself, so // the case can cite from a wisp's comment without knowing how this diff --git a/internal/workapi/sweep.go b/internal/workapi/sweep.go index 3466558e2..24af4ec50 100644 --- a/internal/workapi/sweep.go +++ b/internal/workapi/sweep.go @@ -29,13 +29,13 @@ import ( // by calling the role. See issueops.SweepRequest. func ValidateSweepRequest(in issueops.SweepRequest) error { switch in.Tier { - case issueops.SweepEphemeral, issueops.SweepDurable: + case issueops.SweepEphemeral, issueops.SweepDurable, issueops.SweepWispsPlane: case "": - return fmt.Errorf("%w: sweep requires a tier (%q or %q), and has no default", - issueops.ErrValidation, issueops.SweepEphemeral, issueops.SweepDurable) + return fmt.Errorf("%w: sweep requires a tier (%q, %q or %q), and has no default", + issueops.ErrValidation, issueops.SweepEphemeral, issueops.SweepDurable, issueops.SweepWispsPlane) default: - return fmt.Errorf("%w: %q is not a sweep tier; use %q or %q", - issueops.ErrValidation, in.Tier, issueops.SweepEphemeral, issueops.SweepDurable) + return fmt.Errorf("%w: %q is not a sweep tier; use %q, %q or %q", + issueops.ErrValidation, in.Tier, issueops.SweepEphemeral, issueops.SweepDurable, issueops.SweepWispsPlane) } if in.IDPattern != "" { // filepath.Match reports a malformed pattern on any subject, so one @@ -47,14 +47,32 @@ func ValidateSweepRequest(in issueops.SweepRequest) error { issueops.ErrValidation, in.IDPattern, err) } } - if in.Tier == issueops.SweepDurable && in.ClosedBefore == nil && in.IDPattern == "" { - return fmt.Errorf("%w: a durable sweep requires a closed-before cutoff or an id pattern; "+ + if in.Limit < 0 { + return fmt.Errorf("%w: sweep limit %d is negative; use 0 for no limit", issueops.ErrValidation, in.Limit) + } + if sweepRequiresFilter(in.Tier) && in.ClosedBefore == nil && in.IDPattern == "" { + return fmt.Errorf("%w: a %s sweep requires a closed-before cutoff or an id pattern; "+ "pass the pattern \"*\" to sweep every closed issue deliberately", - issueops.ErrValidation) + issueops.ErrValidation, in.Tier) } return nil } +// sweepRequiresFilter reports whether a tier reaches durable-tier rows and so +// carries the require-a-filter gate. The wisps plane does: it holds no-history +// beads (issueops.SweepWispsPlane). +func sweepRequiresFilter(tier issueops.SweepTier) bool { + return tier == issueops.SweepDurable || tier == issueops.SweepWispsPlane +} + +// SweepSearchesWispsPlaneOnly reports whether a request's candidates come from +// the wisps table alone (issueops.SweepWispsPlane) rather than from a tier +// query merged across both planes. Every Sweeper implementation branches on +// this one answer, so the plane a tier reads is decided in one place. +func SweepSearchesWispsPlaneOnly(in issueops.SweepRequest) bool { + return in.Tier == issueops.SweepWispsPlane +} + // BuildSweepCandidateFilter turns a sweep request into the storage-level // filter that selects its CANDIDATES: the closed rows of one tier, bounded by // the cutoff. @@ -66,8 +84,21 @@ func ValidateSweepRequest(in issueops.SweepRequest) error { // // Call ValidateSweepRequest first; this builder assumes a validated request // and does not re-refuse one. +// +// For SweepWispsPlane the filter carries NO tier constraint: the plane is the +// selection, and the implementation applies it by searching the wisps table +// alone (SweepSearchesWispsPlaneOnly). Every closed row there is a candidate, +// whatever its ephemeral, no_history or wisp_type values. func BuildSweepCandidateFilter(in issueops.SweepRequest) types.IssueFilter { closed := types.StatusClosed + if SweepSearchesWispsPlaneOnly(in) { + filter := types.IssueFilter{Status: &closed} + if in.ClosedBefore != nil { + cutoff := *in.ClosedBefore + filter.ClosedBefore = &cutoff + } + return filter + } ephemeral := in.Tier == issueops.SweepEphemeral // EphemeralTier, not Ephemeral: the tier owns typed wisps whether or not // their minting set the ephemeral flag (older creators set wisp_type but @@ -271,3 +302,91 @@ func PartitionSweepReferenced(candidates []*types.Issue, referenced map[string]b } return kept, referencedCount, sample } + +// BuildSweepLiveDependentScanFilter selects which of a candidate set's +// dependents are LIVE for issueops.SweepRequest.ProtectLiveDependents: the +// named sources, restricted to the not-done statuses. It is the same status +// vocabulary the reference scan asks, so "live" means one thing in a sweep. +func BuildSweepLiveDependentScanFilter(dependentIDs []string, custom []types.CustomStatus) types.IssueFilter { + return types.IssueFilter{IDs: dependentIDs, Statuses: NotDoneStatusesForSweep(custom)} +} + +// SweepLiveDependentSources returns the ids of the rows that depend on a +// candidate through a protecting edge (issueops.SweepLiveDependentEdgeTypes), +// given the edges pointing at the candidates keyed by target. The result is +// sorted and de-duplicated, so the follow-up status read is deterministic. +func SweepLiveDependentSources(incoming map[string][]*types.Dependency) []string { + seen := make(map[string]bool) + for _, deps := range incoming { + for _, dep := range deps { + if dep != nil && isSweepProtectingEdge(dep.Type) { + seen[dep.IssueID] = true + } + } + } + ids := make([]string, 0, len(seen)) + for id := range seen { + ids = append(ids, id) + } + sort.Strings(ids) + return ids +} + +// SweepLiveDependentTargets returns the candidates a live row depends on: the +// targets of protecting edges whose source is in live. +func SweepLiveDependentTargets(incoming map[string][]*types.Dependency, live map[string]bool) map[string]bool { + protected := make(map[string]bool) + for target, deps := range incoming { + for _, dep := range deps { + if dep != nil && isSweepProtectingEdge(dep.Type) && live[dep.IssueID] { + protected[target] = true + break + } + } + } + return protected +} + +func isSweepProtectingEdge(t types.DependencyType) bool { + for _, protecting := range issueops.SweepLiveDependentEdgeTypes { + if t == protecting { + return true + } + } + return false +} + +// PartitionSweepLiveDependents splits candidates into the ones no live row +// depends on and a count of the ones some live row does, preserving candidate +// order. +func PartitionSweepLiveDependents(candidates []*types.Issue, protected map[string]bool) (kept []*types.Issue, protectedCount int) { + kept = make([]*types.Issue, 0, len(candidates)) + for _, issue := range candidates { + if protected[issue.ID] { + protectedCount++ + continue + } + kept = append(kept, issue) + } + return kept, protectedCount +} + +// LimitSweepCandidates applies issueops.SweepRequest.Limit to the candidates +// that survived every protection: the oldest-closed first (closed_at +// ascending, then id, so two runs over one snapshot pick the same rows), at +// most limit of them. It returns the kept rows and how many were left behind. +// A non-positive limit keeps everything. The input slice is not reordered. +func LimitSweepCandidates(candidates []*types.Issue, limit int) (kept []*types.Issue, remaining int) { + if limit <= 0 || len(candidates) <= limit { + return candidates, 0 + } + ordered := append([]*types.Issue(nil), candidates...) + sort.SliceStable(ordered, func(i, j int) bool { + a, b := ordered[i], ordered[j] + if a.ClosedAt != nil && b.ClosedAt != nil && !a.ClosedAt.Equal(*b.ClosedAt) { + return a.ClosedAt.Before(*b.ClosedAt) + } + return a.ID < b.ID + }) + return ordered[:limit], len(ordered) - limit +} diff --git a/internal/workapi/sweep_test.go b/internal/workapi/sweep_test.go index 9c4093cd0..af26fd9fb 100644 --- a/internal/workapi/sweep_test.go +++ b/internal/workapi/sweep_test.go @@ -353,3 +353,119 @@ func sweepCandidateIDs(issues []*types.Issue) []string { } return ids } + +func TestValidateSweepRequestGatesTheWispsPlane(t *testing.T) { + cutoff := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC) + if err := ValidateSweepRequest(issueops.SweepRequest{Tier: issueops.SweepWispsPlane}); !errors.Is(err, issueops.ErrValidation) { + t.Fatalf("unfiltered wisps-plane sweep error = %v, want ErrValidation: it reaches no-history (durable-tier) rows", err) + } + for _, req := range []issueops.SweepRequest{ + {Tier: issueops.SweepWispsPlane, ClosedBefore: &cutoff}, + {Tier: issueops.SweepWispsPlane, IDPattern: "*"}, + } { + if err := ValidateSweepRequest(req); err != nil { + t.Errorf("ValidateSweepRequest(%+v) = %v, want nil", req, err) + } + } +} + +func TestBuildSweepCandidateFilterForTheWispsPlaneHasNoTierConstraint(t *testing.T) { + cutoff := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC) + req := issueops.SweepRequest{Tier: issueops.SweepWispsPlane, ClosedBefore: &cutoff} + if !SweepSearchesWispsPlaneOnly(req) { + t.Fatal("SweepSearchesWispsPlaneOnly(wisps-plane) = false; the plane IS the selection") + } + filter := BuildSweepCandidateFilter(req) + if filter.Status == nil || *filter.Status != types.StatusClosed { + t.Fatalf("Status = %v, want closed", filter.Status) + } + if filter.EphemeralTier != nil || filter.Ephemeral != nil { + t.Fatalf("wisps-plane filter carries a flag constraint (EphemeralTier=%v, Ephemeral=%v); "+ + "that would drop no-history rows", filter.EphemeralTier, filter.Ephemeral) + } + if filter.ClosedBefore == nil || !filter.ClosedBefore.Equal(cutoff) || filter.ClosedBefore == &cutoff { + t.Fatalf("ClosedBefore = %v, want a copy of %v", filter.ClosedBefore, cutoff) + } + for _, tier := range []issueops.SweepTier{issueops.SweepEphemeral, issueops.SweepDurable} { + if SweepSearchesWispsPlaneOnly(issueops.SweepRequest{Tier: tier}) { + t.Errorf("SweepSearchesWispsPlaneOnly(%s) = true; tier queries merge both planes", tier) + } + } +} + +func TestSweepLiveDependentsCountOnlyProtectingEdgesFromLiveSources(t *testing.T) { + dep := func(source, target string, typ types.DependencyType) *types.Dependency { + return &types.Dependency{IssueID: source, DependsOnID: target, Type: typ} + } + incoming := map[string][]*types.Dependency{ + "root": {dep("step-open", "root", types.DepParentChild), dep("step-done", "root", types.DepParentChild)}, + "tracked": {dep("convoy", "tracked", types.DepTracks)}, + "gate": {dep("waiter", "gate", types.DepBlocks)}, + "linked": {dep("reader", "linked", types.DepRelated)}, + "orphan": {dep("step-done", "orphan", types.DepParentChild), nil}, + } + + sources := SweepLiveDependentSources(incoming) + if want := []string{"convoy", "step-done", "step-open", "waiter"}; strings.Join(sources, ",") != strings.Join(want, ",") { + t.Fatalf("SweepLiveDependentSources = %v, want %v (sorted, protecting edges only)", sources, want) + } + + live := map[string]bool{"step-open": true, "convoy": true, "waiter": true, "reader": true} + protected := SweepLiveDependentTargets(incoming, live) + for _, id := range []string{"root", "tracked", "gate"} { + if !protected[id] { + t.Errorf("%s not protected; a live source depends on it through a protecting edge", id) + } + } + for _, id := range []string{"linked", "orphan"} { + if protected[id] { + t.Errorf("%s protected; its only live source is a non-protecting edge or none is live", id) + } + } + + candidates := []*types.Issue{{ID: "root"}, {ID: "linked"}, {ID: "tracked"}, {ID: "orphan"}} + kept, count := PartitionSweepLiveDependents(candidates, protected) + if count != 2 || len(kept) != 2 || kept[0].ID != "linked" || kept[1].ID != "orphan" { + t.Fatalf("PartitionSweepLiveDependents = %d kept %v, count %d; want [linked orphan] and 2", len(kept), kept, count) + } +} + +func TestValidateSweepRequestRefusesANegativeLimit(t *testing.T) { + err := ValidateSweepRequest(issueops.SweepRequest{Tier: issueops.SweepEphemeral, Limit: -1}) + if !errors.Is(err, issueops.ErrValidation) { + t.Fatalf("negative Limit error = %v, want ErrValidation", err) + } + if err := ValidateSweepRequest(issueops.SweepRequest{Tier: issueops.SweepEphemeral, Limit: 0}); err != nil { + t.Fatalf("zero Limit (no cap) error = %v, want nil", err) + } +} + +func TestLimitSweepCandidatesTakesTheOldestClosedFirst(t *testing.T) { + at := func(day int) *time.Time { + ts := time.Date(2026, 1, day, 0, 0, 0, 0, time.UTC) + return &ts + } + candidates := []*types.Issue{ + {ID: "c", ClosedAt: at(3)}, + {ID: "b2", ClosedAt: at(1)}, + {ID: "a", ClosedAt: at(2)}, + {ID: "b1", ClosedAt: at(1)}, + } + + kept, remaining := LimitSweepCandidates(candidates, 3) + var got []string + for _, issue := range kept { + got = append(got, issue.ID) + } + if strings.Join(got, ",") != "b1,b2,a" || remaining != 1 { + t.Fatalf("LimitSweepCandidates(3) = %v, remaining %d; want [b1 b2 a] (closed_at, then id) and 1", got, remaining) + } + if candidates[0].ID != "c" { + t.Fatal("LimitSweepCandidates reordered the caller's slice") + } + for _, limit := range []int{0, 4, 10} { + if kept, remaining := LimitSweepCandidates(candidates, limit); len(kept) != 4 || remaining != 0 { + t.Errorf("LimitSweepCandidates(%d) kept %d, remaining %d; want all 4 and 0", limit, len(kept), remaining) + } + } +} diff --git a/issueops/dependencyeditor.go b/issueops/dependencyeditor.go index 2951dd66e..f9b0683eb 100644 --- a/issueops/dependencyeditor.go +++ b/issueops/dependencyeditor.go @@ -29,6 +29,7 @@ const ( DepDiscoveredFrom = types.DepDiscoveredFrom DepRepliesTo = types.DepRepliesTo DepWaitsFor = types.DepWaitsFor + DepTracks = types.DepTracks ) // DependencyEdge is one directed edge: IssueID depends on DependsOnID, with diff --git a/issueops/sweeper.go b/issueops/sweeper.go index 46f4e528b..4e60eda0a 100644 --- a/issueops/sweeper.go +++ b/issueops/sweeper.go @@ -27,6 +27,21 @@ const ( // it — carries the require-a-filter refusal SweepRequest.ClosedBefore // describes. SweepDurable SweepTier = "durable" + // SweepWispsPlane clears CLOSED rows by STORAGE PLANE rather than by + // tier: every closed row stored in the wisps table, whatever its flags — + // ephemeral wisps and no-history beads alike. `bd purge --wisps-plane`. + // + // It is the one tier that is NOT disjoint from the other two, and that is + // the point of it. A no-history bead lives in the wisps plane but is + // durable-tier (it is not ephemeral and carries no wisp_type), so neither + // SweepEphemeral nor SweepDurable can say "the wisps table" — the unit an + // orchestrator's retention policy is written in. Conversely a legacy + // typed wisp stored in the issues table is ephemeral-tier and NOT in this + // plane. + // + // Because it reaches durable-tier rows, it carries the durable tier's + // require-a-filter gate (SweepRequest.ClosedBefore). + SweepWispsPlane SweepTier = "wisps-plane" ) // SweepRequest describes one bulk clearance of closed rows: which tier, which @@ -68,6 +83,9 @@ type SweepRequest struct { // says so with IDPattern "*", which is a deliberate keystroke rather than // an omitted one. // + // SweepWispsPlane carries the same gate, because the wisps plane holds + // no-history beads, which are durable-tier. + // // The EPHEMERAL tier carries no such gate: an unfiltered sweep of closed // wisps is the ordinary use of `bd purge`, and the tier is transient by // definition. @@ -108,6 +126,33 @@ type SweepRequest struct { // `bd purge` does not, because a wisp's citations are as transient as the // wisp. ProtectReferenced bool + // ProtectLiveDependents skips candidates that a LIVE row depends on + // through a structural edge — a closed parent whose child is still open, + // a closed bead a live convoy tracks, a closed blocker of a live bead — + // so a sweep never deletes the root, container or gate of work that is + // still in progress. + // + // WHICH EDGES are SweepLiveDependentEdgeTypes: parent-child, tracks and + // blocks, from either plane's dependency table. Other edge types (related, + // discovered-from, ...) are links rather than structure and do not protect. + // + // WHICH ROWS ARE LIVE is the same not-done set ProtectReferenced asks: the + // built-in active statuses (open, in_progress, blocked, deferred, pinned, + // hooked) plus every configured custom status whose category is not + // "done", with the same REQUIRED read of the custom statuses. + // + // Unlike ProtectReferenced it is a stored-edge lookup, not a text scan, so + // it costs one read of the edges pointing at the candidates and one of + // their sources. `bd purge` always asks for it; `bd prune` does not. + ProtectLiveDependents bool + // Limit, when positive, caps how many rows ONE sweep deletes, taking the + // OLDEST-CLOSED first (closed_at ascending, then id). The rest are left + // for the next call and counted in SweepResult.Remaining, so a caller can + // drain a large backlog in bounded transactions instead of one that grows + // with the backlog and can outlive the caller's deadline. It applies after + // every protection, so it caps rows that would actually go. Zero means no + // cap; a negative value is ErrValidation. + Limit int // DryRun reports what the sweep WOULD do and deletes nothing. The result // is otherwise the same result — the same counts, the same skips, the same // refusals — computed against the same snapshot the real sweep would have @@ -132,6 +177,9 @@ type SweepSkips struct { // 0 when that field is false; a caller that reads a 0 without having asked // for the protection has learned nothing about whether rows are cited. Referenced int + // LiveDependent counts candidates skipped by ProtectLiveDependents, under + // the same always-0-when-not-asked rule as Referenced. + LiveDependent int // NotClosed, UnknownClosedAt and ClosedAtOrAfterCutoff count candidates // the tier's own query returned and the sweep rechecked and rejected: a // status that is not closed, a closed row with no closed_at stamp at all, @@ -173,6 +221,10 @@ type SweepResult struct { Events int // Skipped reports the candidates the sweep held back and why. Skipped SweepSkips + // Remaining is how many rows the sweep WOULD also have deleted but left + // because of SweepRequest.Limit. Zero when there is no limit or the + // backlog fit; a caller loops while it is non-zero. + Remaining int // ReferencedIDs is a BOUNDED SAMPLE of the ids Skipped.Referenced counts — // at most SweepReferencedSampleLimit of them, in the order the candidate // query returned them. It is a sample and not the set: a caller that needs @@ -184,6 +236,11 @@ type SweepResult struct { ReferencedIDs []string } +// SweepLiveDependentEdgeTypes are the edge types through which a live row +// protects a closed candidate under SweepRequest.ProtectLiveDependents. It is +// published so a caller can state the same policy without copying the list. +var SweepLiveDependentEdgeTypes = []DependencyType{DepParentChild, DepTracks, DepBlocks} + // SweepReferencedSampleLimit bounds SweepResult.ReferencedIDs. It is published // so a caller can tell a truncated sample from a complete one by comparing the // slice's length against it, rather than against a number copied out of an @@ -255,17 +312,20 @@ type Sweeper interface { // // THE ORDER THE NARROWING HAPPENS IN IS PART OF THE ANSWER, because the // skip counters are counted along the way: the tier's closed rows, then - // IDPattern, then the pinned and closed_at rechecks, then the reference - // protection. A pinned row excluded by the pattern is therefore NOT - // counted in Skipped.Pinned — it was never a candidate — and the counters - // describe the set the request actually reached. + // IDPattern, then the pinned and closed_at rechecks, then the + // live-dependent protection, then the reference protection, then Limit + // (oldest-closed first). A pinned row + // excluded by the pattern is therefore NOT counted in Skipped.Pinned — it + // was never a candidate — and the counters describe the set the request + // actually reached. // // REFUSALS, all ErrValidation and all before anything is read: // // - an unset or unrecognized Tier; - // - a SweepDurable request with neither ClosedBefore nor IDPattern (see - // SweepRequest.ClosedBefore for why this lives here); - // - an IDPattern that is not a well-formed glob. + // - a SweepDurable or SweepWispsPlane request with neither ClosedBefore + // nor IDPattern (see SweepRequest.ClosedBefore for why this lives here); + // - an IDPattern that is not a well-formed glob; + // - a negative Limit. // // A REQUEST THAT MATCHES NOTHING IS A ZERO RESULT AND A NIL ERROR, not a // not-found: an empty set of closed rows is the steady state of a swept