diff --git a/.github/scripts/proxied-cmd-test-shards.txt b/.github/scripts/proxied-cmd-test-shards.txt index f29510141..a57cd48ab 100644 --- a/.github/scripts/proxied-cmd-test-shards.txt +++ b/.github/scripts/proxied-cmd-test-shards.txt @@ -5,8 +5,9 @@ # 15-shard split, bin-packed longest-processing-time-first by # estimated cost (bd-init count; per-init migration chains dominate). # Regenerate with scripts/ci/gen_proxied_shard_manifest.py after adding, -# splitting, or reweighting TestProxiedServer* functions. Newly-added -# tests not listed here hash-distribute via proxied-test-shard.sh. +# splitting, or reweighting TestProxiedServer*/TestServerMode* +# functions. Newly-added tests not listed here hash-distribute via +# proxied-test-shard.sh. # # This file is generated: notes added here are erased by the next # regeneration. Add them to the generator instead. Known blind spots diff --git a/.github/scripts/proxied-test-shard.sh b/.github/scripts/proxied-test-shard.sh index 1c23774af..0c0c3f7c6 100755 --- a/.github/scripts/proxied-test-shard.sh +++ b/.github/scripts/proxied-test-shard.sh @@ -3,13 +3,24 @@ # # Usage: proxied-test-shard.sh # -# Discovers all TestProxiedServer* top-level functions across cmd/bd/*_test.go -# (the proxied integration suites plus TestProxiedServerExternalCreate and the -# TestProxiedServerPathHelpers unit test, matching the lane's historical -# `-run '^TestProxiedServer'`). Tests listed in proxied-cmd-test-shards.txt for -# the requested shard count use that committed assignment; newly-added tests -# fall back to hash(name) % total. Runs the matching subset using the pre-built -# test binary at BEADS_TEST_CMD_BINARY (or /tmp/bd-cmd-test). +# Discovers all TestProxiedServer* and TestServerMode* top-level functions +# across cmd/bd/*_test.go (the proxied integration suites plus +# TestProxiedServerExternalCreate and the TestProxiedServerPathHelpers unit +# test, widening the lane's historical `-run '^TestProxiedServer'`). The +# TestServerMode* suites all gate on requireSharedProxiedServer and drive the +# same shared dolt sql-server, and this lane is the only one that sets +# BEADS_TEST_PROXIED_SERVER=1, so it is the only place they run: before the +# widening a top-level TestServerMode* entry point matched no discovery pattern +# here, so it skipped everywhere else and was never executed by CI at all. +# The TestProxiedServer* side is mixed and should not be generalized over: most +# gate on requireSharedProxiedServer, the rest on requireProxiedServerEnv, and +# TestProxiedServerPathHelpers (pure path helpers) has no gate and no build tag, +# so that one also runs in every ordinary `go test ./cmd/bd/` lane. +# +# Tests listed in proxied-cmd-test-shards.txt for the requested shard count use +# that committed assignment; newly-added tests fall back to hash(name) % total. +# Runs the matching subset using the pre-built test binary at +# BEADS_TEST_CMD_BINARY (or /tmp/bd-cmd-test). # # Environment: # BEADS_TEST_PROXIED_SERVER=1 required (tests skip without it) @@ -36,16 +47,17 @@ fi SHARD_INDEX=$(( SHARD_NUMBER - 1 )) MANIFEST="${BEADS_TEST_SHARD_MANIFEST:-.github/scripts/proxied-cmd-test-shards.txt}" -# Discover all top-level TestProxiedServer* functions across every cmd/bd test -# file (integration suites live in *_proxied_integration_test.go, but -# TestProxiedServerExternalCreate and TestProxiedServerPathHelpers do not, so -# match by name rather than by file glob). -ALL_TESTS=$(grep -rh '^func TestProxiedServer' cmd/bd/*_test.go \ - | sed 's/func \(TestProxiedServer[A-Za-z0-9_]*\).*/\1/' \ +# Discover all top-level TestProxiedServer*/TestServerMode* functions across +# every cmd/bd test file (integration suites live in +# *_proxied_integration_test.go, but TestProxiedServerExternalCreate, +# TestProxiedServerPathHelpers and the TestServerMode* suites do not, so match +# by name rather than by file glob). +ALL_TESTS=$(grep -rhE '^func Test(ProxiedServer|ServerMode)' cmd/bd/*_test.go \ + | sed -E 's/func (Test(ProxiedServer|ServerMode)[A-Za-z0-9_]*).*/\1/' \ | sort -u) if [ -z "$ALL_TESTS" ]; then - echo "No TestProxiedServer* functions found" >&2 + echo "No TestProxiedServer*/TestServerMode* functions found" >&2 exit 1 fi @@ -62,7 +74,7 @@ if [ -f "$MANIFEST" ]; then if [ -z "${manifest_total:-}" ]; then continue fi - if [ -n "${extra:-}" ] || ! [[ "$manifest_total" =~ ^[0-9]+$ ]] || ! [[ "$manifest_shard" =~ ^[0-9]+$ ]] || ! [[ "$test_name" =~ ^TestProxiedServer[A-Za-z0-9_]+$ ]]; then + if [ -n "${extra:-}" ] || ! [[ "$manifest_total" =~ ^[0-9]+$ ]] || ! [[ "$manifest_shard" =~ ^[0-9]+$ ]] || ! [[ "$test_name" =~ ^Test(ProxiedServer|ServerMode)[A-Za-z0-9_]+$ ]]; then echo "Invalid shard manifest line: $line" >&2 exit 1 fi diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index b8c91d791..07bdf633f 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -903,9 +903,14 @@ jobs: # Reuses the cmd/bd test binary compiled once by build-artifacts (bd-cmd-test): # the proxied tests share the package and are gated at runtime by # BEADS_TEST_PROXIED_SERVER=1. Shard assignment is committed in - # .github/scripts/proxied-cmd-test-shards.txt; newly-added TestProxiedServer* - # tests hash-distribute until listed there. Adopt the - # `TestProxiedServer` naming convention for new entry points. + # .github/scripts/proxied-cmd-test-shards.txt; newly-added + # TestProxiedServer*/TestServerMode* tests hash-distribute until listed there. + # Prefer the `TestProxiedServer` naming convention for new entry + # points; server-mode-topology suites named TestServerMode* are also + # discovered among the cmd/bd/*_test.go entry points, so neither family is + # left out of this lane. That glob is non-recursive: a container-gated suite + # added under cmd/bd// is still never selected here whatever it is + # named, so keep new proxied/server-mode entry points at the top level. test-proxied-cmd: name: Test (proxied-server cmd ${{ matrix.shard }}/15) if: github.event_name == 'push' && github.ref == 'refs/heads/main' diff --git a/.github/workflows/pr-risk.yml b/.github/workflows/pr-risk.yml index 2972c4b73..3c293aff9 100644 --- a/.github/workflows/pr-risk.yml +++ b/.github/workflows/pr-risk.yml @@ -64,7 +64,8 @@ jobs: # tests live in the same package and are gated at runtime by # BEADS_TEST_PROXIED_SERVER=1, so no separate build is needed. Shard # assignment is committed in .github/scripts/proxied-cmd-test-shards.txt; - # newly-added TestProxiedServer* tests hash-distribute until listed there. + # newly-added TestProxiedServer*/TestServerMode* tests hash-distribute until + # listed there. test-proxied-cmd: name: Test (Proxied Dolt Cmd ${{ matrix.shard }}/${{ strategy.job-total }}) needs: [detect-ci-tier, build-embedded] diff --git a/cmd/bd/BUILD.bazel b/cmd/bd/BUILD.bazel index efd498e15..69f59d1ab 100644 --- a/cmd/bd/BUILD.bazel +++ b/cmd/bd/BUILD.bazel @@ -622,6 +622,7 @@ go_test( "defer_proxied_integration_test.go", "defer_wake_embedded_test.go", "defer_wake_proxied_integration_test.go", + "defer_wake_server_test.go", "delete_embedded_test.go", "delete_input_test.go", "delete_proxied_integration_test.go", diff --git a/cmd/bd/defer_wake_server_test.go b/cmd/bd/defer_wake_server_test.go new file mode 100644 index 000000000..57d6b3749 --- /dev/null +++ b/cmd/bd/defer_wake_server_test.go @@ -0,0 +1,118 @@ +//go:build cgo + +package main + +import ( + "testing" + + "github.com/steveyegge/beads/internal/types" +) + +// TestServerModeDeferAutoWake documents the same defer/wake contract as +// TestEmbeddedDeferAutoWake (defer_wake_embedded_test.go) and +// TestProxiedServerDeferAutoWake (defer_wake_proxied_integration_test.go), but +// against a real dolt sql-server (be-vbhpf): a DATED defer is a snooze, and +// once defer_until passes, the next ready-front read on `bd ready` or +// `bd list --ready` must return the issue to open. In server mode this sweep +// (DoltStore.wakeExpiredDefers) is silently skipped today because the store is +// opened read-only purely from command classification (bd ready/bd list are +// on the read-only allowlist for the CURRENT project, GH#804) — indistinguishable, +// before this fix, from a strict --readonly or foreign-project open that must +// never sweep. +func TestServerModeDeferAutoWake(t *testing.T) { + requireSharedProxiedServer(t) + t.Parallel() + + bd := buildEmbeddedBD(t) + p := newServerModeProject(t, bd, "dws") + + t.Run("expired_dated_defer_wakes_on_ready", func(t *testing.T) { + issue := bdCreate(t, bd, p.dir, "Expired snooze (server mode)", "--type", "task") + bdDefer(t, bd, p.dir, issue.ID, "--until", "2020-01-01") + status, _ := showDeferState(t, bd, p.dir, issue.ID) + if status != "deferred" { + t.Fatalf("precondition: expected status=deferred, got %q", status) + } + + ids := wakeReadyIDs(t, bd, p.dir) + if !ids[issue.ID] { + t.Errorf("expected %s in bd ready after its defer date passed (server mode)", issue.ID) + } + status, deferUntil := showDeferState(t, bd, p.dir, issue.ID) + if status != "open" { + t.Errorf("expected status=open after wake, got %q", status) + } + if deferUntil != nil { + t.Errorf("expected defer_until cleared after wake (undefer's shape), got %v", deferUntil) + } + }) + + t.Run("expired_dated_defer_wakes_on_list_ready", func(t *testing.T) { + issue := bdCreate(t, bd, p.dir, "Expired snooze via list (server mode)", "--type", "task") + bdDefer(t, bd, p.dir, issue.ID, "--until", "2020-01-01") + + issues := bdListJSON(t, bd, p.dir, "--ready") + var found *types.IssueWithCounts + for _, iss := range issues { + if iss.ID == issue.ID { + found = iss + break + } + } + if found == nil { + t.Fatalf("expected %s in bd list --ready after its defer date passed (server mode)", issue.ID) + } + if found.Status != types.StatusOpen { + t.Errorf("expected status=open in bd list --ready, got %q", found.Status) + } + if found.DeferUntil != nil { + t.Errorf("expected defer_until cleared after wake, got %v", found.DeferUntil) + } + }) + + t.Run("dateless_defer_never_wakes", func(t *testing.T) { + issue := bdCreate(t, bd, p.dir, "Indefinite icebox (server mode)", "--type", "task") + bdDefer(t, bd, p.dir, issue.ID) + + ids := wakeReadyIDs(t, bd, p.dir) + if ids[issue.ID] { + t.Errorf("dateless defer %s must not appear in bd ready", issue.ID) + } + status, _ := showDeferState(t, bd, p.dir, issue.ID) + if status != "deferred" { + t.Errorf("dateless defer must stay deferred, got %q", status) + } + }) + + t.Run("future_dated_defer_stays_hidden", func(t *testing.T) { + issue := bdCreate(t, bd, p.dir, "Future snooze (server mode)", "--type", "task") + bdDefer(t, bd, p.dir, issue.ID, "--until", "+8760h") + + ids := wakeReadyIDs(t, bd, p.dir) + if ids[issue.ID] { + t.Errorf("future defer %s must not appear in bd ready", issue.ID) + } + status, _ := showDeferState(t, bd, p.dir, issue.ID) + if status != "deferred" { + t.Errorf("future defer must stay deferred, got %q", status) + } + }) + + t.Run("expired_dated_defer_stays_hidden_under_strict_readonly", func(t *testing.T) { + issue := bdCreate(t, bd, p.dir, "Expired snooze under strict --readonly (server mode)", "--type", "task") + bdDefer(t, bd, p.dir, issue.ID, "--until", "2020-01-01") + status, _ := showDeferState(t, bd, p.dir, issue.ID) + if status != "deferred" { + t.Fatalf("precondition: expected status=deferred, got %q", status) + } + + ids := wakeReadyIDs(t, bd, p.dir, "--readonly") + if ids[issue.ID] { + t.Errorf("expired defer %s must not appear in bd --readonly ready — strict --readonly must never sweep", issue.ID) + } + status, _ = showDeferState(t, bd, p.dir, issue.ID) + if status != "deferred" { + t.Errorf("expired defer must stay deferred under strict --readonly, got %q", status) + } + }) +} diff --git a/cmd/bd/main.go b/cmd/bd/main.go index b83c83f38..f4cc8be21 100644 --- a/cmd/bd/main.go +++ b/cmd/bd/main.go @@ -1747,6 +1747,10 @@ var rootCmd = &cobra.Command{ // Bulk loads outlive the pool's 10s fast-fail on every server // pause (wy-sbgucn); explicit env/config settings still win. PoolReadTimeoutFallback: bulkLoadPoolReadTimeout(cmd), + // Classification-only read (GH#804), never strict --readonly or a + // preview: the store is genuinely writable underneath, so the + // lazy defer-wake sweep may still run (be-vbhpf). + ClassifiedRead: policy.readOnly && !readonlyMode && !previewMode, } // Load config to get database name and server connection settings. diff --git a/internal/storage/dolt/queries.go b/internal/storage/dolt/queries.go index b156cff12..344325b7a 100644 --- a/internal/storage/dolt/queries.go +++ b/internal/storage/dolt/queries.go @@ -47,6 +47,23 @@ func (s *DoltStore) SearchIssuesWithCounts(ctx context.Context, query string, fi return result, err } +// deferWakeSweepEligible reports whether a store opened with the given +// readOnly/classifiedRead flags may run the lazy defer-wake sweep. A +// classified-read open (GH#804: bd ready/bd list are read-only for the +// CURRENT project) is eligible despite readOnly=true because the store is +// genuinely writable underneath; strict --readonly, preview, and +// foreign-project opens are not (be-vbhpf). +// +// Only the strict --readonly and preview cases are decided here, by +// classifiedRead being false. Foreign-project and other auxiliary opens are +// excluded one layer up: they build their own dolt.Config literals and never +// set ClassifiedRead, so they reach this guard already fail-closed. Do not +// look for a foreign-project term in this predicate — there is none, and +// setting ClassifiedRead on such an open would defeat the exclusion. +func deferWakeSweepEligible(readOnly, classifiedRead bool) bool { + return !readOnly || classifiedRead +} + // wakeExpiredDefers runs the lazy defer-wake sweep (issueops.WakeExpiredDefersInTx) // in its own write transaction before a ready-work read. Advisory by contract: // a ready listing must never fail because the sweep could not run, so every @@ -54,8 +71,15 @@ func (s *DoltStore) SearchIssuesWithCounts(ctx context.Context, query string, fi // open write circuit, closed store), with a stderr warning otherwise. It runs // OUTSIDE the read tx below because withReadTx unconditionally rolls back (and // may retry its body on the read-only justification). +// +// A classified-read open (GH#804: bd ready/bd list are read-only for the +// CURRENT project) still runs the sweep — the store is genuinely writable +// underneath, and only the command's own semantics forbid a mutating result. +// A strict --readonly, preview, or foreign-project open must never sweep, +// which is why deferWakeSweepEligible checks classifiedRead rather than the +// guard dropping the readOnly check entirely (be-vbhpf). func (s *DoltStore) wakeExpiredDefers(ctx context.Context) { - if s.readOnly { + if !deferWakeSweepEligible(s.readOnly, s.classifiedRead) { return } err := s.withCircuitWrite(ctx, func(ctx context.Context) error { diff --git a/internal/storage/dolt/readonly_server_policy_test.go b/internal/storage/dolt/readonly_server_policy_test.go index c8cd0bd48..2a0089d7a 100644 --- a/internal/storage/dolt/readonly_server_policy_test.go +++ b/internal/storage/dolt/readonly_server_policy_test.go @@ -134,3 +134,32 @@ func TestServerOpenCanAutoStartHonorsDisableAutoStart(t *testing.T) { t.Fatal("ordinary classified-read (ReadOnly without DisableAutoStart) must retain auto-start behavior") } } + +func TestDeferWakeSweepEligibleHonorsClassifiedRead(t *testing.T) { + cases := []struct { + name string + readOnly bool + classifiedRead bool + want bool + }{ + {name: "writable open sweeps", readOnly: false, classifiedRead: false, want: true}, + {name: "strict read-only never sweeps", readOnly: true, classifiedRead: false, want: false}, + // An ordinary classified-read command (bd ready, bd list) sets + // readOnly but the store is genuinely writable underneath, so the + // defer-wake sweep must still run (regression guard for be-vbhpf: + // the sweep silently early-returned for every read-only-classified + // command in server mode, not just strict --readonly/preview). + {name: "classified-read still sweeps despite readOnly", readOnly: true, classifiedRead: true, want: true}, + // readOnly=false already short-circuits the predicate to true on its + // own, so this case documents totality (every input combination has + // an asserted output), not a state any caller actually produces. + {name: "classifiedRead without readOnly stays eligible", readOnly: false, classifiedRead: true, want: true}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + if got := deferWakeSweepEligible(tc.readOnly, tc.classifiedRead); got != tc.want { + t.Fatalf("deferWakeSweepEligible(readOnly=%v, classifiedRead=%v) = %v, want %v", tc.readOnly, tc.classifiedRead, got, tc.want) + } + }) + } +} diff --git a/internal/storage/dolt/store.go b/internal/storage/dolt/store.go index 7c27d2192..2b120c870 100644 --- a/internal/storage/dolt/store.go +++ b/internal/storage/dolt/store.go @@ -329,6 +329,7 @@ type DoltStore struct { serverEndpoint string // Exact endpoint bound to bootstrap reset authority mu sync.RWMutex // Protects concurrent access readOnly bool // True if opened in read-only mode + classifiedRead bool // True if readOnly came from command classification (GH#804), not strict --readonly/preview/foreign-project; still eligible for the defer-wake sweep (be-vbhpf) credentialKey []byte // Random encryption key for federation credentials // localActiveDatabaseDir is the exact active database directory when this @@ -378,6 +379,19 @@ type Config struct { ReadOnly bool // Open in read-only mode (skip schema init) Preview bool // Non-mutating preview: embedded opens skip schema init and refuse writes + // ClassifiedRead marks a ReadOnly open whose read-only-ness comes purely + // from command classification (GH#804: bd ready/bd list are read-only for + // the CURRENT project) rather than strict --readonly, an explicit preview, + // or a foreign-project lookup. Only a classified-read open is eligible for + // the lazy defer-wake sweep (be-vbhpf) — the others must never mutate. + // + // Strict --readonly and preview are excluded by the policy expression that + // computes this field; foreign-project and auxiliary opens are excluded + // because they construct their own Config and leave this at its false zero + // value. That default is the guarantee — setting it true on such an open + // would make it eligible to sweep. + ClassifiedRead bool + // LenientOpen opens the store leniently: a migration gate refusal (#4259) // or a dirty-working-set refusal (#4566) skips the migration instead of // failing the open. Set for working-set-reconcile commands (bd dolt @@ -2135,6 +2149,7 @@ func newServerMode(ctx context.Context, cfg *Config) (*DoltStore, error) { remotePassword: cfg.RemotePassword, serverMode: true, readOnly: cfg.ReadOnly, + classifiedRead: cfg.ClassifiedRead, autoStartedServerDir: autoStartedDir, } diff --git a/scripts/ci/gen_proxied_shard_manifest.py b/scripts/ci/gen_proxied_shard_manifest.py index 7905eb026..afdc7d5c9 100644 --- a/scripts/ci/gen_proxied_shard_manifest.py +++ b/scripts/ci/gen_proxied_shard_manifest.py @@ -1,7 +1,11 @@ #!/usr/bin/env python3 """Regenerate .github/scripts/proxied-cmd-test-shards.txt by cost-balancing. -Cost proxy for each top-level TestProxiedServer* function is its bd-init count +Discovery must stay in step with proxied-test-shard.sh: both select the +TestProxiedServer* and TestServerMode* entry points, which share the +requireSharedProxiedServer gate and the shared dolt sql-server. + +Cost proxy for each such top-level function is its bd-init count (newSharedProxiedProject occurrences) — inits dominate proxied wall-time. Tests are packed into TOTAL shards longest-processing-time-first so the heaviest shard is minimized. Prints the manifest to stdout. @@ -13,7 +17,7 @@ import re import sys TOTAL = int(sys.argv[1]) if len(sys.argv) > 1 else 15 -func_re = re.compile(r'^func (TestProxiedServer[A-Za-z0-9_]+)\(') +func_re = re.compile(r'^func (Test(?:ProxiedServer|ServerMode)[A-Za-z0-9_]+)\(') costs = {} for path in glob.glob('cmd/bd/*_test.go'): @@ -51,8 +55,9 @@ out.append('#') out.append(f'# {TOTAL}-shard split, bin-packed longest-processing-time-first by') out.append('# estimated cost (bd-init count; per-init migration chains dominate).') out.append('# Regenerate with scripts/ci/gen_proxied_shard_manifest.py after adding,') -out.append('# splitting, or reweighting TestProxiedServer* functions. Newly-added') -out.append('# tests not listed here hash-distribute via proxied-test-shard.sh.') +out.append('# splitting, or reweighting TestProxiedServer*/TestServerMode*') +out.append('# functions. Newly-added tests not listed here hash-distribute via') +out.append('# proxied-test-shard.sh.') out.append('#') out.append('# This file is generated: notes added here are erased by the next') out.append('# regeneration. Add them to the generator instead. Known blind spots')