diff --git a/CHANGELOG.md b/CHANGELOG.md index 310ad7f7d..7884595fb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -447,6 +447,25 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added +- **A long-running schema migration now says so instead of going quiet** + ([#5997](https://github.com/gastownhall/beads/pull/5997)). Migrations are + allowed to take a long time by design — migration 0047's full-table + `is_blocked` recompute is a real example — but until now a slow one and a + wedged one looked identical from outside: `bd` simply stopped printing. A + watchdog now emits a WARN naming the migration's version, name, and elapsed + time once the migration's own SQL has been running past the interval, and + repeats every interval while it keeps running, so an operator reading logs + can tell "still working" from "stopped emitting anything". Coverage is the + migration body specifically — the per-step Dolt commit that follows it on + the production embedded path is outside the watchdog, so a stall there is + still quiet. Set `BEADS_MIGRATION_WATCHDOG_INTERVAL` to change the + 5-minute default; it accepts durations like `10m` and bare seconds like `90`, + and falls back to the default when unset or unparsable. This is observability + only and never a circuit breaker: the migration receives the caller's exact + context, its error is passed through unchanged, and nothing is aborted or + rolled back. The warning is deliberately not terminal-gated, so it survives + `bd serve`, systemd, CI, and piped invocations. + - **Auto-backup runs on a managed-local proxied-server workspace.** The proxied arm of the post-command hook now calls auto-backup, so an explicit `backup.enabled=true` (or `BD_BACKUP_ENABLED=1`) takes the same throttled, diff --git a/docs/reference/configuration.md b/docs/reference/configuration.md index 88915e2fb..20d9e001e 100644 --- a/docs/reference/configuration.md +++ b/docs/reference/configuration.md @@ -503,6 +503,7 @@ Selected commonly-used variables: | `BEADS_ACTOR` | Actor identity (preferred over `BD_ACTOR`, which is a deprecated alias) | | `BEADS_IDENTITY` | Sender identity for `bd mail` | | `BEADS_FSCK_TIMEOUT` | Runtime-only timeout for the pre-push `dolt fsck --quiet` integrity check (default `30s`) | +| `BEADS_MIGRATION_WATCHDOG_INTERVAL` | Interval between soft WARN lines while a schema migration's SQL keeps running (default `5m`; accepts durations like `10m` or bare seconds like `90`). Observability only — it never aborts a migration | | `BEADS_DOLT_SERVER_MODE`, `BEADS_DOLT_SHARED_SERVER`, `BEADS_DOLT_DATA_DIR`, `BEADS_DOLT_PORT`, ... | Embedded/server Dolt overrides | | `BEADS_DOLT_BIN` | Pin the exact external `dolt` CLI binary managed proxied-server mode spawns, overriding PATH lookup (highest precedence; an explicit path that fails validation is an error, not a silent fallback to PATH). On Windows the executable extension may be omitted — `C:\tools\dolt` finds `C:\tools\dolt.exe` via PATHEXT, though a file at the exact spelled path wins if both exist | diff --git a/internal/storage/schema/BUILD.bazel b/internal/storage/schema/BUILD.bazel index 1865d179e..5108ff37d 100644 --- a/internal/storage/schema/BUILD.bazel +++ b/internal/storage/schema/BUILD.bazel @@ -150,6 +150,7 @@ go_test( "migration_0068_add_attribution_status_test.go", "migration_0069_datetime_precision_test.go", "migration_body_drain_test.go", + "migration_watchdog_test.go", "progress_test.go", "remote_migrate_gate_test.go", "schema_skew_test.go", @@ -166,6 +167,7 @@ go_test( "//internal/testutil", "@com_github_data_dog_go_sqlmock//:go-sqlmock", "@com_github_go_sql_driver_mysql//:mysql", + "@org_golang_x_term//:term", ], ) diff --git a/internal/storage/schema/migration_watchdog_test.go b/internal/storage/schema/migration_watchdog_test.go new file mode 100644 index 000000000..d99976158 --- /dev/null +++ b/internal/storage/schema/migration_watchdog_test.go @@ -0,0 +1,308 @@ +package schema + +import ( + "bytes" + "context" + "database/sql" + "errors" + "fmt" + "io" + "os" + "strings" + "sync" + "testing" + "time" + + "golang.org/x/term" +) + +// TestRunMigrationWithWatchdog_NoWarningUnderThreshold covers be-yyzzs: a +// migration that finishes well inside the threshold must produce zero +// watchdog output — the warning is a "still working" signal, not a +// per-migration progress line (that already exists via runMigrations' +// "Applying migration..."/"done" prints). +func TestRunMigrationWithWatchdog_NoWarningUnderThreshold(t *testing.T) { + var buf bytes.Buffer + err := runMigrationWithWatchdog(context.Background(), &buf, 42, "add_date_indexes", 100*time.Millisecond, + func(ctx context.Context) error { return nil }) + + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if got := buf.String(); got != "" { + t.Errorf("want no watchdog output for a fast migration, got %q", got) + } +} + +// TestRunMigrationWithWatchdog_WarnsOnThresholdExceeded covers be-yyzzs: once +// a migration runs past the threshold, a WARN line naming the version, the +// human migration name, and the elapsed time must appear on the writer that +// runMigrations already uses for progress output. +func TestRunMigrationWithWatchdog_WarnsOnThresholdExceeded(t *testing.T) { + var buf bytes.Buffer + err := runMigrationWithWatchdog(context.Background(), &buf, 42, "add_date_indexes", 15*time.Millisecond, + func(ctx context.Context) error { + time.Sleep(40 * time.Millisecond) + return nil + }) + + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + got := buf.String() + if !strings.Contains(got, "WARN") { + t.Errorf("want a WARN-level line, got %q", got) + } + if !strings.Contains(got, fmt.Sprintf("%04d", 42)) { + t.Errorf("want the migration version in the warning, got %q", got) + } + if !strings.Contains(got, "add_date_indexes") { + t.Errorf("want the migration name in the warning, got %q", got) + } + if !strings.Contains(got, "still running") { + t.Errorf("want an elapsed-time/still-running marker, got %q", got) + } +} + +// TestRunMigrationWithWatchdog_RepeatsWarningEveryInterval covers be-yyzzs: +// a migration that keeps running past multiple threshold intervals must get +// a fresh warning each interval, not just once — so an operator watching +// logs can distinguish "still working" from "stopped emitting anything". +func TestRunMigrationWithWatchdog_RepeatsWarningEveryInterval(t *testing.T) { + var buf bytes.Buffer + err := runMigrationWithWatchdog(context.Background(), &buf, 7, "backfill", 10*time.Millisecond, + func(ctx context.Context) error { + time.Sleep(55 * time.Millisecond) + return nil + }) + + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + count := strings.Count(buf.String(), "WARN") + if count < 3 { + t.Errorf("want at least 3 repeated warnings over ~5.5 intervals, got %d in %q", count, buf.String()) + } +} + +// TestRunMigrationWithWatchdog_ContextUnaffected covers be-yyzzs: the +// watchdog is observability only. It must hand fn the exact ctx it was +// given — no wrapping with a new timeout/cancel — and that ctx must still be +// live (Err() == nil) even after the watchdog has fired multiple warnings. +func TestRunMigrationWithWatchdog_ContextUnaffected(t *testing.T) { + type sentinelKey struct{} + ctx := context.WithValue(context.Background(), sentinelKey{}, "sentinel-value") + + var sawValue any + var errDuringRun error + var buf bytes.Buffer + + err := runMigrationWithWatchdog(ctx, &buf, 1, "initial", 10*time.Millisecond, + func(fnCtx context.Context) error { + time.Sleep(35 * time.Millisecond) + sawValue = fnCtx.Value(sentinelKey{}) + errDuringRun = fnCtx.Err() + return nil + }) + + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if sawValue != "sentinel-value" { + t.Errorf("fn did not receive the original ctx (missing sentinel value); got %v", sawValue) + } + if errDuringRun != nil { + t.Errorf("ctx passed to fn was canceled/expired by the watchdog: %v", errDuringRun) + } + if ctx.Err() != nil { + t.Errorf("caller's ctx was canceled/expired by the watchdog: %v", ctx.Err()) + } +} + +// TestRunMigrationWithWatchdog_ErrorPassthrough covers be-yyzzs: the +// watchdog must never alter the migration's outcome. A migration that runs +// long AND ultimately fails must still surface the exact original error — +// the warning is purely additive, not a substitute for or wrapper around it. +func TestRunMigrationWithWatchdog_ErrorPassthrough(t *testing.T) { + sentinel := errors.New("boom: migration 0007 failed") + var buf bytes.Buffer + + err := runMigrationWithWatchdog(context.Background(), &buf, 7, "backfill", 10*time.Millisecond, + func(ctx context.Context) error { + time.Sleep(25 * time.Millisecond) + return sentinel + }) + + if !errors.Is(err, sentinel) { + t.Errorf("want sentinel error passed through unchanged, got %v", err) + } + if !strings.Contains(buf.String(), "WARN") { + t.Errorf("want the watchdog to still have warned before the failure, got %q", buf.String()) + } +} + +// syncWriter serializes writes so the panic test below can read what the +// watchdog has emitted so far without racing it. The other tests in this file +// use a bare bytes.Buffer because the watchdog goroutine is always joined +// before they read; the panic case reads at a point where a leaked goroutine +// would still be writing, which is the leak it exists to catch. +type syncWriter struct { + mu sync.Mutex + buf bytes.Buffer +} + +func (w *syncWriter) Write(p []byte) (int, error) { + w.mu.Lock() + defer w.mu.Unlock() + return w.buf.Write(p) +} + +func (w *syncWriter) String() string { + w.mu.Lock() + defer w.mu.Unlock() + return w.buf.String() +} + +// TestRunMigrationWithWatchdog_StopsWarnerWhenFnPanics pins the deferred +// close(done)/wg.Wait(): Dolt's embedded engine can panic out of DDL, and a +// caller that recovers instead of dying (internal/httpapi/server.go's +// middleware recovers everything except http.ErrAbortHandler and keeps +// serving) leaves the process alive afterwards. Stopping the warner on the +// straight-line path after fn returns is skipped by a panic, which orphans the +// ticker goroutine into warning about a migration that is no longer running — +// on the deliberately un-gated watchdogStderr — for the rest of that process's +// lifetime. +func TestRunMigrationWithWatchdog_StopsWarnerWhenFnPanics(t *testing.T) { + const interval = 10 * time.Millisecond + out := &syncWriter{} + + func() { + defer func() { + if r := recover(); r == nil { + t.Errorf("want fn's panic to propagate to the caller unchanged, got none") + } + }() + _ = runMigrationWithWatchdog(context.Background(), out, 47, "recompute_mixed_is_blocked", interval, + func(ctx context.Context) error { + time.Sleep(35 * time.Millisecond) + panic("dolt engine panicked out of DDL") + }) + }() + + // The warner must already be joined by the time the panic reaches here, so + // the line count cannot grow any further. + settled := strings.Count(out.String(), "WARN") + if settled == 0 { + t.Fatalf("want at least one WARN before the panic, otherwise the check below is vacuous; got %q", out.String()) + } + time.Sleep(5 * interval) + if after := strings.Count(out.String(), "WARN"); after != settled { + t.Errorf("watchdog goroutine outlived the panicking migration: %d WARN lines when the panic unwound, %d after a further %s", settled, after, 5*interval) + } +} + +// TestMigrationWatchdogIntervalDuration covers be-yyzzs: the threshold must +// default to 5 minutes and be overridable via BEADS_MIGRATION_WATCHDOG_INTERVAL, +// following this codebase's existing timeoutFromEnv/BEADS_*_TIMEOUT convention +// (internal/storage/dolt/store.go's cliExecTimeoutEnv/timeoutFromEnv): unset, +// empty, or unparsable values fall back to the default. +func TestMigrationWatchdogIntervalDuration(t *testing.T) { + cases := []struct { + name string + envVal string + setEnv bool + want time.Duration + }{ + {name: "unset_defaults_to_5m", setEnv: false, want: 5 * time.Minute}, + {name: "empty_defaults_to_5m", setEnv: true, envVal: "", want: 5 * time.Minute}, + {name: "valid_duration_string_overrides", setEnv: true, envVal: "10s", want: 10 * time.Second}, + {name: "invalid_value_falls_back_to_default", setEnv: true, envVal: "not-a-duration", want: 5 * time.Minute}, + // internal/storage/dolt/store.go's parseTimeout — the convention this + // function's doc comment claims to follow — documents "bare numbers + // treated as seconds (e.g. \"90\")". Without this, a plausible + // BEADS_MIGRATION_WATCHDOG_INTERVAL=300 silently means 5m rather than + // the 5m the operator was trying to change. + // NB: not "300" — 300s IS the 5m default, so that value cannot tell + // "parsed as seconds" from "fell back". + {name: "bare_seconds_treated_as_seconds", setEnv: true, envVal: "90", want: 90 * time.Second}, + {name: "bare_seconds_larger_than_default", setEnv: true, envVal: "600", want: 10 * time.Minute}, + {name: "bare_zero_falls_back_to_default", setEnv: true, envVal: "0", want: 5 * time.Minute}, + {name: "negative_falls_back_to_default", setEnv: true, envVal: "-5s", want: 5 * time.Minute}, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + if tc.setEnv { + t.Setenv(migrationWatchdogIntervalEnv, tc.envVal) + } else { + // The "unset" arm has to actually unset: t.Setenv registers the + // restore of whatever the ambient value was, then os.Unsetenv + // removes it for the duration of the subtest. Without this, + // a shell exporting BEADS_MIGRATION_WATCHDOG_INTERVAL (this + // repo has a history of ambient BEADS_* reddening tests) flows + // straight into the call and the compiled-in default this + // subtest exists to pin is never exercised. + t.Setenv(migrationWatchdogIntervalEnv, "") + os.Unsetenv(migrationWatchdogIntervalEnv) + } + if got := migrationWatchdogIntervalDuration(); got != tc.want { + t.Errorf("migrationWatchdogIntervalDuration() = %v; want %v", got, tc.want) + } + }) + } +} + +// slowMigrationDB makes the first migration body take long enough for a +// short-interval watchdog to fire, so the wiring between runMigrations and +// runMigrationWithWatchdog can be exercised end to end rather than asserted +// on a package variable. +type slowMigrationDB struct { + mockDB + once sync.Once + d time.Duration +} + +func (s *slowMigrationDB) ExecContext(ctx context.Context, q string, a ...any) (sql.Result, error) { + s.once.Do(func() { time.Sleep(s.d) }) + return s.mockDB.ExecContext(ctx, q, a...) +} + +// TestRunMigrationsWatchdogSurvivesNonTTY covers the review point on #5997: +// the package's progress writer is io.Discard whenever os.Stderr is not a +// terminal, so wiring the watchdog to it makes the warning silent under +// bd serve, systemd, CI, and any piped invocation — precisely the situations +// where a migration that has been running for twenty minutes needs to be +// visible. On a tty it worked fine, which is why this went unnoticed. +func TestRunMigrationsWatchdogSurvivesNonTTY(t *testing.T) { + if term.IsTerminal(int(os.Stderr.Fd())) { + t.Skip("stderr is a terminal here; this test asserts the non-tty path") + } + + // Put the progress writer in the state a non-tty caller really sees. + origStderr := stderr + stderr = defaultStderr() + defer func() { stderr = origStderr }() + if stderr != io.Discard { + t.Fatalf("precondition: progress writer off a tty = %T, want io.Discard", stderr) + } + + var watchdogBuf bytes.Buffer + origWatchdog := watchdogStderr + watchdogStderr = &watchdogBuf + defer func() { watchdogStderr = origWatchdog }() + + origCounter := issueRowCounter + issueRowCounter = func(context.Context, DBConn) (int64, error) { return 0, nil } + defer func() { issueRowCounter = origCounter }() + + t.Setenv(migrationWatchdogIntervalEnv, "10ms") + + db := &slowMigrationDB{d: 80 * time.Millisecond} + if _, err := runMigrations(context.Background(), db, mainSource, 0, 39, false); err != nil { + t.Fatalf("runMigrations: %v", err) + } + + if got := watchdogBuf.String(); !strings.Contains(got, "WARN") { + t.Errorf("no watchdog WARN reached the non-tty writer: the warning is discarded exactly where operators need it. got %q", got) + } +} diff --git a/internal/storage/schema/schema.go b/internal/storage/schema/schema.go index ff5347a68..2ee213083 100644 --- a/internal/storage/schema/schema.go +++ b/internal/storage/schema/schema.go @@ -36,6 +36,19 @@ func defaultStderr() io.Writer { return io.Discard } +// watchdogStderr is where the migration watchdog's WARN lines go. Unlike +// stderr it is deliberately NOT terminal-gated: bd serve, systemd, CI and any +// piped invocation are all non-tty, and they are exactly the cases where a +// migration that has been running for twenty minutes needs to be visible. +// Routing it through stderr would make the warning io.Discard precisely there. +// +// This matches how this package already emits operator warnings that must not +// be swallowed (remote_migrate_gate.go, smart_remote_migrate_gate.go, and the +// skew notices in this file all write to os.Stderr directly); stderr remains +// for the routine per-migration progress lines it was introduced for. +// Overridable in tests. +var watchdogStderr io.Writer = os.Stderr + const largeRigThreshold = 10000 // issueRowCounter returns the current issues-table row count, or an error if @@ -60,6 +73,90 @@ func emitLargeRigNotice(out io.Writer, count int64, err error) { fmt.Fprintf(out, "Large rig detected (%d issues). This migration may take up to 90 seconds; do not interrupt.\n", count) } +// migrationWatchdogInterval is the default interval between soft watchdog +// warnings for a single long-running migration. +const migrationWatchdogInterval = 5 * time.Minute + +// migrationWatchdogIntervalEnv overrides migrationWatchdogInterval. +const migrationWatchdogIntervalEnv = "BEADS_MIGRATION_WATCHDOG_INTERVAL" + +// migrationWatchdogIntervalDuration returns the configured watchdog interval. +// BEADS_MIGRATION_WATCHDOG_INTERVAL overrides the compiled-in default. Valid +// time.ParseDuration strings ("10m", "90s") and bare numbers treated as +// seconds ("90") are both accepted; unset, empty, unparsable, or non-positive +// values fall back to migrationWatchdogInterval. This is the full contract of +// internal/storage/dolt/store.go's parseTimeout, bare-seconds included — +// mirrored rather than reused because that function is unexported in another +// package. +func migrationWatchdogIntervalDuration() time.Duration { + raw := strings.TrimSpace(os.Getenv(migrationWatchdogIntervalEnv)) + if raw == "" { + return migrationWatchdogInterval + } + if d, err := time.ParseDuration(raw); err == nil { + if d > 0 { + return d + } + return migrationWatchdogInterval + } + // Bare numbers mean seconds, matching parseTimeout. Without this an + // operator setting the value to 90 gets 5m and no diagnostic. + if d, err := time.ParseDuration(raw + "s"); err == nil && d > 0 { + return d + } + return migrationWatchdogInterval +} + +// runMigrationWithWatchdog runs fn — the same call runMigrations already +// makes to execMigrationBody — while a background timer watches elapsed +// time. If fn is still running after interval, it logs a WARN line to out +// naming the migration's version, name, and elapsed time, and repeats every +// additional interval for as long as fn keeps running, so an operator +// watching logs can tell "still working" from "stopped emitting anything". +// +// This is observability only, never a circuit breaker: fn receives exactly +// the ctx this function received (no wrapping timeout/cancel), and fn's +// returned error is passed through unchanged. Migrations are allowed to run +// arbitrarily long by design — store.go's own initSchema comment cites +// migration 0047's full-table is_blocked recompute as a real example — and +// no confirmed production hang exists today. See bd show be-m65rs for the +// full rationale before turning this into a hard abort. +func runMigrationWithWatchdog(ctx context.Context, out io.Writer, version int, name string, interval time.Duration, fn func(ctx context.Context) error) error { + start := time.Now() + done := make(chan struct{}) + var wg sync.WaitGroup + wg.Add(1) + go func() { + defer wg.Done() + ticker := time.NewTicker(interval) + defer ticker.Stop() + for { + select { + case <-done: + return + case <-ticker.C: + fmt.Fprintf(out, "WARN: migration %04d (%s) still running after %s — watchdog check every %s, this is not an error\n", + version, name, time.Since(start).Round(time.Millisecond), interval) + } + } + }() + + // Deferred so the warner is stopped on every exit path, including a panic + // out of fn — Dolt's embedded engine can panic out of DDL, and a caller + // that recovers rather than dying (internal/httpapi/server.go's middleware + // recovers everything except http.ErrAbortHandler and keeps serving) would + // otherwise orphan the ticker goroutine, which goes on writing "still + // running" to the un-gated watchdogStderr for a migration that is no longer + // running. A straight-line close/wait after fn returns leaks in exactly + // that case. + defer func() { + close(done) + wg.Wait() + }() + + return fn(ctx) +} + // humanMigrationName turns "0033_add_date_indexes.up.sql" into // "add_date_indexes" for the progress line. func humanMigrationName(filename string) string { @@ -1813,7 +1910,28 @@ func runMigrations(ctx context.Context, db DBConn, src migrationSource, minVersi fmt.Fprintf(stderr, "Applying migration %04d: %s…\n", mf.version, humanMigrationName(mf.name)) start := time.Now() - if err := execMigrationBody(ctx, db, string(data)); err != nil { + // Soft warning only, by design — never turn this into a hard + // timeout/abort. Migrations can legitimately run arbitrarily long + // (see the initSchema comment in internal/storage/dolt/store.go, + // e.g. migration 0047's full-table recompute); aborting mid-DDL + // risks leaving Dolt in a partially-migrated state with no clean + // rollback. Full scoping rationale and rejected alternatives: + // bd show be-m65rs. + // + // The wrap deliberately covers the migration's own SQL body and + // nothing else: the dirty-table snapshot and preMigrationRepair above + // and the cursor INSERT and commitMigrationStep below all run outside + // it, including the per-step DOLT_ADD/DOLT_COMMIT that the comment + // further down calls the expensive, fallible part of this step on the + // production embedded path. Widening the boundary to the whole step + // means moving this loop's snapshot/commit ordering — the atomicity + // contract documented on runMigrations and in commitMigrationStep + // (#4566, #4690) — which is more risk than an observability change + // should carry. So a wedge inside the per-step commit is still silent; + // CHANGELOG.md scopes the user-facing promise to match. + if err := runMigrationWithWatchdog(ctx, watchdogStderr, mf.version, humanMigrationName(mf.name), migrationWatchdogIntervalDuration(), + func(ctx context.Context) error { return execMigrationBody(ctx, db, string(data)) }, + ); err != nil { return count, fmt.Errorf("migration %s: %w", mf.name, err) } sum := sha256.Sum256(data)