fix(schema): carry main-plane wisps ALTERs onto the ignored track for fresh clones (bd-hs7fa) (#5170)

Fresh clones materialize the clone-local (dolt_ignored) tables from the
ignored migration series with the main schema cursor already at-latest, so
a main-plane migration's DDL against one of those tables silently never
reaches them. Observed in prod on the wy-98eh5 VM re-clone: 0060 added
wisps.storage_class on the main plane only, and every shared wisp scan from
a post-0060 binary failed with Error 1054 (same mechanism as wy-pt82l's
wisps.row_lock, healed by ignored/0013).

- ignored/0020: guarded ALTER ADD wisps.storage_class, mirroring 0060.
- ignored/0021: 0049's wisps TEXT->LONGTEXT widening on the ignored track
  (fresh clones otherwise cap large-content columns at 64KB).
- New embedded convergence test diffs every clone-local table's column
  shape between the fresh-init and fresh-clone doors; it found exactly the
  two gaps above and is the permanent engine-level backstop.
- check-migration-hygiene.sh check D: a new main-plane migration with DDL
  on a clone-local table must ship an ignored-series twin in the same
  change (events is in this class since 0062).

Co-authored-by: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
Steve Yegge 2026-07-30 14:10:40 -07:00 • committed by GitHub
commit 63a471a1e4
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
4 changed files with 300 additions and 3 deletions

View file

@ -0,0 +1,165 @@
//go:build cgo
package embeddeddolt_test
import (
"context"
"database/sql"
"fmt"
"sort"
"testing"
"github.com/steveyegge/beads/internal/storage/schema"
)
// TestEmbeddedIgnoredSeriesConvergesWithFreshInitShape pins the bd-hs7fa
// contract: the fresh-clone door and the fresh-init door must produce the
// same schema for every clone-local (dolt_ignored) table.
//
// The two doors build those tables from different sources. A fresh init runs
// the main migration series, whose CREATEs and later ALTERs (0049's LONGTEXT
// widening, 0060's storage_class) all execute. A fresh clone receives only
// committed history — no clone-local tables, no ignored_schema_migrations
// cursor — with the MAIN cursor already at-latest, so none of the main
// series re-runs; the ignored series alone materializes the clone-local
// tables. Any main-plane ALTER against one of these tables that never got an
// ignored-series twin therefore silently never reaches fresh clones
// (observed in prod as wy-98eh5's re-clone missing wisps.storage_class, and
// before that as wy-pt82l missing wisps.row_lock — healed by ignored/0013).
//
// This test simulates both doors against the same binary and diffs every
// clone-local table's column shape (name, type, nullability, default). It is
// the engine-level backstop behind scripts/check-migration-hygiene.sh check
// D, which enforces the twin rule at source-review time.
func TestEmbeddedIgnoredSeriesConvergesWithFreshInitShape(t *testing.T) {
requireEmbedded(t)
ctx := t.Context()
// Door A: fresh init. seedMainSchemaAt runs the main series (whose
// pre-0047 repair creates wisps mid-pass, so the later wisps ALTERs
// fire); MigrateUp then runs the ignored series over the same tables.
initDir := seedMainSchemaAt(t, ctx, schema.LatestVersion())
initConn, closeInit := openPinnedConn(t, ctx, initDir)
defer closeInit()
if _, err := schema.MigrateUp(ctx, initConn); err != nil {
t.Fatalf("fresh-init MigrateUp: %v", err)
}
// Door B: fresh clone. Committed history carries the main tables and the
// at-latest main cursor; the clone-local tables and the ignored cursor
// are working-set state a clone never receives. Drop them (children
// before FK parents) and let MigrateUp materialize everything through
// the ignored series alone.
cloneDir := seedMainSchemaAt(t, ctx, schema.LatestVersion())
cloneConn, closeClone := openPinnedConn(t, ctx, cloneDir)
defer closeClone()
if _, err := schema.MigrateUp(ctx, cloneConn); err != nil {
t.Fatalf("clone-door seed MigrateUp: %v", err)
}
for _, table := range []string{
"wisp_dependencies", "wisp_events", "wisp_comments", "wisp_labels",
"wisp_child_counters", "wisps", "events", "leases", "repo_mtimes",
"local_metadata", "ignored_schema_migrations",
} {
execFrozenGuard(t, ctx, cloneConn, "DROP TABLE IF EXISTS "+table)
}
// Harness artifact, not part of the simulation: seedMainSchemaAt's
// DOLT_ADD('-A') baseline commit runs before MigrateUp seeds the static
// dolt_ignore pattern for leases, so on THIS path leases lands in
// committed history and its drop above shows up as a tracked deletion,
// which the ignored-source dirty-table guard would refuse. A real clone
// never has leases at HEAD. The pattern row must go before the deletion
// can be staged (DOLT_ADD skips ignored tables even for deletions);
// MigrateUp's seedDoltIgnorePatterns re-asserts it immediately after.
execFrozenGuard(t, ctx, cloneConn, "DELETE FROM dolt_ignore WHERE pattern = 'leases'")
mustDrain(t, ctx, cloneConn, "CALL DOLT_ADD('-A')")
mustDrain(t, ctx, cloneConn, "CALL DOLT_COMMIT('-m', 'test: remove clone-local state from history', '--skip-empty')")
if _, err := schema.MigrateUp(ctx, cloneConn); err != nil {
t.Fatalf("clone-door MigrateUp: %v", err)
}
for _, table := range []string{
"wisps", "wisp_labels", "wisp_dependencies", "wisp_events",
"wisp_comments", "wisp_child_counters", "events", "leases",
"repo_mtimes", "local_metadata",
} {
initShape := clonePlaneTableShape(t, ctx, initConn, table)
cloneShape := clonePlaneTableShape(t, ctx, cloneConn, table)
diffClonePlaneShapes(t, table, initShape, cloneShape)
}
}
// clonePlaneColumn is the column identity compared across the two doors.
// Ordinal position is deliberately excluded: an ALTER ADD on the init door
// appends where the ignored series' CREATE may inline, and column order is
// not load-bearing for any bd query.
type clonePlaneColumn struct {
ColumnType string
Nullable string
Default sql.NullString
}
func clonePlaneTableShape(t *testing.T, ctx context.Context, conn *sql.Conn, table string) map[string]clonePlaneColumn {
t.Helper()
rows, err := conn.QueryContext(ctx, `
SELECT COLUMN_NAME, COLUMN_TYPE, IS_NULLABLE, COLUMN_DEFAULT
FROM INFORMATION_SCHEMA.COLUMNS
WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = ?`, table)
if err != nil {
t.Fatalf("read %s columns: %v", table, err)
}
defer rows.Close()
shape := map[string]clonePlaneColumn{}
for rows.Next() {
var name string
var col clonePlaneColumn
if err := rows.Scan(&name, &col.ColumnType, &col.Nullable, &col.Default); err != nil {
t.Fatalf("scan %s column: %v", table, err)
}
shape[name] = col
}
if err := rows.Err(); err != nil {
t.Fatalf("iterate %s columns: %v", table, err)
}
if len(shape) == 0 {
t.Fatalf("table %s has no columns (missing?)", table)
}
return shape
}
func diffClonePlaneShapes(t *testing.T, table string, initShape, cloneShape map[string]clonePlaneColumn) {
t.Helper()
names := map[string]bool{}
for name := range initShape {
names[name] = true
}
for name := range cloneShape {
names[name] = true
}
sorted := make([]string, 0, len(names))
for name := range names {
sorted = append(sorted, name)
}
sort.Strings(sorted)
for _, name := range sorted {
initCol, inInit := initShape[name]
cloneCol, inClone := cloneShape[name]
switch {
case !inClone:
t.Errorf("%s.%s: present on the fresh-init door but missing on the fresh-clone door — a main-plane migration touched this clone-local table without an ignored-series twin", table, name)
case !inInit:
t.Errorf("%s.%s: present on the fresh-clone door but missing on the fresh-init door — the ignored series adds something the main series never did", table, name)
case initCol != cloneCol:
t.Errorf("%s.%s: shape differs between doors: fresh-init %s, fresh-clone %s",
table, name, formatClonePlaneColumn(initCol), formatClonePlaneColumn(cloneCol))
}
}
}
func formatClonePlaneColumn(c clonePlaneColumn) string {
def := "NULL"
if c.Default.Valid {
def = fmt.Sprintf("%q", c.Default.String)
}
return fmt.Sprintf("{type=%s nullable=%s default=%s}", c.ColumnType, c.Nullable, def)
}

View file

@ -0,0 +1,32 @@
-- Ignored migration 0020: ensure wisps.storage_class exists on every clone
-- (bd-hs7fa).
--
-- Synced migration 0060 added storage_class to wisps — but wisps is
-- dolt-ignored (migration 0019), so its schema is clone-local, and a
-- workspace that bootstraps or re-clones from a remote whose
-- schema_migrations cursor is already >= 0060 adopts the cursor without ever
-- executing 0060. Its wisps table (materialized by ignored/0001, which
-- predates storage_class) then permanently lacks the column, and every
-- shared wisp scan from a post-0060 binary fails with Error 1054 (observed
-- in prod on the wy-98eh5 VM re-clone). Same mechanism, same shape, same fix
-- as ignored/0013 (wisps.row_lock, wy-pt82l).
--
-- The guard makes this a no-op on in-place-upgraded workspaces where synced
-- 0060 already added the column, and on workspaces with no local wisps table
-- yet. Definition mirrors 0060 exactly: VARCHAR(16), nullable, no default —
-- NULL means unset and resolves per Protocol v0.1 C1.2
-- (types.Issue.EffectiveStorageClass).
SET @needs_add = IF(
(SELECT COUNT(*) FROM INFORMATION_SCHEMA.TABLES
WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = 'wisps') > 0
AND
(SELECT COUNT(*) FROM INFORMATION_SCHEMA.COLUMNS
WHERE TABLE_SCHEMA = DATABASE()
AND TABLE_NAME = 'wisps'
AND COLUMN_NAME = 'storage_class') = 0,
1, 0
);
SET @sql = IF(@needs_add = 1,
'ALTER TABLE wisps ADD COLUMN storage_class VARCHAR(16)',
'SELECT 1');
PREPARE stmt FROM @sql; EXECUTE stmt; DEALLOCATE PREPARE stmt;

View file

@ -0,0 +1,43 @@
-- Ignored migration 0021: ensure the wisps large-content columns are
-- LONGTEXT on every clone (bd-hs7fa, second finding).
--
-- Synced migration 0049 widened description/design/acceptance_criteria/notes
-- and close_reason from TEXT to LONGTEXT on issues AND wisps — but wisps is
-- dolt-ignored, so a fresh clone materializes it from ignored/0001, whose
-- CREATE still carries the original TEXT columns, and the at-latest main
-- cursor means 0049 never runs there. TEXT caps at 65535 bytes: a wisp
-- with an embedded-image description or a large agent payload that writes
-- fine on an in-place-upgraded workspace fails with Error 1105 on a fresh
-- clone. Same fresh-clone-door mechanism as ignored/0020 (storage_class)
-- and ignored/0013 (row_lock); found by the shape audit in
-- internal/storage/embeddeddolt/migrate_ignored_plane_shape_test.go.
--
-- Definitions mirror 0049's wisps section exactly, including the restated
-- DEFAULTs (MODIFY COLUMN replaces the whole definition; dropping the
-- defaults would regress inserts that omit the column). Guarded on the same
-- COLUMN_TYPE = 'text' probe as 0049, so this is a no-op on
-- in-place-upgraded workspaces and on the fresh-init door, and idempotent
-- on replay.
SET @wisps_needs_fix = (
SELECT IF(COLUMN_TYPE = 'text', 1, 0)
FROM INFORMATION_SCHEMA.COLUMNS
WHERE TABLE_SCHEMA = DATABASE()
AND TABLE_NAME = 'wisps'
AND COLUMN_NAME = 'description'
);
SET @sql = IF(@wisps_needs_fix = 1,
'ALTER TABLE wisps MODIFY COLUMN description LONGTEXT NOT NULL DEFAULT '''', MODIFY COLUMN design LONGTEXT NOT NULL DEFAULT '''', MODIFY COLUMN acceptance_criteria LONGTEXT NOT NULL DEFAULT '''', MODIFY COLUMN notes LONGTEXT NOT NULL DEFAULT ''''',
'SELECT 1');
PREPARE stmt FROM @sql; EXECUTE stmt; DEALLOCATE PREPARE stmt;
SET @wisps_cr_needs_fix = (
SELECT IF(COLUMN_TYPE = 'text', 1, 0)
FROM INFORMATION_SCHEMA.COLUMNS
WHERE TABLE_SCHEMA = DATABASE()
AND TABLE_NAME = 'wisps'
AND COLUMN_NAME = 'close_reason'
);
SET @sql = IF(@wisps_cr_needs_fix = 1,
'ALTER TABLE wisps MODIFY COLUMN close_reason LONGTEXT DEFAULT ''''',
'SELECT 1');
PREPARE stmt FROM @sql; EXECUTE stmt; DEALLOCATE PREPARE stmt;

View file

@ -23,10 +23,17 @@
# exists on the base branch. New files only. Fix-forward with a new
# migration instead: applied migration content is content-hashed
# (PR 4270), so editing history creates cross-clone hash skew.
# D. A new main-plane migration whose DDL touches a clone-local
# (dolt_ignored) table must ship an ignored-series twin in the same
# change. Clone-local tables are materialized on fresh clones by the
# ignored series alone — the main cursor arrives at-latest, so a
# main-plane ALTER silently never reaches them (bd-hs7fa: 0060's
# wisps.storage_class missed every fresh clone; wy-pt82l before it:
# 0054's wisps.row_lock, healed after the fact by ignored/0013).
#
# Check C compares against $BASE_SHA if set (CI passes the PR base), else
# origin/main, else main; it is skipped with a warning when no base is
# resolvable (e.g. shallow clone without the base commit).
# Checks C and D compare against $BASE_SHA if set (CI passes the PR base),
# else origin/main, else main; they are skipped with a warning when no base
# is resolvable (e.g. shallow clone without the base commit).
set -euo pipefail
@ -152,6 +159,56 @@ EOF
fi
fi
# --- Check D: main-plane DDL on clone-local tables needs an ignored twin -----
# Fresh clones materialize the clone-local (dolt_ignored) tables from the
# ignored series with the MAIN cursor already at-latest, so a main-plane
# migration's DDL against one of them never executes there. The main-plane
# migration is still required (it upgrades in-place workspaces, whose ignored
# cursor is likewise at-latest); the twin is what carries the change through
# the fresh-clone door. Data-only statements (UPDATE/INSERT/DELETE) don't
# trigger this: clone-local data is clone-local by design.
#
# The table list mirrors internal/storage/schema/schema.go's
# doltIgnorePatterns + versionGatedDoltIgnorePatterns. The engine-level
# backstop for the same invariant is
# internal/storage/embeddeddolt/migrate_ignored_plane_shape_test.go.
if [ -z "$base" ] || [ -z "${merge_base:-}" ]; then
echo "WARN (ignored twins) no usable base ref; skipping check D." >&2
else
ignored_tables='wisps|wisp_[a-z_]+|repo_mtimes|local_metadata|leases|events|ignored_schema_migrations'
ddl_re="(alter|create|rename|drop)[[:space:]]+table([[:space:]]+if[[:space:]]+(not[[:space:]]+)?exists)?[[:space:]]+[\`']?(${ignored_tables})\b"
# Untracked files count as added too (mirrors check C's working-tree scope:
# in CI the tree is clean so this equals the PR diff).
added_all=$(
{
git diff --name-only --diff-filter=A "$merge_base" -- "$MIG_DIR/*.sql" "$MIG_DIR/ignored/*.sql"
git ls-files --others --exclude-standard -- "$MIG_DIR"
} | grep '\.sql$' | sort -u || true
)
added_main=$(grep -v '/ignored/' <<<"$added_all" || true)
added_ignored=$(grep '/ignored/' <<<"$added_all" || true)
for f in $added_main; do
[ -f "$f" ] || continue
stripped=$(sed 's/--.*$//' "$f" | tr '[:upper:]' '[:lower:]')
hits=$(grep -n -E "$ddl_re" <<<"$stripped" || true)
if [ -n "$hits" ] && [ -z "$added_ignored" ]; then
fail=1
echo "FAIL (ignored twin missing) $f contains DDL against a clone-local table:"
echo "$hits" | sed 's/^/ line /'
cat <<'EOF'
This table is dolt_ignored (clone-local): fresh clones build it from the
ignored migration series with the main cursor already at-latest, so this
DDL will silently never run there (bd-hs7fa). Ship a guarded twin under
internal/storage/schema/migrations/ignored/ in the same PR (precedent:
ignored/0013 for main 0054, ignored/0020 for main 0060). If the change is
genuinely main-plane-only (e.g. the table is being flipped ONTO the
ignored plane by this very migration), the twin is the migration that
materializes the table for clones (precedent: 0062 + ignored/0019).
EOF
fi
done
fi
if [ "$fail" -ne 0 ]; then
echo
echo "Migration hygiene check failed. See docs in scripts/check-migration-hygiene.sh."