fix(schema): carry main-plane wisps ALTERs onto the ignored track for fresh clones (bd-hs7fa) (#5170)
Fresh clones materialize the clone-local (dolt_ignored) tables from the ignored migration series with the main schema cursor already at-latest, so a main-plane migration's DDL against one of those tables silently never reaches them. Observed in prod on the wy-98eh5 VM re-clone: 0060 added wisps.storage_class on the main plane only, and every shared wisp scan from a post-0060 binary failed with Error 1054 (same mechanism as wy-pt82l's wisps.row_lock, healed by ignored/0013). - ignored/0020: guarded ALTER ADD wisps.storage_class, mirroring 0060. - ignored/0021: 0049's wisps TEXT->LONGTEXT widening on the ignored track (fresh clones otherwise cap large-content columns at 64KB). - New embedded convergence test diffs every clone-local table's column shape between the fresh-init and fresh-clone doors; it found exactly the two gaps above and is the permanent engine-level backstop. - check-migration-hygiene.sh check D: a new main-plane migration with DDL on a clone-local table must ship an ignored-series twin in the same change (events is in this class since 0062). Co-authored-by: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
parent
750fd57417
commit
63a471a1e4
4 changed files with 300 additions and 3 deletions
|
|
@ -0,0 +1,165 @@
|
|||
//go:build cgo
|
||||
|
||||
package embeddeddolt_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"fmt"
|
||||
"sort"
|
||||
"testing"
|
||||
|
||||
"github.com/steveyegge/beads/internal/storage/schema"
|
||||
)
|
||||
|
||||
// TestEmbeddedIgnoredSeriesConvergesWithFreshInitShape pins the bd-hs7fa
|
||||
// contract: the fresh-clone door and the fresh-init door must produce the
|
||||
// same schema for every clone-local (dolt_ignored) table.
|
||||
//
|
||||
// The two doors build those tables from different sources. A fresh init runs
|
||||
// the main migration series, whose CREATEs and later ALTERs (0049's LONGTEXT
|
||||
// widening, 0060's storage_class) all execute. A fresh clone receives only
|
||||
// committed history — no clone-local tables, no ignored_schema_migrations
|
||||
// cursor — with the MAIN cursor already at-latest, so none of the main
|
||||
// series re-runs; the ignored series alone materializes the clone-local
|
||||
// tables. Any main-plane ALTER against one of these tables that never got an
|
||||
// ignored-series twin therefore silently never reaches fresh clones
|
||||
// (observed in prod as wy-98eh5's re-clone missing wisps.storage_class, and
|
||||
// before that as wy-pt82l missing wisps.row_lock — healed by ignored/0013).
|
||||
//
|
||||
// This test simulates both doors against the same binary and diffs every
|
||||
// clone-local table's column shape (name, type, nullability, default). It is
|
||||
// the engine-level backstop behind scripts/check-migration-hygiene.sh check
|
||||
// D, which enforces the twin rule at source-review time.
|
||||
func TestEmbeddedIgnoredSeriesConvergesWithFreshInitShape(t *testing.T) {
|
||||
requireEmbedded(t)
|
||||
ctx := t.Context()
|
||||
|
||||
// Door A: fresh init. seedMainSchemaAt runs the main series (whose
|
||||
// pre-0047 repair creates wisps mid-pass, so the later wisps ALTERs
|
||||
// fire); MigrateUp then runs the ignored series over the same tables.
|
||||
initDir := seedMainSchemaAt(t, ctx, schema.LatestVersion())
|
||||
initConn, closeInit := openPinnedConn(t, ctx, initDir)
|
||||
defer closeInit()
|
||||
if _, err := schema.MigrateUp(ctx, initConn); err != nil {
|
||||
t.Fatalf("fresh-init MigrateUp: %v", err)
|
||||
}
|
||||
|
||||
// Door B: fresh clone. Committed history carries the main tables and the
|
||||
// at-latest main cursor; the clone-local tables and the ignored cursor
|
||||
// are working-set state a clone never receives. Drop them (children
|
||||
// before FK parents) and let MigrateUp materialize everything through
|
||||
// the ignored series alone.
|
||||
cloneDir := seedMainSchemaAt(t, ctx, schema.LatestVersion())
|
||||
cloneConn, closeClone := openPinnedConn(t, ctx, cloneDir)
|
||||
defer closeClone()
|
||||
if _, err := schema.MigrateUp(ctx, cloneConn); err != nil {
|
||||
t.Fatalf("clone-door seed MigrateUp: %v", err)
|
||||
}
|
||||
for _, table := range []string{
|
||||
"wisp_dependencies", "wisp_events", "wisp_comments", "wisp_labels",
|
||||
"wisp_child_counters", "wisps", "events", "leases", "repo_mtimes",
|
||||
"local_metadata", "ignored_schema_migrations",
|
||||
} {
|
||||
execFrozenGuard(t, ctx, cloneConn, "DROP TABLE IF EXISTS "+table)
|
||||
}
|
||||
// Harness artifact, not part of the simulation: seedMainSchemaAt's
|
||||
// DOLT_ADD('-A') baseline commit runs before MigrateUp seeds the static
|
||||
// dolt_ignore pattern for leases, so on THIS path leases lands in
|
||||
// committed history and its drop above shows up as a tracked deletion,
|
||||
// which the ignored-source dirty-table guard would refuse. A real clone
|
||||
// never has leases at HEAD. The pattern row must go before the deletion
|
||||
// can be staged (DOLT_ADD skips ignored tables even for deletions);
|
||||
// MigrateUp's seedDoltIgnorePatterns re-asserts it immediately after.
|
||||
execFrozenGuard(t, ctx, cloneConn, "DELETE FROM dolt_ignore WHERE pattern = 'leases'")
|
||||
mustDrain(t, ctx, cloneConn, "CALL DOLT_ADD('-A')")
|
||||
mustDrain(t, ctx, cloneConn, "CALL DOLT_COMMIT('-m', 'test: remove clone-local state from history', '--skip-empty')")
|
||||
if _, err := schema.MigrateUp(ctx, cloneConn); err != nil {
|
||||
t.Fatalf("clone-door MigrateUp: %v", err)
|
||||
}
|
||||
|
||||
for _, table := range []string{
|
||||
"wisps", "wisp_labels", "wisp_dependencies", "wisp_events",
|
||||
"wisp_comments", "wisp_child_counters", "events", "leases",
|
||||
"repo_mtimes", "local_metadata",
|
||||
} {
|
||||
initShape := clonePlaneTableShape(t, ctx, initConn, table)
|
||||
cloneShape := clonePlaneTableShape(t, ctx, cloneConn, table)
|
||||
diffClonePlaneShapes(t, table, initShape, cloneShape)
|
||||
}
|
||||
}
|
||||
|
||||
// clonePlaneColumn is the column identity compared across the two doors.
|
||||
// Ordinal position is deliberately excluded: an ALTER ADD on the init door
|
||||
// appends where the ignored series' CREATE may inline, and column order is
|
||||
// not load-bearing for any bd query.
|
||||
type clonePlaneColumn struct {
|
||||
ColumnType string
|
||||
Nullable string
|
||||
Default sql.NullString
|
||||
}
|
||||
|
||||
func clonePlaneTableShape(t *testing.T, ctx context.Context, conn *sql.Conn, table string) map[string]clonePlaneColumn {
|
||||
t.Helper()
|
||||
rows, err := conn.QueryContext(ctx, `
|
||||
SELECT COLUMN_NAME, COLUMN_TYPE, IS_NULLABLE, COLUMN_DEFAULT
|
||||
FROM INFORMATION_SCHEMA.COLUMNS
|
||||
WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = ?`, table)
|
||||
if err != nil {
|
||||
t.Fatalf("read %s columns: %v", table, err)
|
||||
}
|
||||
defer rows.Close()
|
||||
shape := map[string]clonePlaneColumn{}
|
||||
for rows.Next() {
|
||||
var name string
|
||||
var col clonePlaneColumn
|
||||
if err := rows.Scan(&name, &col.ColumnType, &col.Nullable, &col.Default); err != nil {
|
||||
t.Fatalf("scan %s column: %v", table, err)
|
||||
}
|
||||
shape[name] = col
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
t.Fatalf("iterate %s columns: %v", table, err)
|
||||
}
|
||||
if len(shape) == 0 {
|
||||
t.Fatalf("table %s has no columns (missing?)", table)
|
||||
}
|
||||
return shape
|
||||
}
|
||||
|
||||
func diffClonePlaneShapes(t *testing.T, table string, initShape, cloneShape map[string]clonePlaneColumn) {
|
||||
t.Helper()
|
||||
names := map[string]bool{}
|
||||
for name := range initShape {
|
||||
names[name] = true
|
||||
}
|
||||
for name := range cloneShape {
|
||||
names[name] = true
|
||||
}
|
||||
sorted := make([]string, 0, len(names))
|
||||
for name := range names {
|
||||
sorted = append(sorted, name)
|
||||
}
|
||||
sort.Strings(sorted)
|
||||
for _, name := range sorted {
|
||||
initCol, inInit := initShape[name]
|
||||
cloneCol, inClone := cloneShape[name]
|
||||
switch {
|
||||
case !inClone:
|
||||
t.Errorf("%s.%s: present on the fresh-init door but missing on the fresh-clone door — a main-plane migration touched this clone-local table without an ignored-series twin", table, name)
|
||||
case !inInit:
|
||||
t.Errorf("%s.%s: present on the fresh-clone door but missing on the fresh-init door — the ignored series adds something the main series never did", table, name)
|
||||
case initCol != cloneCol:
|
||||
t.Errorf("%s.%s: shape differs between doors: fresh-init %s, fresh-clone %s",
|
||||
table, name, formatClonePlaneColumn(initCol), formatClonePlaneColumn(cloneCol))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func formatClonePlaneColumn(c clonePlaneColumn) string {
|
||||
def := "NULL"
|
||||
if c.Default.Valid {
|
||||
def = fmt.Sprintf("%q", c.Default.String)
|
||||
}
|
||||
return fmt.Sprintf("{type=%s nullable=%s default=%s}", c.ColumnType, c.Nullable, def)
|
||||
}
|
||||
|
|
@ -0,0 +1,32 @@
|
|||
-- Ignored migration 0020: ensure wisps.storage_class exists on every clone
|
||||
-- (bd-hs7fa).
|
||||
--
|
||||
-- Synced migration 0060 added storage_class to wisps — but wisps is
|
||||
-- dolt-ignored (migration 0019), so its schema is clone-local, and a
|
||||
-- workspace that bootstraps or re-clones from a remote whose
|
||||
-- schema_migrations cursor is already >= 0060 adopts the cursor without ever
|
||||
-- executing 0060. Its wisps table (materialized by ignored/0001, which
|
||||
-- predates storage_class) then permanently lacks the column, and every
|
||||
-- shared wisp scan from a post-0060 binary fails with Error 1054 (observed
|
||||
-- in prod on the wy-98eh5 VM re-clone). Same mechanism, same shape, same fix
|
||||
-- as ignored/0013 (wisps.row_lock, wy-pt82l).
|
||||
--
|
||||
-- The guard makes this a no-op on in-place-upgraded workspaces where synced
|
||||
-- 0060 already added the column, and on workspaces with no local wisps table
|
||||
-- yet. Definition mirrors 0060 exactly: VARCHAR(16), nullable, no default —
|
||||
-- NULL means unset and resolves per Protocol v0.1 C1.2
|
||||
-- (types.Issue.EffectiveStorageClass).
|
||||
SET @needs_add = IF(
|
||||
(SELECT COUNT(*) FROM INFORMATION_SCHEMA.TABLES
|
||||
WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = 'wisps') > 0
|
||||
AND
|
||||
(SELECT COUNT(*) FROM INFORMATION_SCHEMA.COLUMNS
|
||||
WHERE TABLE_SCHEMA = DATABASE()
|
||||
AND TABLE_NAME = 'wisps'
|
||||
AND COLUMN_NAME = 'storage_class') = 0,
|
||||
1, 0
|
||||
);
|
||||
SET @sql = IF(@needs_add = 1,
|
||||
'ALTER TABLE wisps ADD COLUMN storage_class VARCHAR(16)',
|
||||
'SELECT 1');
|
||||
PREPARE stmt FROM @sql; EXECUTE stmt; DEALLOCATE PREPARE stmt;
|
||||
|
|
@ -0,0 +1,43 @@
|
|||
-- Ignored migration 0021: ensure the wisps large-content columns are
|
||||
-- LONGTEXT on every clone (bd-hs7fa, second finding).
|
||||
--
|
||||
-- Synced migration 0049 widened description/design/acceptance_criteria/notes
|
||||
-- and close_reason from TEXT to LONGTEXT on issues AND wisps — but wisps is
|
||||
-- dolt-ignored, so a fresh clone materializes it from ignored/0001, whose
|
||||
-- CREATE still carries the original TEXT columns, and the at-latest main
|
||||
-- cursor means 0049 never runs there. TEXT caps at 65535 bytes: a wisp
|
||||
-- with an embedded-image description or a large agent payload that writes
|
||||
-- fine on an in-place-upgraded workspace fails with Error 1105 on a fresh
|
||||
-- clone. Same fresh-clone-door mechanism as ignored/0020 (storage_class)
|
||||
-- and ignored/0013 (row_lock); found by the shape audit in
|
||||
-- internal/storage/embeddeddolt/migrate_ignored_plane_shape_test.go.
|
||||
--
|
||||
-- Definitions mirror 0049's wisps section exactly, including the restated
|
||||
-- DEFAULTs (MODIFY COLUMN replaces the whole definition; dropping the
|
||||
-- defaults would regress inserts that omit the column). Guarded on the same
|
||||
-- COLUMN_TYPE = 'text' probe as 0049, so this is a no-op on
|
||||
-- in-place-upgraded workspaces and on the fresh-init door, and idempotent
|
||||
-- on replay.
|
||||
SET @wisps_needs_fix = (
|
||||
SELECT IF(COLUMN_TYPE = 'text', 1, 0)
|
||||
FROM INFORMATION_SCHEMA.COLUMNS
|
||||
WHERE TABLE_SCHEMA = DATABASE()
|
||||
AND TABLE_NAME = 'wisps'
|
||||
AND COLUMN_NAME = 'description'
|
||||
);
|
||||
SET @sql = IF(@wisps_needs_fix = 1,
|
||||
'ALTER TABLE wisps MODIFY COLUMN description LONGTEXT NOT NULL DEFAULT '''', MODIFY COLUMN design LONGTEXT NOT NULL DEFAULT '''', MODIFY COLUMN acceptance_criteria LONGTEXT NOT NULL DEFAULT '''', MODIFY COLUMN notes LONGTEXT NOT NULL DEFAULT ''''',
|
||||
'SELECT 1');
|
||||
PREPARE stmt FROM @sql; EXECUTE stmt; DEALLOCATE PREPARE stmt;
|
||||
|
||||
SET @wisps_cr_needs_fix = (
|
||||
SELECT IF(COLUMN_TYPE = 'text', 1, 0)
|
||||
FROM INFORMATION_SCHEMA.COLUMNS
|
||||
WHERE TABLE_SCHEMA = DATABASE()
|
||||
AND TABLE_NAME = 'wisps'
|
||||
AND COLUMN_NAME = 'close_reason'
|
||||
);
|
||||
SET @sql = IF(@wisps_cr_needs_fix = 1,
|
||||
'ALTER TABLE wisps MODIFY COLUMN close_reason LONGTEXT DEFAULT ''''',
|
||||
'SELECT 1');
|
||||
PREPARE stmt FROM @sql; EXECUTE stmt; DEALLOCATE PREPARE stmt;
|
||||
|
|
@ -23,10 +23,17 @@
|
|||
# exists on the base branch. New files only. Fix-forward with a new
|
||||
# migration instead: applied migration content is content-hashed
|
||||
# (PR 4270), so editing history creates cross-clone hash skew.
|
||||
# D. A new main-plane migration whose DDL touches a clone-local
|
||||
# (dolt_ignored) table must ship an ignored-series twin in the same
|
||||
# change. Clone-local tables are materialized on fresh clones by the
|
||||
# ignored series alone — the main cursor arrives at-latest, so a
|
||||
# main-plane ALTER silently never reaches them (bd-hs7fa: 0060's
|
||||
# wisps.storage_class missed every fresh clone; wy-pt82l before it:
|
||||
# 0054's wisps.row_lock, healed after the fact by ignored/0013).
|
||||
#
|
||||
# Check C compares against $BASE_SHA if set (CI passes the PR base), else
|
||||
# origin/main, else main; it is skipped with a warning when no base is
|
||||
# resolvable (e.g. shallow clone without the base commit).
|
||||
# Checks C and D compare against $BASE_SHA if set (CI passes the PR base),
|
||||
# else origin/main, else main; they are skipped with a warning when no base
|
||||
# is resolvable (e.g. shallow clone without the base commit).
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
|
|
@ -152,6 +159,56 @@ EOF
|
|||
fi
|
||||
fi
|
||||
|
||||
# --- Check D: main-plane DDL on clone-local tables needs an ignored twin -----
|
||||
# Fresh clones materialize the clone-local (dolt_ignored) tables from the
|
||||
# ignored series with the MAIN cursor already at-latest, so a main-plane
|
||||
# migration's DDL against one of them never executes there. The main-plane
|
||||
# migration is still required (it upgrades in-place workspaces, whose ignored
|
||||
# cursor is likewise at-latest); the twin is what carries the change through
|
||||
# the fresh-clone door. Data-only statements (UPDATE/INSERT/DELETE) don't
|
||||
# trigger this: clone-local data is clone-local by design.
|
||||
#
|
||||
# The table list mirrors internal/storage/schema/schema.go's
|
||||
# doltIgnorePatterns + versionGatedDoltIgnorePatterns. The engine-level
|
||||
# backstop for the same invariant is
|
||||
# internal/storage/embeddeddolt/migrate_ignored_plane_shape_test.go.
|
||||
if [ -z "$base" ] || [ -z "${merge_base:-}" ]; then
|
||||
echo "WARN (ignored twins) no usable base ref; skipping check D." >&2
|
||||
else
|
||||
ignored_tables='wisps|wisp_[a-z_]+|repo_mtimes|local_metadata|leases|events|ignored_schema_migrations'
|
||||
ddl_re="(alter|create|rename|drop)[[:space:]]+table([[:space:]]+if[[:space:]]+(not[[:space:]]+)?exists)?[[:space:]]+[\`']?(${ignored_tables})\b"
|
||||
# Untracked files count as added too (mirrors check C's working-tree scope:
|
||||
# in CI the tree is clean so this equals the PR diff).
|
||||
added_all=$(
|
||||
{
|
||||
git diff --name-only --diff-filter=A "$merge_base" -- "$MIG_DIR/*.sql" "$MIG_DIR/ignored/*.sql"
|
||||
git ls-files --others --exclude-standard -- "$MIG_DIR"
|
||||
} | grep '\.sql$' | sort -u || true
|
||||
)
|
||||
added_main=$(grep -v '/ignored/' <<<"$added_all" || true)
|
||||
added_ignored=$(grep '/ignored/' <<<"$added_all" || true)
|
||||
for f in $added_main; do
|
||||
[ -f "$f" ] || continue
|
||||
stripped=$(sed 's/--.*$//' "$f" | tr '[:upper:]' '[:lower:]')
|
||||
hits=$(grep -n -E "$ddl_re" <<<"$stripped" || true)
|
||||
if [ -n "$hits" ] && [ -z "$added_ignored" ]; then
|
||||
fail=1
|
||||
echo "FAIL (ignored twin missing) $f contains DDL against a clone-local table:"
|
||||
echo "$hits" | sed 's/^/ line /'
|
||||
cat <<'EOF'
|
||||
This table is dolt_ignored (clone-local): fresh clones build it from the
|
||||
ignored migration series with the main cursor already at-latest, so this
|
||||
DDL will silently never run there (bd-hs7fa). Ship a guarded twin under
|
||||
internal/storage/schema/migrations/ignored/ in the same PR (precedent:
|
||||
ignored/0013 for main 0054, ignored/0020 for main 0060). If the change is
|
||||
genuinely main-plane-only (e.g. the table is being flipped ONTO the
|
||||
ignored plane by this very migration), the twin is the migration that
|
||||
materializes the table for clones (precedent: 0062 + ignored/0019).
|
||||
EOF
|
||||
fi
|
||||
done
|
||||
fi
|
||||
|
||||
if [ "$fail" -ne 0 ]; then
|
||||
echo
|
||||
echo "Migration hygiene check failed. See docs in scripts/check-migration-hygiene.sh."
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue