diff --git a/.github/scripts/embedded-storage-test-shards.txt b/.github/scripts/embedded-storage-test-shards.txt index cef22090a..c3f45aacb 100644 --- a/.github/scripts/embedded-storage-test-shards.txt +++ b/.github/scripts/embedded-storage-test-shards.txt @@ -237,6 +237,7 @@ 40 10 TestGraphCounterContract 40 10 TestIgnoredCursorHealRespectsOperatorOverride 40 10 TestRunInTransactionPostCommitStageFailureIsIndeterminate +40 10 TestVersionedHistoryOffMatchesPreStackGolden_Embedded 40 10 TestWithPeerAuth_WarnsWhenStoredPeerSuppressesAmbientPasswordAmbient 40 11 TestABumpUnderOneStoreIDMovesTheEpochSeenThroughEveryStoreID diff --git a/BUILD.bazel b/BUILD.bazel index fed381d93..0117bc8b7 100644 --- a/BUILD.bazel +++ b/BUILD.bazel @@ -396,6 +396,7 @@ filegroup( "//internal/storage/externaldeps:repo_doc_files", "//internal/storage/fs:repo_doc_files", "//internal/storage/git:repo_doc_files", + "//internal/storage/historyoffequiv:repo_doc_files", "//internal/storage/issueops:repo_doc_files", "//internal/storage/journalscan:repo_doc_files", "//internal/storage/kvkeys:repo_doc_files", @@ -631,6 +632,7 @@ filegroup( "//internal/storage/externaldeps:repo_go_srcs", "//internal/storage/fs:repo_go_srcs", "//internal/storage/git:repo_go_srcs", + "//internal/storage/historyoffequiv:repo_go_srcs", "//internal/storage/issueops:repo_go_srcs", "//internal/storage/journalscan:repo_go_srcs", "//internal/storage/kvkeys:repo_go_srcs", @@ -851,6 +853,7 @@ filegroup( "//internal/storage/externaldeps:repo_go_test_srcs", "//internal/storage/fs:repo_go_test_srcs", "//internal/storage/git:repo_go_test_srcs", + "//internal/storage/historyoffequiv:repo_go_test_srcs", "//internal/storage/issueops:repo_go_test_srcs", "//internal/storage/journalscan:repo_go_test_srcs", "//internal/storage/kvkeys:repo_go_test_srcs", @@ -1080,6 +1083,7 @@ filegroup( "//internal/storage/externaldeps:repo_other_files", "//internal/storage/fs:repo_other_files", "//internal/storage/git:repo_other_files", + "//internal/storage/historyoffequiv:repo_other_files", "//internal/storage/issueops:repo_other_files", "//internal/storage/journalscan:repo_other_files", "//internal/storage/kvkeys:repo_other_files", diff --git a/internal/storage/BUILD.bazel b/internal/storage/BUILD.bazel index 73737e702..23a5a075f 100644 --- a/internal/storage/BUILD.bazel +++ b/internal/storage/BUILD.bazel @@ -170,6 +170,7 @@ filegroup( "//internal/storage/externaldeps:go_srcs", "//internal/storage/fs:go_srcs", "//internal/storage/git:go_srcs", + "//internal/storage/historyoffequiv:go_srcs", "//internal/storage/issueops:go_srcs", "//internal/storage/journalscan:go_srcs", "//internal/storage/kvkeys:go_srcs", diff --git a/internal/storage/dolt/BUILD.bazel b/internal/storage/dolt/BUILD.bazel index 773a53d6b..0ab26a75e 100644 --- a/internal/storage/dolt/BUILD.bazel +++ b/internal/storage/dolt/BUILD.bazel @@ -244,6 +244,7 @@ go_test( "gm2g3g5r_leak_repro_test.go", "graph_counter_contract_test.go", "heartbeater_contract_test.go", + "history_off_differential_test.go", "history_presence_contract_test.go", "history_test.go", "identity_mismatch_message_test.go", @@ -414,6 +415,7 @@ go_test( "//internal/storage/depid", "//internal/storage/doltutil", "//internal/storage/domain", + "//internal/storage/historyoffequiv", "//internal/storage/issueops", "//internal/storage/journalscan", "//internal/storage/rowid", diff --git a/internal/storage/dolt/history_off_differential_test.go b/internal/storage/dolt/history_off_differential_test.go new file mode 100644 index 000000000..f4bcefae9 --- /dev/null +++ b/internal/storage/dolt/history_off_differential_test.go @@ -0,0 +1,35 @@ +package dolt + +import ( + "context" + "database/sql" + "testing" + + "github.com/steveyegge/beads/internal/storage/historyoffequiv" +) + +// TestVersionedHistoryOffMatchesPreStackGolden_Dolt is the server leg's +// versioned-history flag-off differential (internal/storage/historyoffequiv): +// on a store whose versioned history was never turned on, the scripted +// mutation set must leave every table as the code before the +// versioned-history stack left it, apart from that stack's new columns and +// table staying NULL and empty. +func TestVersionedHistoryOffMatchesPreStackGolden_Dolt(t *testing.T) { + store, cleanup := setupConcurrentTestStore(t) + t.Cleanup(cleanup) + if err := store.SetConfig(t.Context(), "issue_prefix", historyoffequiv.Prefix); err != nil { + t.Fatalf("set issue_prefix: %v", err) + } + historyoffequiv.Run(t, historyoffequiv.Leg{ + Name: "server", + Roles: store, + Wake: func(ctx context.Context) error { + store.WakeExpiredDefersAdvisory(ctx) + return nil + }, + Extra: historyoffequiv.DoltStorageSteps(store), + Query: historyoffequiv.SQLQuery(func(context.Context) (*sql.DB, func(), error) { + return store.DB(), func() {}, nil + }), + }) +} diff --git a/internal/storage/domain/db/BUILD.bazel b/internal/storage/domain/db/BUILD.bazel index 0b7df5abb..22f93bff7 100644 --- a/internal/storage/domain/db/BUILD.bazel +++ b/internal/storage/domain/db/BUILD.bazel @@ -67,6 +67,7 @@ go_test( "issue_cross_tier_test.go", "issue_descendants_test.go", "issue_force_close_policy_test.go", + "issue_insert_history_off_test.go", "issue_mergeops_test.go", "issue_scan_parity_test.go", "issue_search_counts_missing_table_test.go", @@ -114,6 +115,7 @@ go_test( "//internal/storage/journalscan", "//internal/storage/schema", "//internal/storage/sqlbuild", + "//internal/storage/sqlcount", "//internal/testutil", "//internal/types", "//issueops", diff --git a/internal/storage/domain/db/issue_insert_history_off_test.go b/internal/storage/domain/db/issue_insert_history_off_test.go new file mode 100644 index 000000000..355cf5598 --- /dev/null +++ b/internal/storage/domain/db/issue_insert_history_off_test.go @@ -0,0 +1,143 @@ +package db + +import ( + "context" + "database/sql" + "database/sql/driver" + "fmt" + "sync/atomic" + "testing" + "time" + + sqlmock "github.com/DATA-DOG/go-sqlmock" + + "github.com/steveyegge/beads/internal/storage/domain" + "github.com/steveyegge/beads/internal/storage/issueops" + "github.com/steveyegge/beads/internal/storage/sqlcount" + "github.com/steveyegge/beads/internal/types" +) + +// TestInsertUpsertWithHistoryOffIssuesThePreStackStatements pins the +// statements Insert's upsert (CreateOnly unset: the quick, todo, gate and +// batch proxied servers' create) issues while versioned history is off. +// +// The participation_generation write fence (#6943) gave this path an +// existence probe (Exists) before the upsert, so the mint can tell a new row +// from one the upsert resolved to. With history off neither mint entry reads +// anything, so the probe is skipped and the create costs the round trips it +// did before the fence. Those are exactly the three statements below, the +// same three the code before the versioned-history stack (4a67d0d028) +// issues: the upsert, the created event's same-content scan, and its insert. +// The events journal is off too (the default); it adds its own statements +// whatever history does. +// +// Nothing else is scripted, and the mock is reached through sqlcount, which +// counts every statement Insert sends. So a probe, or any other read added +// ahead of the upsert or after it while history is off, fails the pin even +// when Insert swallows the error sqlmock answers it with: sqlmock's own +// checks see only an error the caller returns and expectations left unmet, +// and the count sees the rest. The history-on run of the same script is the +// control: there Insert reads (the probe, then the mint), so the pin has to +// refuse it, which shows that the script is exhaustive and that Insert reads +// the activation this transaction is scoped with. +func TestInsertUpsertWithHistoryOffIssuesThePreStackStatements(t *testing.T) { + for _, tc := range []struct { + name string + wisp bool + table, events string + }{ + {name: "issues table", table: "issues", events: "events"}, + {name: "wisps table", wisp: true, table: "wisps", events: "wisp_events"}, + } { + t.Run(tc.name, func(t *testing.T) { + if err := insertUpsertAgainstScript(t, tc.wisp, tc.table, tc.events, false); err != nil { + t.Fatalf("history off: %v", err) + } + }) + } + t.Run("history on is refused by the same script", func(t *testing.T) { + err := insertUpsertAgainstScript(t, false, "issues", "events", true) + if err == nil { + t.Fatal("Insert with history on passed the history-off script; " + + "the script no longer sees the reads history adds, so the history-off cases above prove nothing") + } + t.Logf("history on, as expected: %v", err) + }) +} + +// upsertStatements is how many statements the script below holds, and so +// how many the history-off upsert may send. +const upsertStatements = 3 + +// insertUpsertAgainstScript runs one upsert Insert into table on a mocked +// transaction scoped with versioned history as asked, against the script of +// the three statements the history-off upsert issues. It returns why the +// Insert broke the script, or nil if it sent exactly the scripted statements. +func insertUpsertAgainstScript(t *testing.T, wisp bool, table, events string, history bool) error { + t.Helper() + mock, tx, counts := countedMockTx(t) + // Every store and provider scopes each transaction with its own setting; + // a store whose switch is off scopes false. + t.Cleanup(issueops.ScopeVersionedHistoryTransaction(tx, history)) + + mock.ExpectExec(`^INSERT INTO ` + table + ` \( id, content_hash, .* ON DUPLICATE KEY UPDATE .* row_lock = VALUES\(row_lock\)$`). + WillReturnResult(sqlmock.NewResult(0, 1)) + mock.ExpectQuery(`^SELECT id FROM `+events+` WHERE issue_id = \? AND event_type = \? AND actor = \?`). + WithArgs("bd-1", string(types.EventCreated), "creator", sqlmock.AnyArg(), sqlmock.AnyArg(), sqlmock.AnyArg(), sqlmock.AnyArg()). + WillReturnRows(sqlmock.NewRows([]string{"id"})) + mock.ExpectExec(`^INSERT INTO ` + events + ` \(id, issue_id, event_type, actor, old_value, new_value, comment, created_at\)`). + WillReturnResult(sqlmock.NewResult(0, 1)) + + at := time.Date(2001, 2, 3, 4, 5, 6, 0, time.UTC) + if err := NewIssueSQLRepository(tx).Insert(t.Context(), &types.Issue{ + ID: "bd-1", Title: "upserted", Status: types.StatusOpen, Priority: 2, IssueType: types.TypeTask, + CreatedAt: at, UpdatedAt: at, + }, "creator", domain.InsertIssueOpts{UseWispsTable: wisp}); err != nil { + return fmt.Errorf("the upsert failed against the script: %w", err) + } + if n := counts.Total(); n != upsertStatements { + return fmt.Errorf("the upsert sent %d statements, want exactly the %d scripted; "+ + "sqlmock refused each extra one with an error Insert dropped", n, upsertStatements) + } + if err := mock.ExpectationsWereMet(); err != nil { + return fmt.Errorf("the upsert did not send the scripted statements: %w", err) + } + return nil +} + +var sqlmockSeq atomic.Int64 + +// countedMockTx begins a transaction on a fresh sqlmock connection that is +// reached through sqlcount, so a pin can count every statement the code +// under test sends, including one whose error it swallowed. +func countedMockTx(t *testing.T) (sqlmock.Sqlmock, *sql.Tx, *sqlcount.Counts) { + t.Helper() + dsn := fmt.Sprintf("domain_db_counted_%d", sqlmockSeq.Add(1)) + mockDB, mock, err := sqlmock.NewWithDSN(dsn) + if err != nil { + t.Fatalf("sqlmock.NewWithDSN: %v", err) + } + counts := &sqlcount.Counts{} + db := sql.OpenDB(sqlcount.WrapConnector(sqlmockConnector{dsn: dsn, driver: mockDB.Driver()}, counts)) + t.Cleanup(func() { + _ = db.Close() + _ = mockDB.Close() + }) + mock.ExpectBegin() + tx, err := db.BeginTx(t.Context(), nil) + if err != nil { + t.Fatalf("BeginTx: %v", err) + } + t.Cleanup(func() { _ = tx.Rollback() }) + return mock, tx, counts +} + +// sqlmockConnector opens the sqlmock connection registered under dsn, for +// sql.OpenDB through a wrapping connector. +type sqlmockConnector struct { + dsn string + driver driver.Driver +} + +func (c sqlmockConnector) Connect(context.Context) (driver.Conn, error) { return c.driver.Open(c.dsn) } +func (c sqlmockConnector) Driver() driver.Driver { return c.driver } diff --git a/internal/storage/embeddeddolt/BUILD.bazel b/internal/storage/embeddeddolt/BUILD.bazel index 0391b56e9..5be835710 100644 --- a/internal/storage/embeddeddolt/BUILD.bazel +++ b/internal/storage/embeddeddolt/BUILD.bazel @@ -162,6 +162,7 @@ go_test( "graph_counter_contract_test.go", "graph_walk_plan_test.go", "heartbeater_contract_test.go", + "history_off_differential_test.go", "history_presence_contract_test.go", "history_test.go", "is_blocked_batch_test.go", @@ -267,6 +268,7 @@ go_test( "//internal/storage/doltutil", "//internal/storage/domain/db", "//internal/storage/externaldeps", + "//internal/storage/historyoffequiv", "//internal/storage/issueops", "//internal/storage/schema", "//internal/storage/sqlcount", diff --git a/internal/storage/embeddeddolt/history_off_differential_test.go b/internal/storage/embeddeddolt/history_off_differential_test.go new file mode 100644 index 000000000..505620270 --- /dev/null +++ b/internal/storage/embeddeddolt/history_off_differential_test.go @@ -0,0 +1,39 @@ +//go:build cgo + +package embeddeddolt_test + +import ( + "context" + "database/sql" + "testing" + + "github.com/steveyegge/beads/internal/storage/embeddeddolt" + "github.com/steveyegge/beads/internal/storage/historyoffequiv" +) + +// TestVersionedHistoryOffMatchesPreStackGolden_Embedded is the embedded +// leg's versioned-history flag-off differential (internal/storage/historyoffequiv): +// on a store whose versioned history was never turned on, the scripted +// mutation set must leave every table as the code before the +// versioned-history stack left it, apart from that stack's new columns and +// table staying NULL and empty. +func TestVersionedHistoryOffMatchesPreStackGolden_Embedded(t *testing.T) { + skipUnlessEmbeddedDolt(t) + te := newTestEnv(t, historyoffequiv.Prefix) + historyoffequiv.Run(t, historyoffequiv.Leg{ + Name: "embedded", + Roles: te.store, + Wake: func(ctx context.Context) error { + te.store.WakeExpiredDefersAdvisory(ctx) + return nil + }, + Extra: historyoffequiv.DoltStorageSteps(te.store), + Query: historyoffequiv.SQLQuery(func(ctx context.Context) (*sql.DB, func(), error) { + db, cleanup, err := embeddeddolt.OpenSQL(ctx, te.dataDir, te.database, "main") + if err != nil { + return nil, nil, err + } + return db, func() { _ = cleanup() }, nil + }), + }) +} diff --git a/internal/storage/historyoffequiv/BUILD.bazel b/internal/storage/historyoffequiv/BUILD.bazel new file mode 100644 index 000000000..c9e4b9a8a --- /dev/null +++ b/internal/storage/historyoffequiv/BUILD.bazel @@ -0,0 +1,96 @@ +load("@rules_go//go:def.bzl", "go_library") + +go_library( + name = "historyoffequiv", + srcs = [ + "dump.go", + "historyoffequiv.go", + "scenario.go", + ], + embedsrcs = [ + "golden/embedded.json", + "golden/server.json", + "golden/uow.json", + ], + importpath = "github.com/steveyegge/beads/internal/storage/historyoffequiv", + visibility = ["//:__subpackages__"], + deps = [ + "//internal/storage", + "//internal/types", + "//issueops", + ], +) + +# --- begin go_srcs (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- + +filegroup( + name = "go_srcs", + srcs = glob( + ["**/*.go"], + allow_empty = True, + ), + visibility = ["//:__subpackages__"], +) + +# --- end go_srcs --- + +# --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- + +filegroup( + name = "repo_doc_files", + srcs = glob( + [ + "**/*.md", + "**/*.mdx", + ], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_srcs", + srcs = glob( + ["**/*.go"], + allow_empty = True, + exclude = [ + "**/*_test.go", + "**/node_modules/**", + ], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_test_srcs", + srcs = glob( + ["**/*_test.go"], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_other_files", + srcs = glob( + ["**"], + exclude = [ + "**/*.db", + "**/*.exe", + "**/*.go", + "**/*.md", + "**/*.mdx", + "**/*.out", + "**/*.prof", + "**/*.pyc", + "**/*.test", + "**/__pycache__/**", + "**/node_modules/**", + ], + ), + visibility = ["//:__subpackages__"], +) + +# --- end repo_files --- diff --git a/internal/storage/historyoffequiv/dump.go b/internal/storage/historyoffequiv/dump.go new file mode 100644 index 000000000..b5dbb36e9 --- /dev/null +++ b/internal/storage/historyoffequiv/dump.go @@ -0,0 +1,254 @@ +package historyoffequiv + +import ( + "context" + "encoding/json" + "fmt" + "regexp" + "strconv" + "strings" + "testing" + "time" +) + +// Every column of every base table is dumped: there is no list of tables to +// keep up to date, so a table a later change starts writing is compared the +// moment it exists. Each column is cast to text in SQL by its declared type +// (datetimes to a fixed format with microseconds, JSON to its text, binary to +// hex), and the text is then normalized only where a value is not a function +// of the scenario: +// +// - A datetime inside the run's window (the instants the scenario's writes +// stamp from the wall clock) reads "", and so does an RFC 3339 +// instant inside the window embedded in text (an event's snapshot of the +// row it changed). NULL stays NULL, and an instant the scenario supplied +// (fixedAt, decades outside the window) stays verbatim, so a write that +// starts or stops stamping a column, or that overwrites a supplied +// instant, still changes the digest. +// - A JSON value is re-serialized with sorted keys, so two engines' (or two +// versions') spacing and key order do not read as a difference. +// - The columns in volatileColumns hold values minted from randomness or +// derived from a wall-clock instant. Each reads "" under its +// rule, which keeps NULL, the empty string and the "none" value apart +// from a minted one. +// +// Rows are compared as a sorted multiset per table. + +type volatility int + +const ( + // volatileAlways masks every non-empty value. + volatileAlways volatility = iota + // volatileNonZero masks every value but "0", the column's "none". + volatileNonZero + // volatileUUID masks a UUID-shaped value and keeps any other verbatim. + volatileUUID +) + +// volatileColumns are the columns whose values differ between two runs of the +// same scenario on the same code, each with why. They were found by running +// the scenario twice on each leg and diffing the dumps. +var volatileColumns = map[string]map[string]volatility{ + // An event's id is a digest over its content and its wall-clock + // created_at (issueops.InsertDerivedEvent). + "events": {"id": volatileAlways}, + "wisp_events": {"id": volatileAlways}, + // A live comment's id is the same kind of digest; an imported comment + // keeps the id it arrived with, which stays verbatim. + "comments": {"id": volatileUUID}, + "wisp_comments": {"id": volatileUUID}, + // row_lock is a fresh random CAS token on every write; 0 means none. + "issues": {"row_lock": volatileNonZero}, + "wisps": {"row_lock": volatileNonZero}, + // Per-session coordination tokens (dependency-coordination/v1/..., + // issue-create/v1/...) are random; their keys are digests of the ids + // involved and stay verbatim. + "local_metadata": {"value": volatileAlways}, +} + +var ( + uuidRe = regexp.MustCompile(`^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$`) + rfc3339Re = regexp.MustCompile(`\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(\.\d+)?(Z|[+-]\d{2}:\d{2})`) +) + +func maskVolatile(table, column string, v string) (string, bool) { + rule, ok := volatileColumns[table][column] + if !ok || v == "" { + return v, false + } + switch rule { + case volatileNonZero: + if v == "0" { + return v, false + } + case volatileUUID: + if !uuidRe.MatchString(v) { + return v, false + } + } + return "", true +} + +// Two pseudo-tables carry the Dolt commit graph, compared like any table: the +// messages of the commits the scenario made, in order (a change to what a +// write commits, or whether it commits at all, shows up there), and the +// working set it left behind (a change to what a write stages shows up +// there). +const ( + doltLogTable = "@dolt_log" + doltStatusTable = "@dolt_status" +) + +func doltLogCount(t *testing.T, ctx context.Context, q Query) int { + t.Helper() + rows, err := q(ctx, "SELECT CAST(COUNT(*) AS CHAR) FROM dolt_log") + if err != nil { + t.Fatalf("count dolt_log: %v", err) + } + n, err := strconv.Atoi(*rows[0][0]) + if err != nil { + t.Fatalf("count dolt_log: %v", err) + } + return n +} + +func dumpAll(t *testing.T, ctx context.Context, q Query, windowStart, windowEnd time.Time, logBefore int) dump { + t.Helper() + tableRows, err := q(ctx, `SELECT table_name FROM information_schema.tables + WHERE table_schema = DATABASE() AND table_type = 'BASE TABLE' ORDER BY table_name`) + if err != nil { + t.Fatalf("list tables: %v", err) + } + colRows, err := q(ctx, `SELECT table_name, column_name, data_type FROM information_schema.columns + WHERE table_schema = DATABASE() ORDER BY table_name, ordinal_position`) + if err != nil { + t.Fatalf("list columns: %v", err) + } + type column struct{ name, dataType string } + columns := map[string][]column{} + for _, r := range colRows { + columns[*r[0]] = append(columns[*r[0]], column{name: *r[1], dataType: strings.ToLower(*r[2])}) + } + out := dump{Tables: map[string]tableDump{}} + for _, r := range tableRows { + table := *r[0] + if strings.HasPrefix(table, "dolt_") { + continue // Dolt's own system tables; the commit graph is read below. + } + cols := columns[table] + if len(cols) == 0 { + t.Fatalf("table %s has no columns in information_schema.columns", table) + } + exprs := make([]string, len(cols)) + names := make([]string, len(cols)) + for i, c := range cols { + names[i] = c.name + exprs[i] = textExpr(c.name, c.dataType) + } + rows, err := q(ctx, fmt.Sprintf("SELECT %s FROM `%s`", strings.Join(exprs, ", "), table)) + if err != nil { + t.Fatalf("dump %s: %v", table, err) + } + for _, row := range rows { + for i, c := range cols { + row[i] = normalize(t, table, c.name, c.dataType, row[i], windowStart, windowEnd) + } + } + sortRows(rows) + out.Tables[table] = tableDump{Columns: names, Rows: rows} + } + + // The scenario's commits, newest first as dolt_log lists them, numbered so + // the sorted comparison keeps their order. The committer identity comes + // from the environment (CI sets its own), so only the message is kept. + logRows, err := q(ctx, "SELECT message FROM dolt_log") + if err != nil { + t.Fatalf("read dolt_log: %v", err) + } + made := len(logRows) - logBefore + if made < 0 { + t.Fatalf("dolt_log shrank from %d to %d commits during the scenario", logBefore, len(logRows)) + } + var log [][]*string + for i, r := range logRows[:made] { + seq := fmt.Sprintf("%04d", made-i) + log = append(log, []*string{&seq, r[0]}) + } + sortRows(log) + out.Tables[doltLogTable] = tableDump{Columns: []string{"seq", "message"}, Rows: log} + + status, err := q(ctx, "SELECT table_name, CAST(staged AS CHAR), status FROM dolt_status") + if err != nil { + t.Fatalf("read dolt_status: %v", err) + } + sortRows(status) + out.Tables[doltStatusTable] = tableDump{Columns: []string{"table_name", "staged", "status"}, Rows: status} + return out +} + +func textExpr(name, dataType string) string { + c := "`" + name + "`" + switch dataType { + case "datetime", "timestamp": + return fmt.Sprintf("DATE_FORMAT(%s, '%%Y-%%m-%%d %%H:%%i:%%s.%%f')", c) + case "date": + return fmt.Sprintf("DATE_FORMAT(%s, '%%Y-%%m-%%d')", c) + case "blob", "tinyblob", "mediumblob", "longblob", "binary", "varbinary": + return fmt.Sprintf("HEX(%s)", c) + default: + return fmt.Sprintf("CAST(%s AS CHAR)", c) + } +} + +const datetimeLayout = "2006-01-02 15:04:05.000000" + +func normalize(t *testing.T, table, column, dataType string, v *string, windowStart, windowEnd time.Time) *string { + t.Helper() + if v == nil { + return nil + } + if masked, ok := maskVolatile(table, column, *v); ok { + return &masked + } + switch dataType { + case "datetime", "timestamp": + at, err := time.Parse(datetimeLayout, *v) + if err != nil { + t.Fatalf("%s.%s: datetime %q: %v", table, column, *v, err) + } + if !at.Before(windowStart) && !at.After(windowEnd) { + s := "" + return &s + } + case "json": + var parsed any + dec := json.NewDecoder(strings.NewReader(*v)) + dec.UseNumber() // keep integers past 2^53 exact + if err := dec.Decode(&parsed); err != nil { + return v + } + b, err := json.Marshal(parsed) + if err != nil { + t.Fatalf("%s.%s: re-marshal JSON: %v", table, column, err) + } + s := string(b) + return maskInstants(&s, windowStart, windowEnd) + } + return maskInstants(v, windowStart, windowEnd) +} + +// maskInstants replaces each RFC 3339 instant inside the run's window that v +// embeds with "". +func maskInstants(v *string, windowStart, windowEnd time.Time) *string { + if !rfc3339Re.MatchString(*v) { + return v + } + s := rfc3339Re.ReplaceAllStringFunc(*v, func(m string) string { + at, err := time.Parse(time.RFC3339Nano, m) + if err != nil || at.Before(windowStart) || at.After(windowEnd) { + return m + } + return "" + }) + return &s +} diff --git a/internal/storage/historyoffequiv/golden/embedded.json b/internal/storage/historyoffequiv/golden/embedded.json new file mode 100644 index 000000000..765849833 --- /dev/null +++ b/internal/storage/historyoffequiv/golden/embedded.json @@ -0,0 +1,539 @@ +{ + "recorded_from": "4a67d0d028", + "steps": [ + "create a", + "create b blocked by a", + "create epic", + "create epic.1 under epic", + "create wisp w1", + "create no-history nh", + "create c", + "create d", + "create e", + "update a fields labels metadata", + "update a no-op", + "update w1 to persistent", + "update c to ephemeral", + "update d parent epic", + "add dependencies", + "remove dependency b-\u003ea", + "comment on a and nh", + "claim heartbeat release b", + "claim e and reclaim no lease", + "metadata compare-and-set on a", + "close reopen close epic.1", + "close batch b and refused epic", + "force close blocked no-history nh", + "create batch", + "apply batch", + "defer f1 into the past and wake it", + "delete f2", + "create wisps x y z", + "update wisp x fields labels metadata", + "update wisp x no-op", + "claim and heartbeat refuse wisps", + "reclaim sweep scoped to in-progress wisp x", + "release wisp x", + "add and remove wisp dependencies", + "comment on wisp x", + "metadata compare-and-set on wisp x", + "close reopen wisp x", + "apply batch on the wisps plane", + "close batch wisps x and nh", + "defer wisp z into the past and wake it", + "delete wisp y", + "store create s1", + "store create issues s2 s3", + "store import upsert s1 and new s4", + "store update s2", + "store update s2 no-op", + "store labels", + "store dependencies", + "store comment", + "store claim s3", + "store close and reopen s2", + "store promote wisp sw", + "store transaction", + "store delete s3" + ], + "tables": { + "@dolt_log": { + "columns": [ + "seq", + "message" + ], + "rows": 28, + "sha256": "df2b8cf059f646557d21be3a0eca54f8561525f3d7e7ee714cfb65962d31012f" + }, + "@dolt_status": { + "columns": [ + "table_name", + "staged", + "status" + ], + "rows": 3, + "sha256": "ee3e710041f54216c2ae7e45bc7774cfcaa42aff3f4d96f5996fc4a7cc419f31" + }, + "bd_events_journal": { + "columns": [ + "seq", + "ts", + "op", + "issue_id", + "issue_json", + "dep_json", + "comment_json", + "actor" + ], + "rows": 0, + "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + }, + "bd_events_seq": { + "columns": [ + "id", + "next_seq" + ], + "rows": 1, + "sha256": "13751b39fe07c3673f53b4f2c5ef7f2a28b26c0daed72eb7332c93c1fe520618" + }, + "child_counters": { + "columns": [ + "parent_id", + "last_child" + ], + "rows": 1, + "sha256": "753b8feea7bdc0fc74468faa597a32984edc501e9873e9f4ee1df7b59d56d0a6" + }, + "comments": { + "columns": [ + "id", + "issue_id", + "author", + "text", + "created_at" + ], + "rows": 2, + "sha256": "d5870471fe72b8e50fe898978f24a80a23e07750ab538d47353acfabfb0e9aed" + }, + "compaction_snapshots": { + "columns": [ + "id", + "issue_id", + "compaction_level", + "snapshot_json", + "created_at" + ], + "rows": 0, + "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + }, + "config": { + "columns": [ + "key", + "value" + ], + "rows": 10, + "sha256": "671b33231eb47e4ad897051f4e8d86d3cca88685eeaa9ced537fe24c927fdea9" + }, + "custom_statuses": { + "columns": [ + "name", + "category" + ], + "rows": 0, + "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + }, + "custom_types": { + "columns": [ + "name" + ], + "rows": 0, + "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + }, + "dependencies": { + "columns": [ + "id", + "issue_id", + "type", + "created_at", + "created_by", + "metadata", + "thread_id", + "depends_on_issue_id", + "depends_on_wisp_id", + "depends_on_external" + ], + "rows": 6, + "sha256": "90f7dcd9895000df82d3b062d71a4c8ca81e7d5c9912f60d76515cb3eeee40d6" + }, + "events": { + "columns": [ + "issue_id", + "event_type", + "actor", + "old_value", + "new_value", + "comment", + "created_at", + "id" + ], + "rows": 51, + "sha256": "898f31694ce12dcd693f374634aea358b68239f6954fdcaae3b6342c1242f2bd" + }, + "federation_peers": { + "columns": [ + "name", + "remote_url", + "username", + "password_encrypted", + "sovereignty", + "last_sync", + "created_at", + "updated_at" + ], + "rows": 0, + "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + }, + "ignored_schema_migrations": { + "columns": [ + "version", + "applied_at", + "content_hash" + ], + "rows": 27, + "sha256": "7164db2c129cbaafbd9c426d27e809d043cafc77e8aef1f68d1760905a685acb" + }, + "interactions": { + "columns": [ + "id", + "kind", + "created_at", + "actor", + "issue_id", + "model", + "prompt", + "response", + "error", + "tool_name", + "exit_code", + "parent_id", + "label", + "reason", + "extra" + ], + "rows": 0, + "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + }, + "issue_counter": { + "columns": [ + "prefix", + "last_id" + ], + "rows": 0, + "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + }, + "issue_snapshots": { + "columns": [ + "id", + "issue_id", + "snapshot_time", + "compaction_level", + "original_size", + "compressed_size", + "original_content", + "archived_events" + ], + "rows": 0, + "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + }, + "issue_versions": { + "columns": [ + "issue_id", + "revision", + "epoch", + "durable_state", + "change_actor", + "change_agent", + "change_message", + "change_at", + "removed_at", + "removed_reason", + "attribution_status" + ], + "rows": 0, + "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + }, + "issues": { + "columns": [ + "id", + "content_hash", + "title", + "description", + "design", + "acceptance_criteria", + "notes", + "status", + "priority", + "issue_type", + "assignee", + "estimated_minutes", + "created_at", + "created_by", + "owner", + "updated_at", + "closed_at", + "closed_by_session", + "external_ref", + "spec_id", + "compaction_level", + "compacted_at", + "compacted_at_commit", + "original_size", + "sender", + "ephemeral", + "wisp_type", + "pinned", + "is_template", + "mol_type", + "work_type", + "source_system", + "metadata", + "source_repo", + "close_reason", + "event_kind", + "actor", + "target", + "payload", + "await_type", + "await_id", + "timeout_ns", + "waiters", + "hook_bead", + "role_bead", + "agent_state", + "last_activity", + "role_type", + "rig", + "due_at", + "defer_until", + "no_history", + "started_at", + "is_blocked", + "row_lock", + "storage_class", + "current_revision" + ], + "rows": 15, + "sha256": "1f81df40f4c4d458f20cfb6535ca13468fa03fbd10276c44752977813a4b7e20" + }, + "labels": { + "columns": [ + "issue_id", + "label" + ], + "rows": 8, + "sha256": "0102293e23eec630b67542f93c5d8da6eef5ca627cff8662970e7cb211b013ef" + }, + "leases": { + "columns": [ + "issue_id", + "holder", + "granted_at", + "lease_expires_at", + "heartbeat_at", + "granted_node" + ], + "rows": 1, + "sha256": "1eca894450e8e801850ed2cbf1c746775ad50868acaaf17e033a5da1e4be10aa" + }, + "local_metadata": { + "columns": [ + "key", + "value" + ], + "rows": 31, + "sha256": "c1d1cbc41ca9968c5c7dd380609c28126c4024868dbade26575ce105b8931fb3" + }, + "metadata": { + "columns": [ + "key", + "value" + ], + "rows": 0, + "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + }, + "provenance_events": { + "columns": [ + "id", + "issue_id", + "kind", + "actor", + "ref", + "ref_kind", + "payload", + "source", + "occurred_at", + "created_at" + ], + "rows": 0, + "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + }, + "repo_mtimes": { + "columns": [ + "repo_path", + "jsonl_path", + "mtime_ns", + "last_checked" + ], + "rows": 0, + "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + }, + "routes": { + "columns": [ + "prefix", + "path", + "created_at", + "updated_at" + ], + "rows": 0, + "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + }, + "schema_migrations": { + "columns": [ + "version", + "content_hash" + ], + "rows": 69, + "sha256": "1d7a6ae809f330f8b4c3516af50ce7604c1db3cd4c1177af3e5c2f1991d74e14" + }, + "store_epoch": { + "columns": [ + "id", + "epoch", + "bumped_at", + "bumped_reason" + ], + "rows": 0, + "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + }, + "wisp_child_counters": { + "columns": [ + "parent_id", + "last_child" + ], + "rows": 0, + "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + }, + "wisp_comments": { + "columns": [ + "id", + "issue_id", + "author", + "text", + "created_at" + ], + "rows": 2, + "sha256": "3c5f1e3eac9ff32fb1158c4421909ddd49c9131edff756c4a0e184114da93350" + }, + "wisp_dependencies": { + "columns": [ + "id", + "issue_id", + "depends_on_issue_id", + "depends_on_wisp_id", + "depends_on_external", + "type", + "created_at", + "created_by", + "metadata", + "thread_id" + ], + "rows": 2, + "sha256": "80f14f79545a236513ea88f0e873f3dd477f7763aaa8cbfaa8f659a517c2a909" + }, + "wisp_events": { + "columns": [ + "id", + "issue_id", + "event_type", + "actor", + "old_value", + "new_value", + "comment", + "created_at" + ], + "rows": 25, + "sha256": "1ee25446153c51b39c4de60d7f03bb56172a84e72d28df2dd44ff79aa5fc3b23" + }, + "wisp_labels": { + "columns": [ + "issue_id", + "label" + ], + "rows": 4, + "sha256": "f9c2698a90ceb3bf0ee8c4034af2845b54d65cc7bdd7638965a4682ecd104c37" + }, + "wisps": { + "columns": [ + "id", + "content_hash", + "title", + "description", + "design", + "acceptance_criteria", + "notes", + "status", + "priority", + "issue_type", + "assignee", + "estimated_minutes", + "created_at", + "created_by", + "owner", + "updated_at", + "closed_at", + "closed_by_session", + "external_ref", + "spec_id", + "compaction_level", + "compacted_at", + "compacted_at_commit", + "original_size", + "sender", + "ephemeral", + "wisp_type", + "pinned", + "is_template", + "mol_type", + "work_type", + "source_system", + "metadata", + "source_repo", + "close_reason", + "event_kind", + "actor", + "target", + "payload", + "await_type", + "await_id", + "timeout_ns", + "waiters", + "hook_bead", + "role_bead", + "agent_state", + "last_activity", + "role_type", + "rig", + "due_at", + "defer_until", + "no_history", + "started_at", + "row_lock", + "storage_class", + "current_revision", + "is_blocked" + ], + "rows": 6, + "sha256": "f28b157579dedd4373778dc6050b2d1995e1fe30e545e5c70c6b6ccf08ed75bf" + } + } +} diff --git a/internal/storage/historyoffequiv/golden/server.json b/internal/storage/historyoffequiv/golden/server.json new file mode 100644 index 000000000..73c11eed2 --- /dev/null +++ b/internal/storage/historyoffequiv/golden/server.json @@ -0,0 +1,539 @@ +{ + "recorded_from": "4a67d0d028", + "steps": [ + "create a", + "create b blocked by a", + "create epic", + "create epic.1 under epic", + "create wisp w1", + "create no-history nh", + "create c", + "create d", + "create e", + "update a fields labels metadata", + "update a no-op", + "update w1 to persistent", + "update c to ephemeral", + "update d parent epic", + "add dependencies", + "remove dependency b-\u003ea", + "comment on a and nh", + "claim heartbeat release b", + "claim e and reclaim no lease", + "metadata compare-and-set on a", + "close reopen close epic.1", + "close batch b and refused epic", + "force close blocked no-history nh", + "create batch", + "apply batch", + "defer f1 into the past and wake it", + "delete f2", + "create wisps x y z", + "update wisp x fields labels metadata", + "update wisp x no-op", + "claim and heartbeat refuse wisps", + "reclaim sweep scoped to in-progress wisp x", + "release wisp x", + "add and remove wisp dependencies", + "comment on wisp x", + "metadata compare-and-set on wisp x", + "close reopen wisp x", + "apply batch on the wisps plane", + "close batch wisps x and nh", + "defer wisp z into the past and wake it", + "delete wisp y", + "store create s1", + "store create issues s2 s3", + "store import upsert s1 and new s4", + "store update s2", + "store update s2 no-op", + "store labels", + "store dependencies", + "store comment", + "store claim s3", + "store close and reopen s2", + "store promote wisp sw", + "store transaction", + "store delete s3" + ], + "tables": { + "@dolt_log": { + "columns": [ + "seq", + "message" + ], + "rows": 42, + "sha256": "c43f56fef8a71aa0775d5c79d962b3bc1565305b2c18c3a670941891e376811f" + }, + "@dolt_status": { + "columns": [ + "table_name", + "staged", + "status" + ], + "rows": 1, + "sha256": "8658f682f824193bcb00dd2c9744be1e86b90a249374d368903303dfb62ac195" + }, + "bd_events_journal": { + "columns": [ + "seq", + "ts", + "op", + "issue_id", + "issue_json", + "dep_json", + "comment_json", + "actor" + ], + "rows": 0, + "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + }, + "bd_events_seq": { + "columns": [ + "id", + "next_seq" + ], + "rows": 1, + "sha256": "13751b39fe07c3673f53b4f2c5ef7f2a28b26c0daed72eb7332c93c1fe520618" + }, + "child_counters": { + "columns": [ + "parent_id", + "last_child" + ], + "rows": 1, + "sha256": "753b8feea7bdc0fc74468faa597a32984edc501e9873e9f4ee1df7b59d56d0a6" + }, + "comments": { + "columns": [ + "id", + "issue_id", + "author", + "text", + "created_at" + ], + "rows": 2, + "sha256": "d5870471fe72b8e50fe898978f24a80a23e07750ab538d47353acfabfb0e9aed" + }, + "compaction_snapshots": { + "columns": [ + "id", + "issue_id", + "compaction_level", + "snapshot_json", + "created_at" + ], + "rows": 0, + "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + }, + "config": { + "columns": [ + "key", + "value" + ], + "rows": 10, + "sha256": "671b33231eb47e4ad897051f4e8d86d3cca88685eeaa9ced537fe24c927fdea9" + }, + "custom_statuses": { + "columns": [ + "name", + "category" + ], + "rows": 0, + "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + }, + "custom_types": { + "columns": [ + "name" + ], + "rows": 0, + "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + }, + "dependencies": { + "columns": [ + "id", + "issue_id", + "type", + "created_at", + "created_by", + "metadata", + "thread_id", + "depends_on_issue_id", + "depends_on_wisp_id", + "depends_on_external" + ], + "rows": 6, + "sha256": "90f7dcd9895000df82d3b062d71a4c8ca81e7d5c9912f60d76515cb3eeee40d6" + }, + "events": { + "columns": [ + "issue_id", + "event_type", + "actor", + "old_value", + "new_value", + "comment", + "created_at", + "id" + ], + "rows": 51, + "sha256": "898f31694ce12dcd693f374634aea358b68239f6954fdcaae3b6342c1242f2bd" + }, + "federation_peers": { + "columns": [ + "name", + "remote_url", + "username", + "password_encrypted", + "sovereignty", + "last_sync", + "created_at", + "updated_at" + ], + "rows": 0, + "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + }, + "ignored_schema_migrations": { + "columns": [ + "version", + "applied_at", + "content_hash" + ], + "rows": 27, + "sha256": "7164db2c129cbaafbd9c426d27e809d043cafc77e8aef1f68d1760905a685acb" + }, + "interactions": { + "columns": [ + "id", + "kind", + "created_at", + "actor", + "issue_id", + "model", + "prompt", + "response", + "error", + "tool_name", + "exit_code", + "parent_id", + "label", + "reason", + "extra" + ], + "rows": 0, + "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + }, + "issue_counter": { + "columns": [ + "prefix", + "last_id" + ], + "rows": 0, + "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + }, + "issue_snapshots": { + "columns": [ + "id", + "issue_id", + "snapshot_time", + "compaction_level", + "original_size", + "compressed_size", + "original_content", + "archived_events" + ], + "rows": 0, + "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + }, + "issue_versions": { + "columns": [ + "issue_id", + "revision", + "epoch", + "durable_state", + "change_actor", + "change_agent", + "change_message", + "change_at", + "removed_at", + "removed_reason", + "attribution_status" + ], + "rows": 0, + "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + }, + "issues": { + "columns": [ + "id", + "content_hash", + "title", + "description", + "design", + "acceptance_criteria", + "notes", + "status", + "priority", + "issue_type", + "assignee", + "estimated_minutes", + "created_at", + "created_by", + "owner", + "updated_at", + "closed_at", + "closed_by_session", + "external_ref", + "spec_id", + "compaction_level", + "compacted_at", + "compacted_at_commit", + "original_size", + "sender", + "ephemeral", + "wisp_type", + "pinned", + "is_template", + "mol_type", + "work_type", + "source_system", + "metadata", + "source_repo", + "close_reason", + "event_kind", + "actor", + "target", + "payload", + "await_type", + "await_id", + "timeout_ns", + "waiters", + "hook_bead", + "role_bead", + "agent_state", + "last_activity", + "role_type", + "rig", + "due_at", + "defer_until", + "no_history", + "started_at", + "is_blocked", + "row_lock", + "storage_class", + "current_revision" + ], + "rows": 15, + "sha256": "1f81df40f4c4d458f20cfb6535ca13468fa03fbd10276c44752977813a4b7e20" + }, + "labels": { + "columns": [ + "issue_id", + "label" + ], + "rows": 8, + "sha256": "0102293e23eec630b67542f93c5d8da6eef5ca627cff8662970e7cb211b013ef" + }, + "leases": { + "columns": [ + "issue_id", + "holder", + "granted_at", + "lease_expires_at", + "heartbeat_at", + "granted_node" + ], + "rows": 1, + "sha256": "1eca894450e8e801850ed2cbf1c746775ad50868acaaf17e033a5da1e4be10aa" + }, + "local_metadata": { + "columns": [ + "key", + "value" + ], + "rows": 31, + "sha256": "c1d1cbc41ca9968c5c7dd380609c28126c4024868dbade26575ce105b8931fb3" + }, + "metadata": { + "columns": [ + "key", + "value" + ], + "rows": 0, + "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + }, + "provenance_events": { + "columns": [ + "id", + "issue_id", + "kind", + "actor", + "ref", + "ref_kind", + "payload", + "source", + "occurred_at", + "created_at" + ], + "rows": 0, + "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + }, + "repo_mtimes": { + "columns": [ + "repo_path", + "jsonl_path", + "mtime_ns", + "last_checked" + ], + "rows": 0, + "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + }, + "routes": { + "columns": [ + "prefix", + "path", + "created_at", + "updated_at" + ], + "rows": 0, + "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + }, + "schema_migrations": { + "columns": [ + "version", + "content_hash" + ], + "rows": 69, + "sha256": "1d7a6ae809f330f8b4c3516af50ce7604c1db3cd4c1177af3e5c2f1991d74e14" + }, + "store_epoch": { + "columns": [ + "id", + "epoch", + "bumped_at", + "bumped_reason" + ], + "rows": 0, + "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + }, + "wisp_child_counters": { + "columns": [ + "parent_id", + "last_child" + ], + "rows": 0, + "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + }, + "wisp_comments": { + "columns": [ + "id", + "issue_id", + "author", + "text", + "created_at" + ], + "rows": 2, + "sha256": "3c5f1e3eac9ff32fb1158c4421909ddd49c9131edff756c4a0e184114da93350" + }, + "wisp_dependencies": { + "columns": [ + "id", + "issue_id", + "depends_on_issue_id", + "depends_on_wisp_id", + "depends_on_external", + "type", + "created_at", + "created_by", + "metadata", + "thread_id" + ], + "rows": 2, + "sha256": "80f14f79545a236513ea88f0e873f3dd477f7763aaa8cbfaa8f659a517c2a909" + }, + "wisp_events": { + "columns": [ + "id", + "issue_id", + "event_type", + "actor", + "old_value", + "new_value", + "comment", + "created_at" + ], + "rows": 25, + "sha256": "1ee25446153c51b39c4de60d7f03bb56172a84e72d28df2dd44ff79aa5fc3b23" + }, + "wisp_labels": { + "columns": [ + "issue_id", + "label" + ], + "rows": 4, + "sha256": "f9c2698a90ceb3bf0ee8c4034af2845b54d65cc7bdd7638965a4682ecd104c37" + }, + "wisps": { + "columns": [ + "id", + "content_hash", + "title", + "description", + "design", + "acceptance_criteria", + "notes", + "status", + "priority", + "issue_type", + "assignee", + "estimated_minutes", + "created_at", + "created_by", + "owner", + "updated_at", + "closed_at", + "closed_by_session", + "external_ref", + "spec_id", + "compaction_level", + "compacted_at", + "compacted_at_commit", + "original_size", + "sender", + "ephemeral", + "wisp_type", + "pinned", + "is_template", + "mol_type", + "work_type", + "source_system", + "metadata", + "source_repo", + "close_reason", + "event_kind", + "actor", + "target", + "payload", + "await_type", + "await_id", + "timeout_ns", + "waiters", + "hook_bead", + "role_bead", + "agent_state", + "last_activity", + "role_type", + "rig", + "due_at", + "defer_until", + "no_history", + "started_at", + "row_lock", + "storage_class", + "current_revision", + "is_blocked" + ], + "rows": 6, + "sha256": "f28b157579dedd4373778dc6050b2d1995e1fe30e545e5c70c6b6ccf08ed75bf" + } + } +} diff --git a/internal/storage/historyoffequiv/golden/uow.json b/internal/storage/historyoffequiv/golden/uow.json new file mode 100644 index 000000000..91b2d1f84 --- /dev/null +++ b/internal/storage/historyoffequiv/golden/uow.json @@ -0,0 +1,531 @@ +{ + "recorded_from": "4a67d0d028", + "steps": [ + "create a", + "create b blocked by a", + "create epic", + "create epic.1 under epic", + "create wisp w1", + "create no-history nh", + "create c", + "create d", + "create e", + "update a fields labels metadata", + "update a no-op", + "update w1 to persistent", + "update c to ephemeral", + "update d parent epic", + "add dependencies", + "remove dependency b-\u003ea", + "comment on a and nh", + "claim heartbeat release b", + "claim e and reclaim no lease", + "metadata compare-and-set on a", + "close reopen close epic.1", + "close batch b and refused epic", + "force close blocked no-history nh", + "create batch", + "apply batch", + "defer f1 into the past and wake it", + "delete f2", + "create wisps x y z", + "update wisp x fields labels metadata", + "update wisp x no-op", + "claim and heartbeat refuse wisps", + "reclaim sweep scoped to in-progress wisp x", + "release wisp x", + "add and remove wisp dependencies", + "comment on wisp x", + "metadata compare-and-set on wisp x", + "close reopen wisp x", + "apply batch on the wisps plane", + "close batch wisps x and nh", + "defer wisp z into the past and wake it", + "delete wisp y", + "domain upsert new u1", + "domain upsert existing u1", + "domain upsert new wisp uw", + "domain upsert existing wisp uw", + "import upsert u1 and new u2" + ], + "tables": { + "@dolt_log": { + "columns": [ + "seq", + "message" + ], + "rows": 30, + "sha256": "5f4e90621c1c115e95223419c9c8c8533a342469e359beeac2ec5a4b4ff7ab1b" + }, + "@dolt_status": { + "columns": [ + "table_name", + "staged", + "status" + ], + "rows": 0, + "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + }, + "bd_events_journal": { + "columns": [ + "seq", + "ts", + "op", + "issue_id", + "issue_json", + "dep_json", + "comment_json", + "actor" + ], + "rows": 0, + "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + }, + "bd_events_seq": { + "columns": [ + "id", + "next_seq" + ], + "rows": 1, + "sha256": "13751b39fe07c3673f53b4f2c5ef7f2a28b26c0daed72eb7332c93c1fe520618" + }, + "child_counters": { + "columns": [ + "parent_id", + "last_child" + ], + "rows": 0, + "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + }, + "comments": { + "columns": [ + "id", + "issue_id", + "author", + "text", + "created_at" + ], + "rows": 2, + "sha256": "5b88b60c5a975432baff9ecfc6d0230e3bdc12275471a2db90c6f36b80ab4561" + }, + "compaction_snapshots": { + "columns": [ + "id", + "issue_id", + "compaction_level", + "snapshot_json", + "created_at" + ], + "rows": 0, + "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + }, + "config": { + "columns": [ + "key", + "value" + ], + "rows": 10, + "sha256": "671b33231eb47e4ad897051f4e8d86d3cca88685eeaa9ced537fe24c927fdea9" + }, + "custom_statuses": { + "columns": [ + "name", + "category" + ], + "rows": 0, + "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + }, + "custom_types": { + "columns": [ + "name" + ], + "rows": 0, + "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + }, + "dependencies": { + "columns": [ + "id", + "issue_id", + "type", + "created_at", + "created_by", + "metadata", + "thread_id", + "depends_on_issue_id", + "depends_on_wisp_id", + "depends_on_external" + ], + "rows": 6, + "sha256": "10891c2f5540db5ffec57100e87d971863a67023d963cda658c43c56ce368a69" + }, + "events": { + "columns": [ + "issue_id", + "event_type", + "actor", + "old_value", + "new_value", + "comment", + "created_at", + "id" + ], + "rows": 39, + "sha256": "372c6143415ba862a0d951695e2c942670fc0a24a3b828d6755df97dbb34ade1" + }, + "federation_peers": { + "columns": [ + "name", + "remote_url", + "username", + "password_encrypted", + "sovereignty", + "last_sync", + "created_at", + "updated_at" + ], + "rows": 0, + "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + }, + "ignored_schema_migrations": { + "columns": [ + "version", + "applied_at", + "content_hash" + ], + "rows": 27, + "sha256": "7164db2c129cbaafbd9c426d27e809d043cafc77e8aef1f68d1760905a685acb" + }, + "interactions": { + "columns": [ + "id", + "kind", + "created_at", + "actor", + "issue_id", + "model", + "prompt", + "response", + "error", + "tool_name", + "exit_code", + "parent_id", + "label", + "reason", + "extra" + ], + "rows": 0, + "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + }, + "issue_counter": { + "columns": [ + "prefix", + "last_id" + ], + "rows": 0, + "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + }, + "issue_snapshots": { + "columns": [ + "id", + "issue_id", + "snapshot_time", + "compaction_level", + "original_size", + "compressed_size", + "original_content", + "archived_events" + ], + "rows": 0, + "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + }, + "issue_versions": { + "columns": [ + "issue_id", + "revision", + "epoch", + "durable_state", + "change_actor", + "change_agent", + "change_message", + "change_at", + "removed_at", + "removed_reason", + "attribution_status" + ], + "rows": 0, + "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + }, + "issues": { + "columns": [ + "id", + "content_hash", + "title", + "description", + "design", + "acceptance_criteria", + "notes", + "status", + "priority", + "issue_type", + "assignee", + "estimated_minutes", + "created_at", + "created_by", + "owner", + "updated_at", + "closed_at", + "closed_by_session", + "external_ref", + "spec_id", + "compaction_level", + "compacted_at", + "compacted_at_commit", + "original_size", + "sender", + "ephemeral", + "wisp_type", + "pinned", + "is_template", + "mol_type", + "work_type", + "source_system", + "metadata", + "source_repo", + "close_reason", + "event_kind", + "actor", + "target", + "payload", + "await_type", + "await_id", + "timeout_ns", + "waiters", + "hook_bead", + "role_bead", + "agent_state", + "last_activity", + "role_type", + "rig", + "due_at", + "defer_until", + "no_history", + "started_at", + "is_blocked", + "row_lock", + "storage_class", + "current_revision" + ], + "rows": 12, + "sha256": "92e9eb8264175b0a6723941b3e946cd796afe4539cf9927798ce62800cab67ef" + }, + "labels": { + "columns": [ + "issue_id", + "label" + ], + "rows": 6, + "sha256": "d8cb82847c5ed046c6f264c12a51b1cc128f314fdc501aad3be0bd38cd3ec341" + }, + "leases": { + "columns": [ + "issue_id", + "holder", + "granted_at", + "lease_expires_at", + "heartbeat_at", + "granted_node" + ], + "rows": 1, + "sha256": "1eca894450e8e801850ed2cbf1c746775ad50868acaaf17e033a5da1e4be10aa" + }, + "local_metadata": { + "columns": [ + "key", + "value" + ], + "rows": 31, + "sha256": "c1d1cbc41ca9968c5c7dd380609c28126c4024868dbade26575ce105b8931fb3" + }, + "metadata": { + "columns": [ + "key", + "value" + ], + "rows": 0, + "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + }, + "provenance_events": { + "columns": [ + "id", + "issue_id", + "kind", + "actor", + "ref", + "ref_kind", + "payload", + "source", + "occurred_at", + "created_at" + ], + "rows": 0, + "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + }, + "repo_mtimes": { + "columns": [ + "repo_path", + "jsonl_path", + "mtime_ns", + "last_checked" + ], + "rows": 0, + "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + }, + "routes": { + "columns": [ + "prefix", + "path", + "created_at", + "updated_at" + ], + "rows": 0, + "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + }, + "schema_migrations": { + "columns": [ + "version", + "content_hash" + ], + "rows": 69, + "sha256": "1d7a6ae809f330f8b4c3516af50ce7604c1db3cd4c1177af3e5c2f1991d74e14" + }, + "store_epoch": { + "columns": [ + "id", + "epoch", + "bumped_at", + "bumped_reason" + ], + "rows": 0, + "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + }, + "wisp_child_counters": { + "columns": [ + "parent_id", + "last_child" + ], + "rows": 0, + "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + }, + "wisp_comments": { + "columns": [ + "id", + "issue_id", + "author", + "text", + "created_at" + ], + "rows": 2, + "sha256": "3c5f1e3eac9ff32fb1158c4421909ddd49c9131edff756c4a0e184114da93350" + }, + "wisp_dependencies": { + "columns": [ + "id", + "issue_id", + "depends_on_issue_id", + "depends_on_wisp_id", + "depends_on_external", + "type", + "created_at", + "created_by", + "metadata", + "thread_id" + ], + "rows": 2, + "sha256": "80f14f79545a236513ea88f0e873f3dd477f7763aaa8cbfaa8f659a517c2a909" + }, + "wisp_events": { + "columns": [ + "id", + "issue_id", + "event_type", + "actor", + "old_value", + "new_value", + "comment", + "created_at" + ], + "rows": 27, + "sha256": "34d262640e93ffdc948862667d4bc5f01a9f655bc22d8e11abdfc777731a5771" + }, + "wisp_labels": { + "columns": [ + "issue_id", + "label" + ], + "rows": 4, + "sha256": "f9c2698a90ceb3bf0ee8c4034af2845b54d65cc7bdd7638965a4682ecd104c37" + }, + "wisps": { + "columns": [ + "id", + "content_hash", + "title", + "description", + "design", + "acceptance_criteria", + "notes", + "status", + "priority", + "issue_type", + "assignee", + "estimated_minutes", + "created_at", + "created_by", + "owner", + "updated_at", + "closed_at", + "closed_by_session", + "external_ref", + "spec_id", + "compaction_level", + "compacted_at", + "compacted_at_commit", + "original_size", + "sender", + "ephemeral", + "wisp_type", + "pinned", + "is_template", + "mol_type", + "work_type", + "source_system", + "metadata", + "source_repo", + "close_reason", + "event_kind", + "actor", + "target", + "payload", + "await_type", + "await_id", + "timeout_ns", + "waiters", + "hook_bead", + "role_bead", + "agent_state", + "last_activity", + "role_type", + "rig", + "due_at", + "defer_until", + "no_history", + "started_at", + "row_lock", + "storage_class", + "current_revision", + "is_blocked" + ], + "rows": 7, + "sha256": "f6885bac56aee7d91b48fdf56c657128ff0b530fa186fabfb52bbf271a3bc09b" + } + } +} diff --git a/internal/storage/historyoffequiv/historyoffequiv.go b/internal/storage/historyoffequiv/historyoffequiv.go new file mode 100644 index 000000000..afcdfc7f2 --- /dev/null +++ b/internal/storage/historyoffequiv/historyoffequiv.go @@ -0,0 +1,528 @@ +// Package historyoffequiv is the versioned-history flag-off differential +// (gastownhall/beads#6132, #6135: "flag-off must remain byte-identical to +// today, proven by differential test"). +// +// Each storage leg (embedded Dolt, Dolt server, unit of work) runs one +// scripted mutation set through its own front doors on a store whose +// versioned history was never turned on, the default. Run then dumps every +// column of every table, digests each table, and compares the digests with +// the leg's golden. The goldens were RECORDED FROM THE CODE BEFORE THE +// VERSIONED-HISTORY STACK (origin/main 4a67d0d028, the parent of #6943). The +// schema added since, listed in schemaAdditions under the rule spelled out +// there, is the only difference the comparison tolerates; any other +// difference in any table is a flag-off behavior change and fails the test. +// +// # Maintaining the goldens +// +// Run records instead of checking when GoldenDirEnv is set. Recording and +// checking prepare a run the same way: each schemaAdditions entry the +// database has is checked under the rule and set aside before any table is +// digested. A golden recorded from any tree whose flag-off storage behavior +// matches the committed goldens is therefore byte-identical to them apart +// from recorded_from, and recording from the current tree is also how a +// failure is read: diff its full dump against one recorded from the base. +// +// - A migration whose schema flag-off leaves untouched (nullable columns +// that stay NULL, tables that stay empty) appends an entry to +// schemaAdditions in its own PR: those columns and tables, and its +// bookkeeping rows. The goldens do not change. +// - A deliberate change to what flag-off storage writes, including a +// migration that writes rows or values while history is off, +// re-records the affected legs from its PR's tree. The PR shows the +// golden diff, and that the full dumps differ from a recording of its +// base only where the change means them to. +// - A change to the scenario re-records every leg. Record it from +// 4a67d0d028 while the harness still builds there (copy this package +// and the three legs' test files into a worktree of that commit), so +// the new steps are anchored to the pre-stack code too; once it no +// longer builds there, from the PR's base. +// - A Dolt upgrade that changes only how the engine renders a stored +// value (a JSON or datetime cast) re-records the legs that run the new +// engine from the PR's tree, and the PR shows that the full dumps +// differ from a recording of its base, made with the old engine, only +// in that rendering. The embedded leg runs the engine go.mod pins; the +// server and unit-of-work legs run the dolt binary the test finds (CI: +// DOLT_VERSION in tools/bazel/dolt.bzl). +// +// recorded_from names the tree a golden was recorded from: a commit, or a +// commit and the PR on top of it. +package historyoffequiv + +import ( + "context" + "crypto/sha256" + "database/sql" + "embed" + "encoding/hex" + "encoding/json" + "fmt" + "os" + "path/filepath" + "slices" + "sort" + "strconv" + "strings" + "testing" + "time" +) + +// Prefix is the issue prefix every scenario id carries. A leg configures its +// store with it before Run. +const Prefix = "ho" + +// Query runs one read-only statement against the leg's database (the one its +// store writes) and returns every row, each value as text and nil for SQL +// NULL. Run casts every column to text in the statement itself, so a leg's +// driver never decides how a value is formatted. +type Query func(ctx context.Context, query string) ([][]*string, error) + +// Step is one mutation of the scenario, named for failure messages and for +// the golden's step list. +type Step struct { + Name string + Run func(ctx context.Context) error +} + +// Leg is what a backend hands Run. Its store is constructed the way the leg's +// tests construct one and is never handed SetVersionedHistoryEnabled, so +// "history off" is the default a store starts with rather than a switch +// turned off (FR-7). +type Leg struct { + // Name selects the golden: golden/.json. + Name string + // Roles is the leg's role-accessor surface. Every leg drives the same + // SharedSteps through it. + Roles Roles + // Wake runs the leg's lazy defer-wake sweep, the one write no role + // exposes. + Wake func(ctx context.Context) error + // Extra are the leg's own steps, run after the shared ones: the store + // legs' storage.DoltStorage methods (DoltStorageSteps), the unit-of-work + // leg's domain upsert. + Extra []Step + // Query reads the leg's database. + Query Query +} + +// GoldenDirEnv switches Run from checking to recording: set to a directory, +// Run writes the leg's golden there instead of comparing, with a full dump +// of the rows it digested beside it (.full.json.txt) for diffing two +// recordings by hand. RecordedFromEnv names the tree being recorded from; +// Run copies it into the golden. When to re-record is under "Maintaining the +// goldens" in the package doc. +const ( + GoldenDirEnv = "HISTORYOFFEQUIV_GOLDEN_DIR" + RecordedFromEnv = "HISTORYOFFEQUIV_RECORDED_FROM" +) + +// schemaAddition is schema added since the pre-stack code the goldens are +// anchored to that flag-off storage leaves untouched, grouped by the change +// that added it. +type schemaAddition struct { + // By names the change, for failure messages. + By string + // Columns are its nullable columns, by table. + Columns map[string][]string + // Tables are its tables. + Tables []string + // Migrations are the migration runner's bookkeeping rows for its own + // migrations, by bookkeeping table (each keyed by its version column). + Migrations map[string][]string +} + +// schemaAdditions is THE ONLY DIFFERENCE from the goldens this test +// tolerates. The rule, applied to every run before any table is digested, +// whether the run records a golden or checks one: +// +// 1. A listed column, where the database has it, must be NULL in every +// row; it is then left out of its table's dump, so the rest of every row +// still has to match the golden byte for byte. A listed column holding +// any value fails: with versioned history off nothing may write it. +// 2. A listed table, where the database has it, must be empty; it is then +// left out. +// 3. A listed bookkeeping row, where the database has it, must be there +// once; it is then left out. Every other row of a bookkeeping table is +// compared like any table's. +// +// After that, checking demands the golden's tables and columns exactly, and +// every table's digest over the golden's columns must equal the golden's. +// Nothing else is excused: not a value, not a row, not a row count. A later +// migration whose schema flag-off leaves untouched appends its own entry +// here (see "Maintaining the goldens" in the package doc). +var schemaAdditions = []schemaAddition{{ + By: "the versioned-history stack (#6943, #6661, #6664)", + Columns: map[string][]string{ + // Migration 0070: the participation_generation write fence (#6943). + // A create with history off leaves it NULL, indistinguishable from + // a legacy row (design ยง16.2b, FR-7). + "issues": {"participation_generation"}, + // Migration 0070 mirrors it, inert, on the wisps table (FR-8). + "wisps": {"participation_generation"}, + // Migration 0071 (#6661). + "issue_versions": {"removed_restriction"}, + }, + // Migration 0072 (#6664). + Tables: []string{"epoch_minted_addresses"}, + // 0070-0072 on the main series, and ignored/0028 (the wisps half of + // 0070) on the dolt_ignored series. + Migrations: map[string][]string{ + "schema_migrations": {"70", "71", "72"}, + "ignored_schema_migrations": {"28"}, + }, +}} + +// runWindow widens the span of the run on each side to bound the wall-clock +// instants the scenario's writes stamp. It is generous so that no time zone a +// DATETIME may be read back in, and no rounding of a sub-second instant, can +// put such an instant outside it. The scenario's own timestamps are decades +// away (fixedAt), so a column whose value moved from a scenario-supplied +// instant to "now" (or back) still shows up in the digest. +const runWindow = 36 * time.Hour + +// Run applies the shared scenario and the leg's extra steps, then checks +// every table against the leg's golden (or records it). +func Run(t *testing.T, leg Leg) { + t.Helper() + if leg.Name == "" || leg.Roles == nil || leg.Wake == nil || leg.Query == nil { + t.Fatalf("historyoffequiv.Run: leg %q needs a Name, Roles, Wake and Query", leg.Name) + } + ctx := context.Background() + before := doltLogCount(t, ctx, leg.Query) + start := time.Now().UTC() + steps := append(SharedSteps(leg.Roles, leg.Wake), leg.Extra...) + var names []string + for _, step := range steps { + if err := step.Run(ctx); err != nil { + t.Fatalf("step %q: %v", step.Name, err) + } + names = append(names, step.Name) + } + end := time.Now().UTC() + got := setAsideAdditions(t, dumpAll(t, ctx, leg.Query, start.Add(-runWindow), end.Add(runWindow), before)) + got.Steps = names + + if dir := os.Getenv(GoldenDirEnv); dir != "" { + if t.Failed() { + t.Fatalf("not recording golden/%s.json: the run breaks the schemaAdditions rule (above)", leg.Name) + } + writeGolden(t, dir, leg.Name, got) + return + } + checkGolden(t, leg.Name, got) +} + +// dump is every table's normalized rows. +type dump struct { + Steps []string + Tables map[string]tableDump +} + +type tableDump struct { + Columns []string + Rows [][]*string +} + +type golden struct { + RecordedFrom string `json:"recorded_from"` + Steps []string `json:"steps"` + Tables map[string]goldenTable `json:"tables"` +} + +type goldenTable struct { + Columns []string `json:"columns"` + Rows int `json:"rows"` + SHA256 string `json:"sha256"` +} + +//go:embed golden/*.json +var goldenFS embed.FS + +// setAsideAdditions applies the schemaAdditions rule to a run's dump: it +// checks every listed column, table and bookkeeping row the database has, +// reporting each breach, and returns the dump without them. Recording and +// checking both digest only what it returns. +func setAsideAdditions(t *testing.T, d dump) dump { + t.Helper() + columns := map[string]map[string]string{} // table -> column -> By + tables := map[string]string{} + migrations := map[string]map[string]string{} // table -> version -> By + for _, a := range schemaAdditions { + for table, cols := range a.Columns { + for _, c := range cols { + if columns[table] == nil { + columns[table] = map[string]string{} + } + columns[table][c] = a.By + } + } + for _, table := range a.Tables { + tables[table] = a.By + } + for table, versions := range a.Migrations { + for _, v := range versions { + if migrations[table] == nil { + migrations[table] = map[string]string{} + } + migrations[table][v] = a.By + } + } + } + + out := dump{Steps: d.Steps, Tables: map[string]tableDump{}} + for _, table := range sortedKeys(d.Tables) { + td := d.Tables[table] + if by, ok := tables[table]; ok { + if len(td.Rows) != 0 { + t.Errorf("table %s, added by %s, has %d rows with versioned history off, want none:\n%s", + table, by, len(td.Rows), formatRows(td.Columns, td.Rows)) + } + continue + } + var keep []int + for i, c := range td.Columns { + by, listed := columns[table][c] + if !listed { + keep = append(keep, i) + continue + } + for _, row := range td.Rows { + if row[i] != nil { + t.Errorf("%s.%s, added by %s, holds %q with versioned history off, want NULL in every row; row: %s", + table, c, by, *row[i], formatRow(row)) + } + } + } + kept := tableDump{Columns: make([]string, len(keep))} + for j, i := range keep { + kept.Columns[j] = td.Columns[i] + } + version := slices.Index(kept.Columns, "version") + if migrations[table] != nil && version < 0 { + t.Errorf("bookkeeping table %s has no version column: %v", table, td.Columns) + } + seen := map[string]int{} + for _, row := range td.Rows { + projected := make([]*string, len(keep)) + for j, i := range keep { + projected[j] = row[i] + } + if version >= 0 && projected[version] != nil { + if _, listed := migrations[table][*projected[version]]; listed { + seen[*projected[version]]++ + continue + } + } + kept.Rows = append(kept.Rows, projected) + } + for _, v := range sortedKeys(seen) { + if seen[v] != 1 { + t.Errorf("%s has %d rows for migration %s, added by %s, want at most 1", table, seen[v], v, migrations[table][v]) + } + } + sortRows(kept.Rows) + out.Tables[table] = kept + } + return out +} + +// checkGolden compares a set-aside dump with golden/.json. +func checkGolden(t *testing.T, leg string, got dump) { + t.Helper() + b, err := goldenFS.ReadFile("golden/" + leg + ".json") + if err != nil { + t.Fatalf("no golden for leg %q: record one with %s (see the package doc): %v", leg, GoldenDirEnv, err) + } + var want golden + if err := json.Unmarshal(b, &want); err != nil { + t.Fatalf("golden/%s.json: %v", leg, err) + } + from := fmt.Sprintf("golden/%s.json (recorded from %s)", leg, want.RecordedFrom) + if !slices.Equal(got.Steps, want.Steps) { + t.Fatalf("the scenario's steps differ from those %s ran, so its digests no longer describe this scenario; "+ + "re-record it (\"Maintaining the goldens\" in the package doc):\ngot: %q\nwant: %q", + from, got.Steps, want.Steps) + } + unexplained := "; a migration whose schema flag-off leaves untouched lists it in schemaAdditions, " + + "and a change to what flag-off writes re-records the goldens (\"Maintaining the goldens\" in the package doc)" + for _, table := range sortedKeys(got.Tables) { + if _, ok := want.Tables[table]; !ok { + t.Errorf("table %s is not in %s and not in schemaAdditions%s:\n%s", + table, from, unexplained, formatRows(got.Tables[table].Columns, got.Tables[table].Rows)) + } + } + for _, table := range sortedKeys(want.Tables) { + wt := want.Tables[table] + gt, ok := got.Tables[table] + if !ok { + t.Errorf("table %s is in %s but not in this database", table, from) + continue + } + index := map[string]int{} + for i, c := range gt.Columns { + index[c] = i + } + ok = true + for _, c := range wt.Columns { + if _, present := index[c]; !present { + t.Errorf("%s.%s is in %s but not in this database", table, c, from) + ok = false + } + } + for _, c := range gt.Columns { + if !slices.Contains(wt.Columns, c) { + t.Errorf("%s.%s is not in %s and not in schemaAdditions%s", table, c, from, unexplained) + ok = false + } + } + if !ok { + continue + } + projected := make([][]*string, 0, len(gt.Rows)) + for _, row := range gt.Rows { + p := make([]*string, len(wt.Columns)) + for j, c := range wt.Columns { + p[j] = row[index[c]] + } + projected = append(projected, p) + } + sortRows(projected) + if d := digest(projected); d != wt.SHA256 || len(projected) != wt.Rows { + t.Errorf("%s differs from %s: got %d rows %s, want %d rows %s; rows now (columns %v):\n%s", + table, from, len(projected), d, wt.Rows, wt.SHA256, wt.Columns, formatRows(wt.Columns, projected)) + } + } +} + +// writeGolden records a set-aside dump as golden/.json in dir. +func writeGolden(t *testing.T, dir, leg string, d dump) { + t.Helper() + g := golden{RecordedFrom: os.Getenv(RecordedFromEnv), Steps: d.Steps, Tables: map[string]goldenTable{}} + if g.RecordedFrom == "" { + t.Fatalf("%s is set but %s is not: name the tree this golden is recorded from", GoldenDirEnv, RecordedFromEnv) + } + full := map[string][]string{} + for table, td := range d.Tables { + g.Tables[table] = goldenTable{Columns: td.Columns, Rows: len(td.Rows), SHA256: digest(td.Rows)} + full[table] = append(full[table], strings.Join(td.Columns, " | ")) + for _, row := range td.Rows { + full[table] = append(full[table], formatRow(row)) + } + } + b, err := json.MarshalIndent(g, "", " ") + if err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(dir, leg+".json"), append(b, '\n'), 0o600); err != nil { + t.Fatal(err) + } + fb, _ := json.MarshalIndent(full, "", " ") + _ = os.WriteFile(filepath.Join(dir, leg+".full.json.txt"), fb, 0o600) +} + +func digest(rows [][]*string) string { + h := sha256.New() + for _, row := range rows { + h.Write([]byte(formatRow(row))) + h.Write([]byte{'\n'}) + } + return hex.EncodeToString(h.Sum(nil)) +} + +func formatRow(row []*string) string { + parts := make([]string, len(row)) + for i, v := range row { + if v == nil { + parts[i] = "NULL" + } else { + parts[i] = strconv.Quote(*v) + } + } + return strings.Join(parts, " ") +} + +func formatRows(columns []string, rows [][]*string) string { + lines := []string{" " + strings.Join(columns, " | ")} + for _, row := range rows { + lines = append(lines, " "+formatRow(row)) + } + return strings.Join(lines, "\n") +} + +func sortRows(rows [][]*string) { + sort.SliceStable(rows, func(i, j int) bool { return formatRow(rows[i]) < formatRow(rows[j]) }) +} + +func sortedKeys[V any](m map[string]V) []string { + keys := make([]string, 0, len(m)) + for k := range m { + keys = append(keys, k) + } + sort.Strings(keys) + return keys +} + +// SQLQuery adapts a leg that reads through database/sql. open returns a +// handle on the leg's database and a function that releases it. +func SQLQuery(open func(ctx context.Context) (*sql.DB, func(), error)) Query { + return func(ctx context.Context, query string) ([][]*string, error) { + db, release, err := open(ctx) + if err != nil { + return nil, err + } + defer release() + rows, err := db.QueryContext(ctx, query) + if err != nil { + return nil, err + } + defer rows.Close() + cols, err := rows.Columns() + if err != nil { + return nil, err + } + var out [][]*string + for rows.Next() { + raw := make([]sql.NullString, len(cols)) + dest := make([]any, len(cols)) + for i := range raw { + dest[i] = &raw[i] + } + if err := rows.Scan(dest...); err != nil { + return nil, err + } + row := make([]*string, len(cols)) + for i, v := range raw { + if v.Valid { + s := v.String + row[i] = &s + } + } + out = append(out, row) + } + return out, rows.Err() + } +} + +// AnyRows converts rows of driver values (a raw-SQL use case's result) to +// Query's shape. +func AnyRows(rows [][]any) [][]*string { + out := make([][]*string, 0, len(rows)) + for _, r := range rows { + row := make([]*string, len(r)) + for i, v := range r { + switch v := v.(type) { + case nil: + case []byte: + s := string(v) + row[i] = &s + case string: + s := v + row[i] = &s + default: + s := fmt.Sprint(v) + row[i] = &s + } + } + out = append(out, row) + } + return out +} diff --git a/internal/storage/historyoffequiv/scenario.go b/internal/storage/historyoffequiv/scenario.go new file mode 100644 index 000000000..90dadd04b --- /dev/null +++ b/internal/storage/historyoffequiv/scenario.go @@ -0,0 +1,539 @@ +package historyoffequiv + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "time" + + "github.com/steveyegge/beads/internal/storage" + "github.com/steveyegge/beads/internal/types" + publicops "github.com/steveyegge/beads/issueops" +) + +// Roles is the role-accessor surface all three legs offer. The shared +// scenario reaches storage only through it (and Leg.Wake), so each leg runs +// the scenario through its own implementations: the embedded store's and the +// server store's issueops-backed roles, and the unit-of-work provider's +// domain-backed ones. +type Roles interface { + IssueLifecycle() (publicops.Lifecycle, error) + BatchCreator() (publicops.BatchCreator, error) + BatchApplier() (publicops.BatchApplier, error) + BatchCloser() (publicops.BatchCloser, error) + DependencyEditor() (publicops.DependencyEditor, error) + Commenter() (publicops.Commenter, error) + IssueClaimer() (publicops.Claimer, error) + Heartbeater() (publicops.Heartbeater, error) + Releaser() (publicops.Releaser, error) + LeaseReclaimer() (publicops.LeaseReclaimer, error) + MetadataCAS() (publicops.MetadataCAS, error) + Deleter() (publicops.Deleter, error) +} + +// fixedAt is every timestamp the scenario supplies. It is decades outside +// any run's window, so a column that keeps it reads it verbatim in the dump +// and a column that loses it to the wall clock reads "". +var fixedAt = time.Date(2001, 2, 3, 4, 5, 6, 0, time.UTC) + +func id(suffix string) string { return Prefix + "-" + suffix } + +func newIssue(suffix, title string, issueType types.IssueType) *types.Issue { + return &types.Issue{ + ID: id(suffix), Title: title, Description: "about " + suffix, Status: types.StatusOpen, + Priority: 2, IssueType: issueType, CreatedAt: fixedAt, UpdatedAt: fixedAt, + } +} + +func set[T any](v T) publicops.Field[T] { return publicops.Field[T]{Set: true, Value: v} } + +func raw(s string) *json.RawMessage { + m := json.RawMessage(s) + return &m +} + +// SharedSteps is the scenario every leg runs: each write role on the issues +// plane, then each again on the wisps plane (an ephemeral wisp, and the +// no-history wisp nh along the way), including the paths the +// versioned-history stack rewired (single and batch creates, the persistence +// moves between planes, the defer wake), refusals that must write nothing (a +// batch close item close policy refuses), and no-op updates that must leave +// every row alone. On the wisps plane the lease roles refuse (Claim, +// Heartbeat) or find nothing (Reclaim), as their contracts say, and those +// steps check the refusal. Neither reclaim sweep finds an expired lease: a +// role cannot age one, so they cover the sweep's reads and commit, not a +// revert. +func SharedSteps(r Roles, wake func(context.Context) error) []Step { + create := func(req publicops.CreateRequest) func(context.Context) error { + req.Actor = "alice" + return func(ctx context.Context) error { + lifecycle, err := r.IssueLifecycle() + if err != nil { + return err + } + _, err = lifecycle.Create(ctx, req) + return err + } + } + update := func(issueID string, patch publicops.IssuePatch) func(context.Context) error { + return func(ctx context.Context) error { + lifecycle, err := r.IssueLifecycle() + if err != nil { + return err + } + _, err = lifecycle.Update(ctx, publicops.UpdateRequest{Actor: "bob", IssueID: issueID, Patch: patch}) + return err + } + } + + a := newIssue("a", "alpha", types.TypeTask) + a.Labels = []string{"one", "two"} + a.Metadata = json.RawMessage(`{"k":1,"s":"x"}`) + a.Notes = "first note" + b := newIssue("b", "bravo", types.TypeBug) + b.Priority = 1 + epic := newIssue("epic", "the epic", types.TypeEpic) + child := newIssue("epic.1", "epic child", types.TypeTask) + w1 := newIssue("w1", "a wisp", types.TypeTask) + w1.Ephemeral = true + w1.Labels = []string{"wl"} + nh := newIssue("nh", "no history", types.TypeTask) + nh.NoHistory = true + c := newIssue("c", "charlie", types.TypeFeature) + c.Labels = []string{"moving"} + d := newIssue("d", "delta", types.TypeTask) + e := newIssue("e", "echo", types.TypeTask) + // The wisps-plane block's wisps: x carries the field, lease, comment, + // metadata and close traffic, y the refused claim and the dependency, + // apply and delete traffic, z the defer wake. + wisp := func(suffix, title string) *types.Issue { + w := newIssue(suffix, title, types.TypeTask) + w.Ephemeral = true + return w + } + x := wisp("x", "x-ray") + x.Labels = []string{"xl", "xm"} + x.Metadata = json.RawMessage(`{"k":1}`) + y, z := wisp("y", "yankee"), wisp("z", "zulu") + + return []Step{ + {"create a", create(publicops.CreateRequest{Issue: a})}, + {"create b blocked by a", create(publicops.CreateRequest{Issue: b, + Dependencies: []publicops.CreateDependency{{TargetID: id("a"), Type: types.DepBlocks}}})}, + {"create epic", create(publicops.CreateRequest{Issue: epic})}, + {"create epic.1 under epic", create(publicops.CreateRequest{Issue: child, ParentID: id("epic")})}, + {"create wisp w1", create(publicops.CreateRequest{Issue: w1})}, + {"create no-history nh", create(publicops.CreateRequest{Issue: nh})}, + {"create c", create(publicops.CreateRequest{Issue: c})}, + {"create d", create(publicops.CreateRequest{Issue: d})}, + {"create e", create(publicops.CreateRequest{Issue: e})}, + + {"update a fields labels metadata", update(id("a"), publicops.IssuePatch{ + Title: set("alpha two"), + Priority: set(3), + Assignee: set("bob"), + Status: set(types.StatusInProgress), + AppendNotes: set("second note"), + Labels: publicops.LabelPatch{Add: []string{"three"}, Remove: []string{"two"}}, + Metadata: publicops.MetadataPatch{Set: map[string]json.RawMessage{"k": json.RawMessage(`2`)}}, + })}, + {"update a no-op", update(id("a"), publicops.IssuePatch{Title: set("alpha two")})}, + {"update w1 to persistent", update(id("w1"), publicops.IssuePatch{Persistence: set(types.PersistenceModePersistent)})}, + {"update c to ephemeral", update(id("c"), publicops.IssuePatch{Persistence: set(types.PersistenceModeEphemeral)})}, + {"update d parent epic", update(id("d"), publicops.IssuePatch{ParentID: set(id("epic"))})}, + + {"add dependencies", func(ctx context.Context) error { + editor, err := r.DependencyEditor() + if err != nil { + return err + } + _, err = editor.AddDependencies(ctx, publicops.AddDependenciesRequest{Actor: "carol", Edges: []publicops.DependencyEdge{ + {IssueID: id("d"), DependsOnID: id("b"), Type: types.DepBlocks}, + {IssueID: id("e"), DependsOnID: id("a"), Type: types.DepRelated}, + {IssueID: id("nh"), DependsOnID: id("d"), Type: types.DepBlocks}, + }}) + return err + }}, + {"remove dependency b->a", func(ctx context.Context) error { + editor, err := r.DependencyEditor() + if err != nil { + return err + } + _, err = editor.RemoveDependency(ctx, publicops.RemoveDependencyRequest{Actor: "carol", IssueID: id("b"), DependsOnID: id("a")}) + return err + }}, + {"comment on a and nh", func(ctx context.Context) error { + commenter, err := r.Commenter() + if err != nil { + return err + } + if _, err := commenter.AddComment(ctx, publicops.AddCommentRequest{Author: "carol", IssueID: id("a"), Text: "hello a"}); err != nil { + return err + } + _, err = commenter.AddComment(ctx, publicops.AddCommentRequest{Author: "carol", IssueID: id("nh"), Text: "hello nh"}) + return err + }}, + {"claim heartbeat release b", func(ctx context.Context) error { + claimer, err := r.IssueClaimer() + if err != nil { + return err + } + if _, err := claimer.Claim(ctx, publicops.ClaimRequest{Actor: "dave", IssueID: id("b")}); err != nil { + return fmt.Errorf("claim: %w", err) + } + heartbeater, err := r.Heartbeater() + if err != nil { + return err + } + if _, err := heartbeater.Heartbeat(ctx, publicops.HeartbeatRequest{Actor: "dave", IssueID: id("b")}); err != nil { + return fmt.Errorf("heartbeat: %w", err) + } + releaser, err := r.Releaser() + if err != nil { + return err + } + _, err = releaser.Release(ctx, publicops.ReleaseRequest{Actor: "dave", IssueID: id("b")}) + return err + }}, + {"claim e and reclaim no lease", func(ctx context.Context) error { + claimer, err := r.IssueClaimer() + if err != nil { + return err + } + if _, err := claimer.Claim(ctx, publicops.ClaimRequest{Actor: "erin", IssueID: id("e")}); err != nil { + return fmt.Errorf("claim: %w", err) + } + reclaimer, err := r.LeaseReclaimer() + if err != nil { + return err + } + // e's lease has not expired, so the sweep reverts nothing. + _, err = reclaimer.Reclaim(ctx, publicops.ReclaimRequest{Actor: "reaper", + Filter: publicops.ReclaimFilter{IDs: []string{id("e")}}}) + return err + }}, + {"metadata compare-and-set on a", func(ctx context.Context) error { + cas, err := r.MetadataCAS() + if err != nil { + return err + } + _, err = cas.CompareAndSetKey(ctx, publicops.CompareAndSetKeyRequest{Actor: "frank", IssueID: id("a"), Key: "cas", Value: raw(`"v1"`)}) + return err + }}, + {"close reopen close epic.1", func(ctx context.Context) error { + lifecycle, err := r.IssueLifecycle() + if err != nil { + return err + } + if _, err := lifecycle.Close(ctx, publicops.CloseRequest{Actor: "bob", IssueID: id("epic.1"), Reason: "done"}); err != nil { + return fmt.Errorf("close: %w", err) + } + if _, err := lifecycle.Reopen(ctx, publicops.ReopenRequest{Actor: "bob", IssueID: id("epic.1"), Reason: "not yet"}); err != nil { + return fmt.Errorf("reopen: %w", err) + } + _, err = lifecycle.Close(ctx, publicops.CloseRequest{Actor: "bob", IssueID: id("epic.1"), Reason: "done again"}) + return err + }}, + {"close batch b and refused epic", func(ctx context.Context) error { + closer, err := r.BatchCloser() + if err != nil { + return err + } + // b closes; epic keeps its open child d, so close policy refuses + // that item and the batch keeps its survivor. + _, err = closer.CloseBatch(ctx, publicops.CloseBatchRequest{Actor: "bob", Items: []publicops.BatchCloseItem{ + {IssueID: id("b"), Reason: "b done"}, {IssueID: id("epic"), Reason: "epic done"}, + }}) + return err + }}, + {"force close blocked no-history nh", func(ctx context.Context) error { + lifecycle, err := r.IssueLifecycle() + if err != nil { + return err + } + _, err = lifecycle.Close(ctx, publicops.CloseRequest{Actor: "bob", IssueID: id("nh"), Reason: "nh done", Force: true}) + return err + }}, + {"create batch", func(ctx context.Context) error { + creator, err := r.BatchCreator() + if err != nil { + return err + } + f1 := newIssue("f1", "foxtrot one", types.TypeTask) + f1.Labels = []string{"batch"} + f2 := newIssue("f2", "foxtrot two", types.TypeTask) + fw := newIssue("fw", "foxtrot wisp", types.TypeTask) + fw.Ephemeral = true + _, err = creator.CreateBatch(ctx, publicops.CreateBatchRequest{Actor: "gina", Items: []publicops.BatchCreateItem{ + {Issue: f1}, + {Issue: f2, Dependencies: []publicops.CreateDependency{{TargetID: id("f1"), Type: types.DepBlocks}}}, + {Issue: fw}, + }}) + return err + }}, + {"apply batch", func(ctx context.Context) error { + applier, err := r.BatchApplier() + if err != nil { + return err + } + g1 := newIssue("g1", "golf one", types.TypeTask) + g2 := newIssue("g2", "golf two", types.TypeBug) + g2.Labels = []string{"applied"} + _, err = applier.ApplyBatch(ctx, publicops.ApplyBatchRequest{Actor: "hank", Items: []publicops.ApplyItem{ + {Kind: publicops.ItemCreate, Create: &publicops.CreateItem{Key: "g1", Issue: g1}}, + {Kind: publicops.ItemCreate, Create: &publicops.CreateItem{Key: "g2", Issue: g2}}, + {Kind: publicops.ItemDepAdd, DepAdd: &publicops.DepAddItem{Source: publicops.Ref{Key: "g2"}, Target: publicops.Ref{Key: "g1"}, Type: types.DepBlocks}}, + {Kind: publicops.ItemUpdate, Update: &publicops.UpdateItem{Target: publicops.Ref{ID: id("d")}, Patch: publicops.IssuePatch{Assignee: set("hank")}}}, + {Kind: publicops.ItemClose, Close: &publicops.CloseItem{Target: publicops.Ref{Key: "g1"}, Reason: "applied"}}, + }}) + return err + }}, + {"defer f1 into the past and wake it", func(ctx context.Context) error { + past := fixedAt + if err := update(id("f1"), publicops.IssuePatch{Status: set(types.StatusDeferred), DeferUntil: set(&past)})(ctx); err != nil { + return fmt.Errorf("defer: %w", err) + } + return wake(ctx) + }}, + {"delete f2", func(ctx context.Context) error { + deleter, err := r.Deleter() + if err != nil { + return err + } + _, err = deleter.Delete(ctx, publicops.DeleteRequest{Actor: "ivan", IDs: []string{id("f2")}, Force: true}) + return err + }}, + + // The wisps plane, role by role. + {"create wisps x y z", func(ctx context.Context) error { + for _, w := range []*types.Issue{x, y, z} { + if err := create(publicops.CreateRequest{Issue: w})(ctx); err != nil { + return fmt.Errorf("create %s: %w", w.ID, err) + } + } + return nil + }}, + {"update wisp x fields labels metadata", update(id("x"), publicops.IssuePatch{ + Title: set("x-ray two"), + Priority: set(1), + Assignee: set("bob"), + Status: set(types.StatusInProgress), + AppendNotes: set("wisp note"), + Labels: publicops.LabelPatch{Add: []string{"xn"}, Remove: []string{"xm"}}, + Metadata: publicops.MetadataPatch{Set: map[string]json.RawMessage{"k": json.RawMessage(`2`)}}, + })}, + {"update wisp x no-op", update(id("x"), publicops.IssuePatch{Title: set("x-ray two")})}, + {"claim and heartbeat refuse wisps", func(ctx context.Context) error { + // The lease roles do not lease a wisp: Claim answers ErrNotFound, + // Heartbeat ErrNotClaimable, and neither may write. + claimer, err := r.IssueClaimer() + if err != nil { + return err + } + if _, err := claimer.Claim(ctx, publicops.ClaimRequest{Actor: "dave", IssueID: id("y")}); !errors.Is(err, publicops.ErrNotFound) { + return fmt.Errorf("claim wisp y = %v, want ErrNotFound", err) + } + heartbeater, err := r.Heartbeater() + if err != nil { + return err + } + if _, err := heartbeater.Heartbeat(ctx, publicops.HeartbeatRequest{Actor: "bob", IssueID: id("x")}); !errors.Is(err, publicops.ErrNotClaimable) { + return fmt.Errorf("heartbeat wisp x = %v, want ErrNotClaimable", err) + } + return nil + }}, + {"reclaim sweep scoped to in-progress wisp x", func(ctx context.Context) error { + reclaimer, err := r.LeaseReclaimer() + if err != nil { + return err + } + // The sweep only reads the issues table, so it reverts nothing. + _, err = reclaimer.Reclaim(ctx, publicops.ReclaimRequest{Actor: "reaper", + Filter: publicops.ReclaimFilter{IDs: []string{id("x")}}}) + return err + }}, + {"release wisp x", func(ctx context.Context) error { + releaser, err := r.Releaser() + if err != nil { + return err + } + _, err = releaser.Release(ctx, publicops.ReleaseRequest{Actor: "bob", IssueID: id("x")}) + return err + }}, + {"add and remove wisp dependencies", func(ctx context.Context) error { + editor, err := r.DependencyEditor() + if err != nil { + return err + } + if _, err := editor.AddDependencies(ctx, publicops.AddDependenciesRequest{Actor: "carol", Edges: []publicops.DependencyEdge{ + {IssueID: id("y"), DependsOnID: id("x"), Type: types.DepBlocks}, + {IssueID: id("x"), DependsOnID: id("a"), Type: types.DepRelated}, + }}); err != nil { + return fmt.Errorf("add: %w", err) + } + _, err = editor.RemoveDependency(ctx, publicops.RemoveDependencyRequest{Actor: "carol", IssueID: id("y"), DependsOnID: id("x")}) + return err + }}, + {"comment on wisp x", func(ctx context.Context) error { + commenter, err := r.Commenter() + if err != nil { + return err + } + _, err = commenter.AddComment(ctx, publicops.AddCommentRequest{Author: "carol", IssueID: id("x"), Text: "hello x"}) + return err + }}, + {"metadata compare-and-set on wisp x", func(ctx context.Context) error { + cas, err := r.MetadataCAS() + if err != nil { + return err + } + _, err = cas.CompareAndSetKey(ctx, publicops.CompareAndSetKeyRequest{Actor: "frank", IssueID: id("x"), Key: "cas", Value: raw(`"w1"`)}) + return err + }}, + {"close reopen wisp x", func(ctx context.Context) error { + lifecycle, err := r.IssueLifecycle() + if err != nil { + return err + } + if _, err := lifecycle.Close(ctx, publicops.CloseRequest{Actor: "bob", IssueID: id("x"), Reason: "x done"}); err != nil { + return fmt.Errorf("close: %w", err) + } + _, err = lifecycle.Reopen(ctx, publicops.ReopenRequest{Actor: "bob", IssueID: id("x"), Reason: "x not yet"}) + return err + }}, + {"apply batch on the wisps plane", func(ctx context.Context) error { + applier, err := r.BatchApplier() + if err != nil { + return err + } + h1 := wisp("h1", "hotel wisp") + h1.Labels = []string{"applied"} + _, err = applier.ApplyBatch(ctx, publicops.ApplyBatchRequest{Actor: "hank", Items: []publicops.ApplyItem{ + {Kind: publicops.ItemCreate, Create: &publicops.CreateItem{Key: "h1", Issue: h1}}, + {Kind: publicops.ItemDepAdd, DepAdd: &publicops.DepAddItem{Source: publicops.Ref{ID: id("y")}, Target: publicops.Ref{Key: "h1"}, Type: types.DepBlocks}}, + {Kind: publicops.ItemUpdate, Update: &publicops.UpdateItem{Target: publicops.Ref{ID: id("y")}, Patch: publicops.IssuePatch{Assignee: set("hank")}}}, + {Kind: publicops.ItemClose, Close: &publicops.CloseItem{Target: publicops.Ref{Key: "h1"}, Reason: "applied"}}, + }}) + return err + }}, + {"close batch wisps x and nh", func(ctx context.Context) error { + closer, err := r.BatchCloser() + if err != nil { + return err + } + // x closes; nh is already closed, so the batch closes x alone. + _, err = closer.CloseBatch(ctx, publicops.CloseBatchRequest{Actor: "bob", Items: []publicops.BatchCloseItem{ + {IssueID: id("x"), Reason: "x done again"}, {IssueID: id("nh"), Reason: "nh again"}, + }}) + return err + }}, + {"defer wisp z into the past and wake it", func(ctx context.Context) error { + past := fixedAt + if err := update(id("z"), publicops.IssuePatch{Status: set(types.StatusDeferred), DeferUntil: set(&past)})(ctx); err != nil { + return fmt.Errorf("defer: %w", err) + } + return wake(ctx) + }}, + {"delete wisp y", func(ctx context.Context) error { + deleter, err := r.Deleter() + if err != nil { + return err + } + _, err = deleter.Delete(ctx, publicops.DeleteRequest{Actor: "ivan", IDs: []string{id("y")}, Force: true}) + return err + }}, + } +} + +// DoltStorageSteps are the store legs' extra steps: the storage.DoltStorage +// methods most of bd's commands still write through, including the import +// upsert (CreateIssuesWithFullOptions) whose create-versus-update choice the +// stack rewired, promotion out of the wisps table, and RunInTransaction. +func DoltStorageSteps(s storage.DoltStorage) []Step { + return []Step{ + {"store create s1", func(ctx context.Context) error { + s1 := newIssue("s1", "sierra one", types.TypeTask) + s1.Labels = []string{"store"} + return s.CreateIssue(ctx, s1, "sam") + }}, + {"store create issues s2 s3", func(ctx context.Context) error { + return s.CreateIssues(ctx, []*types.Issue{newIssue("s2", "sierra two", types.TypeTask), newIssue("s3", "sierra three", types.TypeBug)}, "sam") + }}, + {"store import upsert s1 and new s4", func(ctx context.Context) error { + s1 := newIssue("s1", "sierra one imported", types.TypeTask) + s1.Labels = []string{"store", "imported"} + s1.UpdatedAt = fixedAt.Add(time.Hour) + s4 := newIssue("s4", "sierra four", types.TypeTask) + s4.Dependencies = []*types.Dependency{{IssueID: id("s4"), DependsOnID: id("s1"), Type: types.DepBlocks, CreatedAt: fixedAt}} + s4.Comments = []*types.Comment{{ID: "ho-imported-comment", IssueID: id("s4"), Author: "importer", Text: "imported", CreatedAt: fixedAt}} + sw := newIssue("sw", "sierra wisp", types.TypeTask) + sw.Ephemeral = true + return s.CreateIssuesWithFullOptions(ctx, []*types.Issue{s1, s4, sw}, "importer", + storage.BatchCreateOptions{SkipPrefixValidation: true}) + }}, + {"store update s2", func(ctx context.Context) error { + return s.UpdateIssue(ctx, id("s2"), map[string]interface{}{"title": "sierra two updated", "priority": 1, "assignee": "sam"}, "sam") + }}, + {"store update s2 no-op", func(ctx context.Context) error { + return s.UpdateIssue(ctx, id("s2"), map[string]interface{}{"title": "sierra two updated"}, "sam") + }}, + {"store labels", func(ctx context.Context) error { + if err := s.AddLabel(ctx, id("s2"), "added", "sam"); err != nil { + return err + } + return s.RemoveLabel(ctx, id("s1"), "store", "sam") + }}, + {"store dependencies", func(ctx context.Context) error { + if err := s.AddDependency(ctx, &types.Dependency{IssueID: id("s3"), DependsOnID: id("s2"), Type: types.DepBlocks}, "sam"); err != nil { + return err + } + return s.RemoveDependency(ctx, id("s4"), id("s1"), "sam") + }}, + {"store comment", func(ctx context.Context) error { + _, err := s.AddIssueComment(ctx, id("s3"), "sam", "store comment") + return err + }}, + {"store claim s3", func(ctx context.Context) error { + bulk, ok := s.(storage.BulkIssueStore) + if !ok { + return fmt.Errorf("%T is not a storage.BulkIssueStore", s) + } + return bulk.ClaimIssue(ctx, id("s3"), "tess") + }}, + {"store close and reopen s2", func(ctx context.Context) error { + if err := s.CloseIssue(ctx, id("s2"), "closed by store", "sam", "session-1"); err != nil { + return err + } + return s.ReopenIssue(ctx, id("s2"), "reopened by store", "sam") + }}, + {"store promote wisp sw", func(ctx context.Context) error { + bulk, ok := s.(storage.BulkIssueStore) + if !ok { + return fmt.Errorf("%T is not a storage.BulkIssueStore", s) + } + return bulk.PromoteFromEphemeral(ctx, id("sw"), "sam") + }}, + {"store transaction", func(ctx context.Context) error { + return s.RunInTransaction(ctx, "bd: historyoffequiv transaction", func(tx storage.Transaction) error { + t1 := newIssue("t1", "tango one", types.TypeTask) + if err := tx.CreateIssue(ctx, t1, "tx"); err != nil { + return err + } + if err := tx.UpdateIssue(ctx, id("t1"), map[string]interface{}{"description": "changed in tx"}, "tx"); err != nil { + return err + } + if err := tx.AddLabel(ctx, id("t1"), "tx", "tx"); err != nil { + return err + } + if err := tx.AddDependency(ctx, &types.Dependency{IssueID: id("t1"), DependsOnID: id("s1"), Type: types.DepRelated}, "tx"); err != nil { + return err + } + return tx.CloseIssue(ctx, id("s4"), "closed in tx", "tx", "") + }) + }}, + {"store delete s3", func(ctx context.Context) error { + return s.DeleteIssue(ctx, id("s3")) + }}, + } +} diff --git a/internal/storage/issueops/BUILD.bazel b/internal/storage/issueops/BUILD.bazel index 3ade280ae..094dd2847 100644 --- a/internal/storage/issueops/BUILD.bazel +++ b/internal/storage/issueops/BUILD.bazel @@ -210,6 +210,7 @@ go_test( "version_history_number_admission_test.go", "version_history_preenable_scan_test.go", "version_history_test.go", + "wake_defers_test.go", "wisp_routing_test.go", ], data = glob(["testdata/**"]) + [ @@ -228,6 +229,7 @@ go_test( "//internal/storage/journalscan", "//internal/storage/rowid", "//internal/storage/sqlbuild", + "//internal/storage/sqlcount", "//internal/types", "//internal/validation", "//issueops", diff --git a/internal/storage/issueops/wake_defers_test.go b/internal/storage/issueops/wake_defers_test.go new file mode 100644 index 000000000..fae1b596e --- /dev/null +++ b/internal/storage/issueops/wake_defers_test.go @@ -0,0 +1,127 @@ +package issueops + +import ( + "context" + "database/sql" + "database/sql/driver" + "fmt" + "slices" + "sync/atomic" + "testing" + + "github.com/DATA-DOG/go-sqlmock" + + "github.com/steveyegge/beads/internal/storage/sqlcount" +) + +// With versioned history off the mint is a no-op, so the sweep has no reason +// to look at an expired row before it wakes it, nor after. With history on it +// reads the woken row to mint its version; a later change that also checks +// each row against the mint before waking it (so that one unversionable row +// cannot abort the whole sweep) must keep that check free when history is +// off. +// +// The mock scripts exactly the statements a one-row sweep issues: the +// expired-row SELECT, the UPDATE, the event's same-content SELECT and its +// INSERT, and the empty wisps pass. Nothing else is expected, and the mock is +// reached through sqlcount, which counts every statement the sweep sends. So +// a sweep that loaded the row (or read its participation) before or after +// waking it fails the pin even if it swallowed the error sqlmock answers that +// statement with, which sqlmock's own checks would not see. The history-on +// run of the same script is the control: the mint's read must make the pin +// refuse it. +func TestWakeExpiredDefersWithHistoryOffLoadsNothingBeforeWaking(t *testing.T) { + if err := wakeAgainstScript(t, false); err != nil { + t.Fatalf("history off: %v", err) + } + t.Run("history on is refused by the same script", func(t *testing.T) { + err := wakeAgainstScript(t, true) + if err == nil { + t.Fatal("the sweep with history on passed the history-off script; " + + "the script no longer sees the reads history adds, so the history-off run above proves nothing") + } + t.Logf("history on, as expected: %v", err) + }) +} + +// wakeStatements is how many statements the script below holds, and so how +// many the history-off sweep may send. +const wakeStatements = 5 + +// wakeAgainstScript runs a one-row sweep on a mocked transaction scoped with +// versioned history as asked, against the script of the statements the +// history-off sweep issues. It returns why the sweep broke the script, or nil +// if it sent exactly the scripted statements and woke the scripted row. +func wakeAgainstScript(t *testing.T, history bool) error { + t.Helper() + mock, tx, counts := countedMockTx(t) + if history { + t.Cleanup(ScopeVersionedHistoryTransaction(tx, true)) + } + // Otherwise tx is not scoped for versioned history, which is how every + // transaction reads while it is off. + mock.ExpectQuery(`SELECT id FROM issues WHERE status = 'deferred'`). + WillReturnRows(sqlmock.NewRows([]string{"id"}).AddRow("bd-1")) + mock.ExpectExec(`UPDATE issues SET status = 'open', defer_until = NULL`). + WithArgs(sqlmock.AnyArg(), sqlmock.AnyArg(), "bd-1"). + WillReturnResult(sqlmock.NewResult(0, 1)) + mock.ExpectQuery(`SELECT id FROM events WHERE issue_id`). + WillReturnRows(sqlmock.NewRows([]string{"id"})) + mock.ExpectExec(`INSERT INTO events`). + WillReturnResult(sqlmock.NewResult(0, 1)) + mock.ExpectQuery(`SELECT id FROM wisps WHERE status = 'deferred'`). + WillReturnRows(sqlmock.NewRows([]string{"id"})) + + got, err := WakeExpiredDefersInTx(t.Context(), tx) + if err != nil { + return fmt.Errorf("the sweep failed against the script: %w", err) + } + if n := counts.Total(); n != wakeStatements { + return fmt.Errorf("the sweep sent %d statements, want exactly the %d scripted; "+ + "sqlmock refused each extra one with an error the sweep dropped", n, wakeStatements) + } + if err := mock.ExpectationsWereMet(); err != nil { + return fmt.Errorf("the sweep did not send the scripted statements: %w", err) + } + if !slices.Equal(got.Issues, []string{"bd-1"}) || len(got.Wisps) != 0 { + return fmt.Errorf("woken = issues %v, wisps %v, want [bd-1] and none", got.Issues, got.Wisps) + } + return nil +} + +var sqlmockSeq atomic.Int64 + +// countedMockTx begins a transaction on a fresh sqlmock connection that is +// reached through sqlcount, so a pin can count every statement the code +// under test sends, including one whose error it swallowed. +func countedMockTx(t *testing.T) (sqlmock.Sqlmock, *sql.Tx, *sqlcount.Counts) { + t.Helper() + dsn := fmt.Sprintf("issueops_counted_%d", sqlmockSeq.Add(1)) + mockDB, mock, err := sqlmock.NewWithDSN(dsn) + if err != nil { + t.Fatalf("sqlmock.NewWithDSN: %v", err) + } + counts := &sqlcount.Counts{} + db := sql.OpenDB(sqlcount.WrapConnector(sqlmockConnector{dsn: dsn, driver: mockDB.Driver()}, counts)) + t.Cleanup(func() { + _ = db.Close() + _ = mockDB.Close() + }) + mock.ExpectBegin() + tx, err := db.BeginTx(t.Context(), nil) + if err != nil { + t.Fatalf("BeginTx: %v", err) + } + t.Cleanup(func() { _ = tx.Rollback() }) + return mock, tx, counts +} + +// sqlmockConnector opens the sqlmock connection registered under dsn, for +// sql.OpenDB through a wrapping connector. +type sqlmockConnector struct { + dsn string + driver driver.Driver +} + +func (c sqlmockConnector) Connect(context.Context) (driver.Conn, error) { return c.driver.Open(c.dsn) } +func (c sqlmockConnector) Driver() driver.Driver { return c.driver } diff --git a/internal/storage/uow/BUILD.bazel b/internal/storage/uow/BUILD.bazel index 96c3d988c..19101d195 100644 --- a/internal/storage/uow/BUILD.bazel +++ b/internal/storage/uow/BUILD.bazel @@ -125,6 +125,7 @@ go_test( "external_doltserver_provider_test.go", "graph_counter_contract_test.go", "heartbeater_contract_test.go", + "history_off_differential_test.go", "importer_contract_test.go", "importer_uow_test.go", "issue_claimer_test.go", @@ -198,6 +199,7 @@ go_test( "//internal/storage/dbproxy/server", "//internal/storage/domain", "//internal/storage/domain/db", + "//internal/storage/historyoffequiv", "//internal/storage/issueops", "//internal/storage/schema", "//internal/testutil", diff --git a/internal/storage/uow/history_off_differential_test.go b/internal/storage/uow/history_off_differential_test.go new file mode 100644 index 000000000..aabe0eefe --- /dev/null +++ b/internal/storage/uow/history_off_differential_test.go @@ -0,0 +1,92 @@ +package uow + +import ( + "context" + "testing" + "time" + + "github.com/steveyegge/beads/internal/storage/domain" + "github.com/steveyegge/beads/internal/storage/historyoffequiv" + "github.com/steveyegge/beads/internal/types" + publicops "github.com/steveyegge/beads/issueops" +) + +// TestVersionedHistoryOffMatchesPreStackGolden_UOW is the unit-of-work leg's +// versioned-history flag-off differential (internal/storage/historyoffequiv): +// on a provider whose versioned history was never turned on, the scripted +// mutation set must leave every table as the code before the +// versioned-history stack left it, apart from that stack's new columns and +// table staying NULL and empty. +// +// Beyond the shared role scenario this leg drives the domain create that +// leaves CreateOnly unset (the quick, todo and batch proxied servers' path): +// it is an upsert, and the stack gave it an existence probe that must not run +// while history is off (see TestInsertUpsertWithHistoryOffIssuesThePreStackStatements +// in domain/db). The importer role's upsert of an existing row runs too. +func TestVersionedHistoryOffMatchesPreStackGolden_UOW(t *testing.T) { + ctx := context.Background() + provider := newUOWRoleFixtureProvider(t, ctx, historyoffequiv.Prefix) + roles, ok := provider.(historyoffequiv.Roles) + if !ok { + t.Fatalf("provider %T does not offer every role the scenario drives", provider) + } + fixedAt := time.Date(2001, 2, 3, 4, 5, 6, 0, time.UTC) + issue := func(id, title string) *types.Issue { + return &types.Issue{ID: id, Title: title, Description: "about " + id, Status: types.StatusOpen, + Priority: 2, IssueType: types.TypeTask, CreatedAt: fixedAt, UpdatedAt: fixedAt} + } + domainCreate := func(wisp bool, is *types.Issue) func(context.Context) error { + return func(ctx context.Context) error { + return RunTx(ctx, provider, func(ctx context.Context, uw UnitOfWork) (string, error) { + params := domain.CreateIssueParams{Issue: is, ExplicitID: is.ID} + var err error + if wisp { + _, err = uw.IssueUseCase().CreateWisp(ctx, params, "uma") + } else { + _, err = uw.IssueUseCase().CreateIssue(ctx, params, "uma") + } + return "bd: create " + is.ID, err + }) + } + } + historyoffequiv.Run(t, historyoffequiv.Leg{ + Name: "uow", + Roles: roles, + Wake: func(ctx context.Context) error { + _, err := WakeExpiredDefers(ctx, provider) + return err + }, + Extra: []historyoffequiv.Step{ + {Name: "domain upsert new u1", Run: domainCreate(false, issue("ho-u1", "uniform one"))}, + {Name: "domain upsert existing u1", Run: domainCreate(false, issue("ho-u1", "uniform one again"))}, + {Name: "domain upsert new wisp uw", Run: domainCreate(true, issue("ho-uw", "uniform wisp"))}, + {Name: "domain upsert existing wisp uw", Run: domainCreate(true, issue("ho-uw", "uniform wisp again"))}, + {Name: "import upsert u1 and new u2", Run: func(ctx context.Context) error { + importer, err := provider.(interface { + Importer() (publicops.Importer, error) + }).Importer() + if err != nil { + return err + } + u1 := issue("ho-u1", "uniform one imported") + u1.UpdatedAt = fixedAt.Add(time.Hour) + u2 := issue("ho-u2", "uniform two") + u2.Labels = []string{"imported"} + u2.Dependencies = []*types.Dependency{{IssueID: "ho-u2", DependsOnID: "ho-u1", Type: types.DepBlocks, CreatedAt: fixedAt}} + u2.Comments = []*types.Comment{{ID: "ho-u2-comment", IssueID: "ho-u2", Author: "importer", Text: "imported", CreatedAt: fixedAt}} + _, err = importer.ImportBatch(ctx, publicops.ImportBatchRequest{Actor: "importer", + Issues: []*types.Issue{u1, u2}, SkipPrefixValidation: true}) + return err + }}, + }, + Query: func(ctx context.Context, query string) ([][]*string, error) { + return RunTxRead(ctx, provider, func(ctx context.Context, uw UnitOfWork) ([][]*string, error) { + result, err := uw.RawSQLUseCase().Query(ctx, query) + if err != nil { + return nil, err + } + return historyoffequiv.AnyRows(result.Rows), nil + }) + }, + }) +} diff --git a/tools/bazel/BUILD.bazel b/tools/bazel/BUILD.bazel index 410728ae0..73e966ddc 100644 --- a/tools/bazel/BUILD.bazel +++ b/tools/bazel/BUILD.bazel @@ -259,6 +259,7 @@ release_cross_build( "//internal/storage/externaldeps:externaldeps", "//internal/storage/fs:fs", "//internal/storage/git:git", + "//internal/storage/historyoffequiv:historyoffequiv", "//internal/storage/issueops:issueops", "//internal/storage/journalscan:journalscan", "//internal/storage/kvkeys:kvkeys",