feat(events): typed journal truncation + retention floors (API-3.1)
A sequential journal read could not tell "nothing new" from "your prefix was
pruned": `WHERE seq > ?` answers both with an empty result. A consumer resuming
past a prune therefore either stalled forever on an empty success or silently
skipped to the current floor and dropped every record in between.
ReadEventsInTx now returns *storage.EventsJournalTruncatedError{Since, Floor,
Head} when the checkpoint sits below the retained window, and `bd events
tail/export` surface it as a machine-readable failure
({"code":"events_journal_truncated","since","floor","head"}) with a resume hint
in text mode. The decision lives in one place, ComputeEventsTruncation, so every
read plumbing answers a pruned-past checkpoint identically. The contiguous case
— the --follow poll — costs no extra query.
Both retention floors defaulted to 0, so a single `bd events prune --before
<large>` on a default-configured project deleted the whole journal and stranded
its consumer. They now default to 7 days / 100k rows; 0 still disables a floor
explicitly.
Also freezes the op vocabulary: seven journaled ops split into the six-op public
wire set plus engine-only `comment`, checkable from one place. The freeze is
guarded structurally — a new EventOp constant that nobody classifies fails the
test.
Drops the dead tailSelect duplicate of the read query, which would have bypassed
the truncation check had anything started calling it.
Epoch and project_id stay control-plane fields: no engine column, no DDL change.
This commit is contained in:
parent
3bfed2431b
commit
0a3fe0f9bd
11 changed files with 734 additions and 19 deletions
|
|
@ -3,9 +3,9 @@ package main
|
|||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"strconv"
|
||||
"time"
|
||||
|
||||
"github.com/spf13/cobra"
|
||||
|
|
@ -69,7 +69,14 @@ Record contract (stable for external consumers):
|
|||
comment object {"id","author","text","created_at","source"} for comment; omitted otherwise
|
||||
|
||||
Poll with the highest seq seen to consume new mutations incrementally, or pass
|
||||
--follow to keep printing new records as they are committed (Ctrl-C to stop).`,
|
||||
--follow to keep printing new records as they are committed (Ctrl-C to stop).
|
||||
|
||||
Retention boundary: if --since is below the oldest retained record — the prefix
|
||||
you asked for was pruned — the read FAILS instead of silently skipping ahead or
|
||||
returning an empty success. With --json the failure carries
|
||||
{"code":"events_journal_truncated","since":N,"floor":F,"head":H}: floor is the
|
||||
oldest seq still retained, head the highest ever assigned. Resume from floor-1
|
||||
to continue with a known gap, or rebuild from a full export.`,
|
||||
SilenceUsage: true,
|
||||
SilenceErrors: true,
|
||||
RunE: func(cmd *cobra.Command, _ []string) error {
|
||||
|
|
@ -152,15 +159,29 @@ type eventRecord struct {
|
|||
Comment json.RawMessage `json:"comment,omitempty"`
|
||||
}
|
||||
|
||||
// tailSelect builds the read query. CAST(ts AS CHAR) normalizes the DATETIME to
|
||||
// a string across drivers.
|
||||
func tailSelect(limit int) string {
|
||||
q := `SELECT seq, CAST(ts AS CHAR), op, issue_id, issue_json, dep_json, comment_json
|
||||
FROM bd_events_journal WHERE seq > ? ORDER BY seq ASC`
|
||||
if limit > 0 {
|
||||
q += " LIMIT " + strconv.Itoa(limit)
|
||||
// reportEventsTruncated renders a pruned-past checkpoint as a machine-readable
|
||||
// failure. A consumer must be able to branch on this without parsing prose, so
|
||||
// JSON mode carries the code and the window it can still serve; the caller then
|
||||
// decides whether to resume from floor-1 and accept the gap or rebuild.
|
||||
func reportEventsTruncated(err error) error {
|
||||
var trunc *storage.EventsJournalTruncatedError
|
||||
if !errors.As(err, &trunc) {
|
||||
return HandleErrorRespectJSON("reading events journal: %v", err)
|
||||
}
|
||||
return q
|
||||
if jsonOutput {
|
||||
if encErr := outputJSON(map[string]any{
|
||||
"error": trunc.Error(),
|
||||
"code": storage.EventsJournalTruncatedCode,
|
||||
"since": trunc.Since,
|
||||
"floor": trunc.Floor,
|
||||
"head": trunc.Head,
|
||||
}); encErr != nil {
|
||||
return encErr
|
||||
}
|
||||
return &exitError{Code: 1}
|
||||
}
|
||||
return HandleErrorWithHint(trunc.Error(),
|
||||
fmt.Sprintf("resume with --since %d to continue from the oldest retained record (accepting the gap), or re-import from scratch", trunc.Floor-1))
|
||||
}
|
||||
|
||||
func runEventsTail(ctx context.Context, since int64, limit int, follow bool) error {
|
||||
|
|
@ -183,7 +204,7 @@ func runEventsTail(ctx context.Context, since int64, limit int, follow bool) err
|
|||
|
||||
last, err := emit(since)
|
||||
if err != nil {
|
||||
return HandleErrorRespectJSON("reading events journal: %v", err)
|
||||
return reportEventsTruncated(err)
|
||||
}
|
||||
if !follow {
|
||||
return nil
|
||||
|
|
@ -199,7 +220,7 @@ func runEventsTail(ctx context.Context, since int64, limit int, follow bool) err
|
|||
return nil
|
||||
case <-ticker.C:
|
||||
if last, err = emit(last); err != nil {
|
||||
return HandleErrorRespectJSON("reading events journal: %v", err)
|
||||
return reportEventsTruncated(err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
102
cmd/bd/events_truncation_cli_test.go
Normal file
102
cmd/bd/events_truncation_cli_test.go
Normal file
|
|
@ -0,0 +1,102 @@
|
|||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"os"
|
||||
"os/exec"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/steveyegge/beads/internal/storage"
|
||||
)
|
||||
|
||||
// TestEventsTailReportsTruncationToTheCLI is the end-to-end guard for the
|
||||
// retention boundary as a consumer actually meets it: through the binary, not
|
||||
// through the store API.
|
||||
//
|
||||
// It exists because the store returning a typed error is only half the
|
||||
// contract — the CLI has two read call sites (the one-shot read and the
|
||||
// --follow poll) and wiring only one of them still lets `bd events export`
|
||||
// present a pruned journal as a complete history. A store-level test cannot see
|
||||
// that; this one fails if either path regresses.
|
||||
func TestEventsTailReportsTruncationToTheCLI(t *testing.T) {
|
||||
if os.Getenv("BEADS_TEST_EMBEDDED_DOLT") != "1" {
|
||||
t.Skip("set BEADS_TEST_EMBEDDED_DOLT=1 to run embedded dolt integration tests")
|
||||
}
|
||||
t.Parallel()
|
||||
|
||||
bd := buildEmbeddedBD(t)
|
||||
dir, _, _ := bdInit(t, bd, "--prefix", "evt")
|
||||
|
||||
// Disable both retention floors so the prune can actually reach the rows;
|
||||
// the shipped defaults are non-zero precisely to prevent this by accident.
|
||||
env := append(os.Environ(),
|
||||
"BD_EVENTS_JOURNAL=1",
|
||||
"BD_EVENTS_JOURNAL_RETAIN_DAYS=0",
|
||||
"BD_EVENTS_JOURNAL_RETAIN_ROWS=0",
|
||||
)
|
||||
run := func(args ...string) (string, error) {
|
||||
t.Helper()
|
||||
cmd := exec.Command(bd, args...)
|
||||
cmd.Dir = dir
|
||||
cmd.Env = env
|
||||
out, err := cmd.CombinedOutput()
|
||||
return string(out), err
|
||||
}
|
||||
|
||||
for _, title := range []string{"one", "two", "three", "four", "five"} {
|
||||
if out, err := run("create", title); err != nil {
|
||||
t.Fatalf("create %s: %v\n%s", title, err, out)
|
||||
}
|
||||
}
|
||||
if out, err := run("events", "prune", "--before", "4"); err != nil {
|
||||
t.Fatalf("prune: %v\n%s", err, out)
|
||||
}
|
||||
|
||||
// Both read commands must fail rather than serve the surviving suffix.
|
||||
for _, args := range [][]string{
|
||||
{"events", "export", "--json"},
|
||||
{"events", "tail", "--since", "0", "--json"},
|
||||
} {
|
||||
out, err := run(args...)
|
||||
if err == nil {
|
||||
t.Fatalf("%v succeeded on a pruned-past checkpoint; want a truncation failure\n%s", args, out)
|
||||
}
|
||||
var got struct {
|
||||
Code string `json:"code"`
|
||||
Since int64 `json:"since"`
|
||||
Floor int64 `json:"floor"`
|
||||
Head int64 `json:"head"`
|
||||
}
|
||||
if decErr := json.Unmarshal([]byte(firstJSONObject(out)), &got); decErr != nil {
|
||||
t.Fatalf("%v: output is not a JSON object: %v\n%s", args, decErr, out)
|
||||
}
|
||||
if got.Code != storage.EventsJournalTruncatedCode {
|
||||
t.Errorf("%v: code = %q, want %q\n%s", args, got.Code, storage.EventsJournalTruncatedCode, out)
|
||||
}
|
||||
if got.Since != 0 || got.Floor != 4 || got.Head != 5 {
|
||||
t.Errorf("%v: since/floor/head = %d/%d/%d, want 0/4/5\n%s", args, got.Since, got.Floor, got.Head, out)
|
||||
}
|
||||
}
|
||||
|
||||
// Resuming from the retained floor-1 still works and returns the surviving
|
||||
// records — a truncation must not be a dead end.
|
||||
out, err := run("events", "tail", "--since", "3")
|
||||
if err != nil {
|
||||
t.Fatalf("resume from floor-1: %v\n%s", err, out)
|
||||
}
|
||||
if lines := strings.Count(strings.TrimSpace(out), "\n") + 1; lines != 2 {
|
||||
t.Errorf("resume from floor-1 emitted %d records, want 2\n%s", lines, out)
|
||||
}
|
||||
}
|
||||
|
||||
// firstJSONObject extracts the JSON object from combined output, tolerating any
|
||||
// non-JSON preamble the binary may log around it.
|
||||
func firstJSONObject(out string) string {
|
||||
start := strings.Index(out, "{")
|
||||
end := strings.LastIndex(out, "}")
|
||||
if start < 0 || end < start {
|
||||
return out
|
||||
}
|
||||
return out[start : end+1]
|
||||
}
|
||||
|
|
@ -177,11 +177,17 @@ func Initialize() error {
|
|||
// enable per-workspace via `bd config set events-journal true` or
|
||||
// BD_EVENTS_JOURNAL=1. Read it with `bd events tail/export`.
|
||||
v.SetDefault("events-journal", false)
|
||||
// Retention floors for `bd events prune`. 0 disables a floor. retain-days
|
||||
// keeps rows younger than N days; retain-rows always keeps the newest N rows.
|
||||
// Retention floors for `bd events prune`. retain-days keeps rows younger
|
||||
// than N days; retain-rows always keeps the newest N rows. Both default ON:
|
||||
// a journal is only ever enabled because something is consuming it, and
|
||||
// with both floors at 0 a single `bd events prune --before <large>` deletes
|
||||
// the whole journal, stranding a consumer whose checkpoint is now below the
|
||||
// floor with no way to recover the lost span. The defaults buy a consumer a
|
||||
// week of downtime, or 100k mutations of backlog, whichever is larger.
|
||||
// Setting either to 0 explicitly disables that floor.
|
||||
// Env: BD_EVENTS_JOURNAL_RETAIN_DAYS / BD_EVENTS_JOURNAL_RETAIN_ROWS.
|
||||
v.SetDefault("events-journal-retain-days", 0)
|
||||
v.SetDefault("events-journal-retain-rows", 0)
|
||||
v.SetDefault("events-journal-retain-days", 7)
|
||||
v.SetDefault("events-journal-retain-rows", 100000)
|
||||
v.SetDefault("db", "")
|
||||
v.SetDefault("actor", "")
|
||||
v.SetDefault("issue-prefix", "")
|
||||
|
|
|
|||
|
|
@ -426,7 +426,9 @@ func TestEventsJournalAccessorServerStore(t *testing.T) {
|
|||
if n != 3 {
|
||||
t.Fatalf("prune retain-rows=2 deleted %d, want 3", n)
|
||||
}
|
||||
after, err := store.ReadEventsJournal(ctx, 0, 0)
|
||||
// Resume from the retained floor-1: reading from 0 now fails typed rather
|
||||
// than presenting the surviving suffix as a complete history.
|
||||
after, err := store.ReadEventsJournal(ctx, rows[2].Seq, 0)
|
||||
must(err, "read after")
|
||||
if len(after) != 2 {
|
||||
t.Fatalf("after prune %d rows, want 2", len(after))
|
||||
|
|
|
|||
|
|
@ -79,7 +79,10 @@ func TestEventsJournalAccessor(t *testing.T) {
|
|||
if n != 3 {
|
||||
t.Fatalf("prune with retain-rows=2 deleted %d, want 3 (keep newest 2)", n)
|
||||
}
|
||||
after, err := store.ReadEventsJournal(ctx, 0, 0)
|
||||
// Reading from the retained floor-1 returns the surviving window. Reading
|
||||
// from 0 now fails typed instead of presenting the suffix as a whole
|
||||
// history — see TestEventsJournalRestartAndRetentionBoundary.
|
||||
after, err := store.ReadEventsJournal(ctx, rows[2].Seq, 0)
|
||||
must(err, "read after prune")
|
||||
if len(after) != 2 {
|
||||
t.Fatalf("after prune %d rows, want 2", len(after))
|
||||
|
|
|
|||
144
internal/storage/embeddeddolt/journal_truncation_test.go
Normal file
144
internal/storage/embeddeddolt/journal_truncation_test.go
Normal file
|
|
@ -0,0 +1,144 @@
|
|||
//go:build cgo
|
||||
|
||||
package embeddeddolt_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"testing"
|
||||
|
||||
"github.com/steveyegge/beads/internal/storage"
|
||||
"github.com/steveyegge/beads/internal/types"
|
||||
)
|
||||
|
||||
// TestEventsJournalRestartAndRetentionBoundary is the AC03 end-to-end guard: an
|
||||
// exporter that restarts must resume the acknowledged contiguous prefix, and one
|
||||
// whose checkpoint was pruned past must be told so rather than silently skipping
|
||||
// to the current floor or receiving an empty success.
|
||||
//
|
||||
// It runs against the real embedded store — the same read path `bd events
|
||||
// tail --since` takes — so the counter self-heal, the prune floors, and the
|
||||
// truncation check are all exercised together rather than mocked.
|
||||
func TestEventsJournalRestartAndRetentionBoundary(t *testing.T) {
|
||||
env := newTestEnv(t, "trn")
|
||||
ctx := context.Background()
|
||||
store := env.store
|
||||
store.SetEventsJournalEnabled(true)
|
||||
|
||||
must := func(err error, what string) {
|
||||
t.Helper()
|
||||
if err != nil {
|
||||
t.Fatalf("%s: %v", what, err)
|
||||
}
|
||||
}
|
||||
create := func(id string) {
|
||||
t.Helper()
|
||||
must(store.CreateIssue(ctx, &types.Issue{
|
||||
ID: id, Title: "t-" + id, IssueType: types.TypeTask, Status: types.StatusOpen,
|
||||
}, "actor"), "create "+id)
|
||||
}
|
||||
|
||||
for _, id := range []string{"trn-1", "trn-2", "trn-3", "trn-4", "trn-5", "trn-6"} {
|
||||
create(id)
|
||||
}
|
||||
|
||||
all, err := store.ReadEventsJournal(ctx, 0, 0)
|
||||
must(err, "read all")
|
||||
if len(all) != 6 {
|
||||
t.Fatalf("read %d rows, want 6", len(all))
|
||||
}
|
||||
head := all[len(all)-1].Seq
|
||||
|
||||
// Restart: a consumer that acknowledged through row 3 resumes at 4 with no
|
||||
// loss and no error.
|
||||
ack := all[2].Seq
|
||||
resumed, err := store.ReadEventsJournal(ctx, ack, 0)
|
||||
must(err, "resume after restart")
|
||||
if len(resumed) != 3 || resumed[0].Seq != ack+1 {
|
||||
t.Fatalf("restart resume = %d rows starting at %d, want 3 starting at %d", len(resumed), resumed[0].Seq, ack+1)
|
||||
}
|
||||
|
||||
// Caught up: reading from the head is an empty success, never a truncation.
|
||||
tip, err := store.ReadEventsJournal(ctx, head, 0)
|
||||
must(err, "read at head")
|
||||
if len(tip) != 0 {
|
||||
t.Fatalf("read at head returned %d rows, want 0", len(tip))
|
||||
}
|
||||
|
||||
// Prune the first three rows. A consumer still at the acknowledged
|
||||
// watermark is exactly at the new floor-1 and must keep working.
|
||||
n, err := store.PruneEventsJournal(ctx, all[3].Seq, 0, 0)
|
||||
must(err, "prune")
|
||||
if n != 3 {
|
||||
t.Fatalf("pruned %d rows, want 3", n)
|
||||
}
|
||||
stillFine, err := store.ReadEventsJournal(ctx, ack, 0)
|
||||
must(err, "read at the retained floor")
|
||||
if len(stillFine) != 3 || stillFine[0].Seq != ack+1 {
|
||||
t.Fatalf("read at floor-1 = %d rows starting at %d, want 3 starting at %d", len(stillFine), stillFine[0].Seq, ack+1)
|
||||
}
|
||||
|
||||
// Pruned past: a checkpoint one below the floor-1 boundary must fail typed,
|
||||
// carrying the window the engine can still serve.
|
||||
_, err = store.ReadEventsJournal(ctx, ack-1, 0)
|
||||
trunc := requireTruncated(t, err)
|
||||
if trunc.Since != ack-1 {
|
||||
t.Errorf("Since = %d, want %d", trunc.Since, ack-1)
|
||||
}
|
||||
if trunc.Floor != all[3].Seq {
|
||||
t.Errorf("Floor = %d, want %d", trunc.Floor, all[3].Seq)
|
||||
}
|
||||
if trunc.Head != head {
|
||||
t.Errorf("Head = %d, want %d", trunc.Head, head)
|
||||
}
|
||||
|
||||
// `bd events export` (--since 0) over a pruned journal must not silently
|
||||
// present the surviving suffix as a complete history.
|
||||
_, err = store.ReadEventsJournal(ctx, 0, 0)
|
||||
if exported := requireTruncated(t, err); exported.Floor != all[3].Seq {
|
||||
t.Errorf("export Floor = %d, want %d", exported.Floor, all[3].Seq)
|
||||
}
|
||||
|
||||
// The limit path takes the same decision: a capped read from a pruned
|
||||
// checkpoint must not return a truncated window as success.
|
||||
_, err = store.ReadEventsJournal(ctx, 0, 2)
|
||||
requireTruncated(t, err)
|
||||
|
||||
// Prune everything. An empty journal is indistinguishable from "nothing
|
||||
// new" at the SQL level, so this is the case that would otherwise strand a
|
||||
// consumer forever on a poll loop.
|
||||
if _, err := store.PruneEventsJournal(ctx, head+1, 0, 0); err != nil {
|
||||
t.Fatalf("prune all: %v", err)
|
||||
}
|
||||
empty := requireTruncated(t, secondErr(store.ReadEventsJournal(ctx, ack, 0)))
|
||||
if empty.Floor != head+1 || empty.Head != head {
|
||||
t.Errorf("fully-pruned window = [%d..%d], want [%d..%d]", empty.Floor, empty.Head, head+1, head)
|
||||
}
|
||||
// A consumer that had already reached the head is caught up, not truncated.
|
||||
if _, err := store.ReadEventsJournal(ctx, head, 0); err != nil {
|
||||
t.Errorf("read at head of a fully pruned journal must succeed empty, got %v", err)
|
||||
}
|
||||
|
||||
// Seq never resets across a prune, so the next mutation continues the
|
||||
// history rather than colliding with a consumer's dedupe window.
|
||||
create("trn-7")
|
||||
next, err := store.ReadEventsJournal(ctx, head, 0)
|
||||
must(err, "read after post-prune create")
|
||||
if len(next) != 1 || next[0].Seq != head+1 {
|
||||
t.Fatalf("post-prune create = %+v, want a single row at seq %d", next, head+1)
|
||||
}
|
||||
}
|
||||
|
||||
func requireTruncated(t *testing.T, err error) *storage.EventsJournalTruncatedError {
|
||||
t.Helper()
|
||||
var trunc *storage.EventsJournalTruncatedError
|
||||
if err == nil {
|
||||
t.Fatal("expected a truncation error, got nil (silent skip or empty success)")
|
||||
}
|
||||
if !errors.As(err, &trunc) {
|
||||
t.Fatalf("error is not *EventsJournalTruncatedError: %T %v", err, err)
|
||||
}
|
||||
return trunc
|
||||
}
|
||||
|
||||
func secondErr(_ []storage.EventsJournalRow, err error) error { return err }
|
||||
|
|
@ -59,6 +59,42 @@ const (
|
|||
EventCommentWrite EventOp = "comment"
|
||||
)
|
||||
|
||||
// The engine journals SEVEN ops; the public event vocabulary is SIX. The
|
||||
// seventh, EventCommentWrite, is engine-only: the hosted exporter advances its
|
||||
// source cursor across a comment row without minting a wire event, so a comment
|
||||
// never reaches an external consumer as an event of its own (its effect is
|
||||
// already visible in the next issue snapshot). That reconciliation is the frozen
|
||||
// wire contract, not an implementation detail to be re-litigated per consumer.
|
||||
//
|
||||
// The split is stated here, once, so both halves are checkable from one place:
|
||||
// adding an op without deciding which side it lands on breaks
|
||||
// TestEventOpVocabularyIsFrozen.
|
||||
var (
|
||||
// WireEventOps is the closed public vocabulary, in canonical order. These
|
||||
// are the only ops that may appear in an event delivered to a consumer.
|
||||
wireEventOps = []EventOp{EventCreate, EventUpdate, EventClose, EventDelete, EventDepAdd, EventDepRemove}
|
||||
// engineOnlyEventOps are journaled but never minted as wire events.
|
||||
engineOnlyEventOps = []EventOp{EventCommentWrite}
|
||||
)
|
||||
|
||||
// WireEventOps returns the six-op public event vocabulary in canonical order.
|
||||
func WireEventOps() []EventOp { return append([]EventOp(nil), wireEventOps...) }
|
||||
|
||||
// EngineOnlyEventOps returns the journaled ops that mint no wire event.
|
||||
func EngineOnlyEventOps() []EventOp { return append([]EventOp(nil), engineOnlyEventOps...) }
|
||||
|
||||
// IsWireEventOp reports whether op is part of the public event vocabulary. A
|
||||
// journaled op that is not one is engine-only and must be skipped — not
|
||||
// dropped, not faulted on — by anything projecting the journal outward.
|
||||
func IsWireEventOp(op EventOp) bool {
|
||||
for _, w := range wireEventOps {
|
||||
if op == w {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// EventDep is the edge payload recorded for dependency operations.
|
||||
type EventDep struct {
|
||||
Kind string `json:"kind"`
|
||||
|
|
|
|||
|
|
@ -59,7 +59,81 @@ func BuildEventsPruneWhere(before int64, retainDays int, now time.Time, rowsCeil
|
|||
// ReadEventsInTx returns journal rows with seq greater than since, ordered by
|
||||
// seq ascending. limit > 0 caps the result. It runs inside the caller's
|
||||
// transaction so it works on every store plumbing (embedded, server, proxied).
|
||||
//
|
||||
// It returns *storage.EventsJournalTruncatedError when since sits below the
|
||||
// retained window — see checkEventsTruncation for why that is a hard failure
|
||||
// rather than a silent skip.
|
||||
func ReadEventsInTx(ctx context.Context, tx DBTX, since int64, limit int) ([]storage.EventsJournalRow, error) {
|
||||
out, err := readEventsRowsInTx(ctx, tx, since, limit)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := ComputeEventsTruncation(since, out, func() (int64, error) {
|
||||
return readEventsHeadInTx(ctx, tx)
|
||||
}); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// ComputeEventsTruncation decides whether the rows a read returned are the
|
||||
// caller's contiguous continuation from since, or the remains of a window whose
|
||||
// prefix a prune already deleted. It returns *storage.EventsJournalTruncatedError
|
||||
// in the latter case and nil otherwise.
|
||||
//
|
||||
// This is the ONE place the decision lives, so every read plumbing that serves
|
||||
// a `--since` — the DBTX path and anything projecting the journal outward —
|
||||
// answers a pruned-past checkpoint identically. readHead is only invoked in the
|
||||
// ambiguous cases below.
|
||||
//
|
||||
// Seqs are gapless by construction (see nextEventSeq) and prune only ever
|
||||
// deletes a prefix (`seq < before`), so the only hole a reader can observe is a
|
||||
// missing prefix. That makes the common case free: when the first row returned
|
||||
// is exactly since+1 the read is contiguous and no extra query runs — which
|
||||
// matters because `bd events tail --follow` calls this every second.
|
||||
//
|
||||
// Only the two ambiguous shapes cost a counter read:
|
||||
// - rows start above since+1: the prefix was pruned, floor is the first row.
|
||||
// - no rows at all: either genuinely caught up (since >= head) or the whole
|
||||
// journal was pruned out from under the caller (since < head), which is
|
||||
// indistinguishable from "nothing new" at the SQL level and is exactly the
|
||||
// silent-loss case this exists to catch.
|
||||
func ComputeEventsTruncation(since int64, rows []storage.EventsJournalRow, readHead func() (int64, error)) error {
|
||||
if len(rows) > 0 && rows[0].Seq == since+1 {
|
||||
return nil
|
||||
}
|
||||
head, err := readHead()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if len(rows) > 0 {
|
||||
return &storage.EventsJournalTruncatedError{Since: since, Floor: rows[0].Seq, Head: head}
|
||||
}
|
||||
if since >= head {
|
||||
return nil
|
||||
}
|
||||
// Empty journal: the floor is one past the head — nothing is retained, and
|
||||
// the caller's checkpoint is provably behind it.
|
||||
return &storage.EventsJournalTruncatedError{Since: since, Floor: head + 1, Head: head}
|
||||
}
|
||||
|
||||
// readEventsHeadInTx returns the highest seq the counter has ever assigned.
|
||||
// Prune never touches the counter, so this is the head of the journal's history
|
||||
// even when every row has been deleted. A missing counter row means no mutation
|
||||
// has ever been journaled here, which is a head of 0.
|
||||
func readEventsHeadInTx(ctx context.Context, tx DBTX) (int64, error) {
|
||||
var head int64
|
||||
err := tx.QueryRowContext(ctx, "SELECT next_seq FROM bd_events_seq WHERE id = 0").Scan(&head)
|
||||
if errors.Is(err, sql.ErrNoRows) {
|
||||
return 0, nil
|
||||
}
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("journal: read seq counter: %w", err)
|
||||
}
|
||||
return head, nil
|
||||
}
|
||||
|
||||
func readEventsRowsInTx(ctx context.Context, tx DBTX, since int64, limit int) ([]storage.EventsJournalRow, error) {
|
||||
// CAST(ts AS CHAR) normalizes the DATETIME to a stable string across drivers.
|
||||
q := `SELECT seq, CAST(ts AS CHAR), op, issue_id, issue_json, dep_json, comment_json
|
||||
FROM bd_events_journal WHERE seq > ? ORDER BY seq ASC`
|
||||
|
|
|
|||
160
internal/storage/issueops/journal_truncation_test.go
Normal file
160
internal/storage/issueops/journal_truncation_test.go
Normal file
|
|
@ -0,0 +1,160 @@
|
|||
package issueops
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"testing"
|
||||
|
||||
"github.com/steveyegge/beads/internal/storage"
|
||||
)
|
||||
|
||||
func rowsAt(seqs ...int64) []storage.EventsJournalRow {
|
||||
out := make([]storage.EventsJournalRow, 0, len(seqs))
|
||||
for _, s := range seqs {
|
||||
out = append(out, storage.EventsJournalRow{Seq: s, Op: string(EventCreate)})
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// TestComputeEventsTruncation covers the decision table a consumer's resume
|
||||
// depends on. The cases that must NOT fail matter as much as the ones that
|
||||
// must: a false truncation would fault a healthy exporter on every poll.
|
||||
func TestComputeEventsTruncation(t *testing.T) {
|
||||
cases := []struct {
|
||||
name string
|
||||
since int64
|
||||
rows []storage.EventsJournalRow
|
||||
head int64
|
||||
wantErr bool
|
||||
wantFloor int64
|
||||
wantHead int64
|
||||
}{
|
||||
{
|
||||
name: "fresh consumer reads from the start",
|
||||
since: 0,
|
||||
rows: rowsAt(1, 2, 3),
|
||||
head: 3,
|
||||
},
|
||||
{
|
||||
name: "restart resumes the contiguous prefix",
|
||||
since: 7,
|
||||
rows: rowsAt(8, 9),
|
||||
head: 9,
|
||||
},
|
||||
{
|
||||
name: "caught up: no rows and since is the head",
|
||||
since: 9,
|
||||
head: 9,
|
||||
},
|
||||
{
|
||||
name: "empty journal, nothing ever written",
|
||||
since: 0,
|
||||
head: 0,
|
||||
},
|
||||
{
|
||||
name: "prune below the checkpoint leaves the resume contiguous",
|
||||
since: 7,
|
||||
rows: rowsAt(8, 9),
|
||||
head: 9,
|
||||
},
|
||||
{
|
||||
name: "checkpoint pruned past: rows resume above since+1",
|
||||
since: 3,
|
||||
rows: rowsAt(11, 12),
|
||||
head: 12,
|
||||
wantErr: true,
|
||||
wantFloor: 11,
|
||||
wantHead: 12,
|
||||
},
|
||||
{
|
||||
name: "off by one: exactly one record pruned",
|
||||
since: 7,
|
||||
rows: rowsAt(9, 10),
|
||||
head: 10,
|
||||
wantErr: true,
|
||||
wantFloor: 9,
|
||||
wantHead: 10,
|
||||
},
|
||||
{
|
||||
name: "whole journal pruned: empty result is not caught up",
|
||||
since: 5,
|
||||
head: 10,
|
||||
wantErr: true,
|
||||
wantFloor: 11,
|
||||
wantHead: 10,
|
||||
},
|
||||
{
|
||||
name: "head regression is not reported as truncation",
|
||||
since: 20,
|
||||
head: 10,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
err := ComputeEventsTruncation(tc.since, tc.rows, func() (int64, error) { return tc.head, nil })
|
||||
var trunc *storage.EventsJournalTruncatedError
|
||||
switch {
|
||||
case !tc.wantErr && err != nil:
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
case !tc.wantErr:
|
||||
return
|
||||
case err == nil:
|
||||
t.Fatal("expected a truncation error, got nil")
|
||||
case !errors.As(err, &trunc):
|
||||
t.Fatalf("error is not *EventsJournalTruncatedError: %T %v", err, err)
|
||||
}
|
||||
if trunc.Since != tc.since {
|
||||
t.Errorf("Since = %d, want %d", trunc.Since, tc.since)
|
||||
}
|
||||
if trunc.Floor != tc.wantFloor {
|
||||
t.Errorf("Floor = %d, want %d", trunc.Floor, tc.wantFloor)
|
||||
}
|
||||
if trunc.Head != tc.wantHead {
|
||||
t.Errorf("Head = %d, want %d", trunc.Head, tc.wantHead)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestComputeEventsTruncationSkipsHeadReadWhenContiguous pins the hot path:
|
||||
// `bd events tail --follow` polls once a second, so a healthy resume must not
|
||||
// pay for the counter read.
|
||||
func TestComputeEventsTruncationSkipsHeadReadWhenContiguous(t *testing.T) {
|
||||
calls := 0
|
||||
err := ComputeEventsTruncation(4, rowsAt(5, 6), func() (int64, error) {
|
||||
calls++
|
||||
return 6, nil
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
if calls != 0 {
|
||||
t.Errorf("readHead called %d times on a contiguous read, want 0", calls)
|
||||
}
|
||||
}
|
||||
|
||||
func TestComputeEventsTruncationPropagatesHeadReadError(t *testing.T) {
|
||||
sentinel := errors.New("counter unavailable")
|
||||
err := ComputeEventsTruncation(3, nil, func() (int64, error) { return 0, sentinel })
|
||||
if !errors.Is(err, sentinel) {
|
||||
t.Fatalf("err = %v, want the readHead error", err)
|
||||
}
|
||||
var trunc *storage.EventsJournalTruncatedError
|
||||
if errors.As(err, &trunc) {
|
||||
t.Error("an unreadable counter must not be reported as truncation")
|
||||
}
|
||||
}
|
||||
|
||||
// TestEventsJournalTruncatedErrorMessage keeps the message actionable: an
|
||||
// operator reading it must be able to see the lost span without doing math.
|
||||
func TestEventsJournalTruncatedErrorMessage(t *testing.T) {
|
||||
err := &storage.EventsJournalTruncatedError{Since: 3, Floor: 11, Head: 12}
|
||||
want := "events journal truncated: checkpoint 3 is below the retained window [11..12]; records 4..10 were pruned"
|
||||
if got := err.Error(); got != want {
|
||||
t.Errorf("Error() = %q, want %q", got, want)
|
||||
}
|
||||
if fmt.Sprintf("%v", error(err)) != want {
|
||||
t.Error("error does not format through the error interface")
|
||||
}
|
||||
}
|
||||
131
internal/storage/issueops/journal_vocabulary_test.go
Normal file
131
internal/storage/issueops/journal_vocabulary_test.go
Normal file
|
|
@ -0,0 +1,131 @@
|
|||
package issueops
|
||||
|
||||
import (
|
||||
"go/ast"
|
||||
"go/parser"
|
||||
"go/token"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// TestEventOpVocabularyIsFrozen pins the journal's op vocabulary and its split
|
||||
// into the six-op public wire set plus the engine-only remainder.
|
||||
//
|
||||
// The two sets are declared independently of the EventOp constants, so this
|
||||
// fails on any drift in either direction: a new EventOp constant that nobody
|
||||
// assigned to a side, a wire op quietly renamed, or an engine-only op promoted
|
||||
// to the wire without the contract change that would require. Consumers dedupe
|
||||
// and switch on these strings, so a rename is a breaking change even when it
|
||||
// compiles.
|
||||
func TestEventOpVocabularyIsFrozen(t *testing.T) {
|
||||
const (
|
||||
wantWire = 6
|
||||
wantEngineOnly = 1
|
||||
)
|
||||
|
||||
frozenWire := []EventOp{"create", "update", "close", "delete", "dep_add", "dep_remove"}
|
||||
frozenEngineOnly := []EventOp{"comment"}
|
||||
|
||||
if got := WireEventOps(); !equalOps(got, frozenWire) {
|
||||
t.Errorf("wire event vocabulary changed: got %v, frozen %v", got, frozenWire)
|
||||
}
|
||||
if got := EngineOnlyEventOps(); !equalOps(got, frozenEngineOnly) {
|
||||
t.Errorf("engine-only event vocabulary changed: got %v, frozen %v", got, frozenEngineOnly)
|
||||
}
|
||||
if len(frozenWire) != wantWire || len(frozenEngineOnly) != wantEngineOnly {
|
||||
t.Fatalf("vocabulary sizes changed: %d wire, %d engine-only", len(frozenWire), len(frozenEngineOnly))
|
||||
}
|
||||
|
||||
for _, op := range frozenWire {
|
||||
if !IsWireEventOp(op) {
|
||||
t.Errorf("IsWireEventOp(%q) = false, want true", op)
|
||||
}
|
||||
}
|
||||
for _, op := range frozenEngineOnly {
|
||||
if IsWireEventOp(op) {
|
||||
t.Errorf("IsWireEventOp(%q) = true, want false: engine-only ops must not reach the wire", op)
|
||||
}
|
||||
}
|
||||
if IsWireEventOp("not_an_op") {
|
||||
t.Error("IsWireEventOp accepted an unknown op")
|
||||
}
|
||||
}
|
||||
|
||||
// TestEveryDeclaredEventOpIsClassified is the structural half: it reads the
|
||||
// EventOp constants out of journal.go's AST and asserts every one of them lands
|
||||
// on exactly one side of the freeze. Declaring a new op without classifying it
|
||||
// therefore cannot ship — which is the failure mode a hand-maintained list has.
|
||||
func TestEveryDeclaredEventOpIsClassified(t *testing.T) {
|
||||
declared := declaredEventOps(t)
|
||||
if len(declared) == 0 {
|
||||
t.Fatal("found no EventOp constants in journal.go — was the type renamed?")
|
||||
}
|
||||
|
||||
classified := map[EventOp]int{}
|
||||
for _, op := range WireEventOps() {
|
||||
classified[op]++
|
||||
}
|
||||
for _, op := range EngineOnlyEventOps() {
|
||||
classified[op]++
|
||||
}
|
||||
|
||||
for _, op := range declared {
|
||||
switch classified[op] {
|
||||
case 1:
|
||||
// classified exactly once
|
||||
case 0:
|
||||
t.Errorf("EventOp %q is journaled but classified on neither side: add it to wireEventOps or engineOnlyEventOps", op)
|
||||
default:
|
||||
t.Errorf("EventOp %q is classified as both wire and engine-only", op)
|
||||
}
|
||||
}
|
||||
if len(declared) != len(classified) {
|
||||
t.Errorf("classified %d ops but %d are declared: %v", len(classified), len(declared), declared)
|
||||
}
|
||||
}
|
||||
|
||||
// declaredEventOps returns the string value of every `X EventOp = "y"` constant
|
||||
// declared in journal.go.
|
||||
func declaredEventOps(t *testing.T) []EventOp {
|
||||
t.Helper()
|
||||
file, err := parser.ParseFile(token.NewFileSet(), "journal.go", nil, 0)
|
||||
if err != nil {
|
||||
t.Fatalf("parse journal.go: %v", err)
|
||||
}
|
||||
|
||||
var ops []EventOp
|
||||
for _, decl := range file.Decls {
|
||||
gen, ok := decl.(*ast.GenDecl)
|
||||
if !ok || gen.Tok != token.CONST {
|
||||
continue
|
||||
}
|
||||
for _, spec := range gen.Specs {
|
||||
vs, ok := spec.(*ast.ValueSpec)
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
if ident, ok := vs.Type.(*ast.Ident); !ok || ident.Name != "EventOp" {
|
||||
continue
|
||||
}
|
||||
for _, val := range vs.Values {
|
||||
lit, ok := val.(*ast.BasicLit)
|
||||
if !ok || lit.Kind != token.STRING {
|
||||
continue
|
||||
}
|
||||
ops = append(ops, EventOp(lit.Value[1:len(lit.Value)-1]))
|
||||
}
|
||||
}
|
||||
}
|
||||
return ops
|
||||
}
|
||||
|
||||
func equalOps(a, b []EventOp) bool {
|
||||
if len(a) != len(b) {
|
||||
return false
|
||||
}
|
||||
for i := range a {
|
||||
if a[i] != b[i] {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
|
@ -9,6 +9,7 @@ import (
|
|||
"context"
|
||||
"database/sql"
|
||||
"errors"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"github.com/steveyegge/beads/internal/types"
|
||||
|
|
@ -238,6 +239,40 @@ type EventsJournalRow struct {
|
|||
CommentJSON string
|
||||
}
|
||||
|
||||
// EventsJournalTruncatedCode is the stable machine-readable code a consumer
|
||||
// matches on when its checkpoint has fallen below the retained journal window.
|
||||
const EventsJournalTruncatedCode = "events_journal_truncated"
|
||||
|
||||
// EventsJournalTruncatedError reports that a sequential read cannot resume from
|
||||
// the caller's checkpoint because the rows it needs next were pruned.
|
||||
//
|
||||
// Without it, `WHERE seq > since` cannot distinguish "nothing new" from "your
|
||||
// prefix is gone": a consumer resuming past a prune would either see an empty
|
||||
// success and stall forever, or silently skip to the current floor and lose
|
||||
// every record in between. Both are silent data loss, so the read fails loudly
|
||||
// instead and hands back the window it can actually serve.
|
||||
//
|
||||
// Floor is the lowest seq still retained, or Head+1 when the journal holds no
|
||||
// rows at all. Head is the highest seq the counter has ever assigned; it never
|
||||
// decreases under a prune, so Floor > Head means "fully pruned, caught up to
|
||||
// Head". A consumer that receives this must decide explicitly — resume from
|
||||
// Floor-1 and accept the gap, or rebuild from scratch — and the engine does not
|
||||
// decide for it.
|
||||
type EventsJournalTruncatedError struct {
|
||||
// Since is the checkpoint the caller presented.
|
||||
Since int64
|
||||
// Floor is the lowest retained seq (Head+1 when nothing is retained).
|
||||
Floor int64
|
||||
// Head is the highest seq ever assigned.
|
||||
Head int64
|
||||
}
|
||||
|
||||
func (e *EventsJournalTruncatedError) Error() string {
|
||||
return fmt.Sprintf(
|
||||
"events journal truncated: checkpoint %d is below the retained window [%d..%d]; records %d..%d were pruned",
|
||||
e.Since, e.Floor, e.Head, e.Since+1, e.Floor-1)
|
||||
}
|
||||
|
||||
// EventsJournalAccessor reads and prunes the durable events journal
|
||||
// (bd_events_journal) through the store's own transaction machinery. Unlike
|
||||
// RawDBAccessor — which only the server-mode store provides — this works on the
|
||||
|
|
@ -245,7 +280,8 @@ type EventsJournalRow struct {
|
|||
// Callers that need the journal should type-assert to this interface.
|
||||
type EventsJournalAccessor interface {
|
||||
// ReadEventsJournal returns rows with seq greater than since, ordered by
|
||||
// seq ascending, optionally capped by limit (0 = no cap).
|
||||
// seq ascending, optionally capped by limit (0 = no cap). It returns
|
||||
// *EventsJournalTruncatedError when since sits below the retained window.
|
||||
ReadEventsJournal(ctx context.Context, since int64, limit int) ([]EventsJournalRow, error)
|
||||
// PruneEventsJournal deletes rows with seq below before, honoring the
|
||||
// retain-days / retain-rows floors (0 = floor disabled), and returns the
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue