diff --git a/backend/types.go b/backend/types.go index 34e598257..55eef4f09 100644 --- a/backend/types.go +++ b/backend/types.go @@ -34,6 +34,8 @@ type ( MoleculeLastActivity = types.MoleculeLastActivity MoleculeProgressStats = types.MoleculeProgressStats PersistenceMode = types.PersistenceMode + ProvKind = types.ProvKind + ProvenanceEvent = types.ProvenanceEvent ReclaimFilter = types.ReclaimFilter ReclaimedLease = types.ReclaimedLease SortPolicy = types.SortPolicy diff --git a/cmd/bd/provenance.go b/cmd/bd/provenance.go new file mode 100644 index 000000000..a7d7e7038 --- /dev/null +++ b/cmd/bd/provenance.go @@ -0,0 +1,209 @@ +// Package main implements the bd CLI provenance event-log commands. +package main + +import ( + "fmt" + "time" + + "github.com/spf13/cobra" + "github.com/steveyegge/beads/internal/storage/issueops" + "github.com/steveyegge/beads/internal/types" + "github.com/steveyegge/beads/internal/ui" + "github.com/steveyegge/beads/internal/utils" +) + +var provenanceCmd = &cobra.Command{ + Use: "provenance", + GroupID: "issues", + Short: "Append-only provenance event log", + SilenceUsage: true, + SilenceErrors: true, + Long: `Record and read provenance events: typed bindings from an issue to an +opaque external artifact (a git SHA, PR, work-id, transcript, or branch). + +The log is append-only — there is no update or delete. bd never interprets the +actor or ref; only kind and ref-kind are structurally validated. Recording is +idempotent on a deterministic id, so a producer firing twice is harmless.`, +} + +var ( + provIssue string + provKind string + provSource string + provActor string + provRef string + provRefKind string + provAt string + provPayload string + // provLogKind backs `provenance log --kind`; kept separate from provKind + // (which backs `record --kind`) so the two flags cannot race a shared + // package var across command runs. + provLogKind string +) + +var provenanceRecordCmd = &cobra.Command{ + Use: "record --issue --kind --source ", + Short: "Record a provenance event (idempotent)", + SilenceUsage: true, + SilenceErrors: true, + Long: `Record a provenance event. The event is appended idempotently: a +deterministic id is computed from source:issue:kind:(ref or --at), so re-running +the same record is a no-op. + +An event recorded without --ref requires --at so the id is caller-owned.`, + RunE: func(cmd *cobra.Command, args []string) error { + CheckReadonly("provenance record") + ctx := rootCtx + + issueID, err := utils.ResolvePartialID(ctx, store, provIssue) + if err != nil { + return HandleErrorRespectJSON("resolving %s: %v", provIssue, err) + } + + ev := types.ProvenanceEvent{ + IssueID: issueID, + Kind: types.ProvKind(provKind), + Source: provSource, + } + if provActor != "" { + ev.Actor = &provActor + } + if provRef != "" { + ev.Ref = &provRef + } + if provRefKind != "" { + ev.RefKind = &provRefKind + } + if provPayload != "" { + ev.Payload = &provPayload + } + if provAt != "" { + at, err := time.Parse(time.RFC3339, provAt) + if err != nil { + return HandleErrorRespectJSON("--at must be an RFC3339 timestamp (e.g. 2026-06-19T12:00:00Z): %v", err) + } + atUTC := at.UTC() + ev.OccurredAt = &atUTC + } + + // Fail early with the same structural rules the store enforces, so the CLI + // gives friendly feedback before opening a transaction. + if err := issueops.ValidateProvenanceEvent(ev); err != nil { + return HandleErrorRespectJSON("%v", err) + } + + id, inserted, err := store.RecordProvenanceEvent(ctx, ev) + if err != nil { + return HandleErrorRespectJSON("%v", err) + } + if inserted { + commandDidWrite.Store(true) + } + + if jsonOutput { + return outputJSON(map[string]interface{}{ + "id": id, + "inserted": inserted, + "issue_id": issueID, + "kind": provKind, + }) + } + if inserted { + fmt.Printf("%s Recorded %s provenance %s on %s\n", ui.RenderPass("✓"), provKind, id, issueID) + } else { + fmt.Printf("%s Provenance %s already recorded (id %s)\n", ui.RenderAccent("•"), provKind, id) + } + return nil + }, +} + +var provenanceLogCmd = &cobra.Command{ + Use: "log ", + Short: "List provenance events for an issue", + Args: cobra.ExactArgs(1), + SilenceUsage: true, + SilenceErrors: true, + RunE: func(cmd *cobra.Command, args []string) error { + ctx := rootCtx + issueID, err := utils.ResolvePartialID(ctx, store, args[0]) + if err != nil { + return HandleErrorRespectJSON("resolving %s: %v", args[0], err) + } + events, err := store.GetProvenanceEvents(ctx, issueID, provLogKind) + if err != nil { + return HandleErrorRespectJSON("%v", err) + } + return outputProvenanceEvents(events) + }, +} + +var provenanceByRefCmd = &cobra.Command{ + Use: "by-ref ", + Short: "List provenance events bound to a ref", + Args: cobra.ExactArgs(1), + SilenceUsage: true, + SilenceErrors: true, + RunE: func(cmd *cobra.Command, args []string) error { + ctx := rootCtx + events, err := store.GetProvenanceByRef(ctx, args[0]) + if err != nil { + return HandleErrorRespectJSON("%v", err) + } + return outputProvenanceEvents(events) + }, +} + +func outputProvenanceEvents(events []types.ProvenanceEvent) error { + if jsonOutput { + if events == nil { + events = []types.ProvenanceEvent{} + } + return outputJSON(events) + } + if len(events) == 0 { + fmt.Println("No provenance events") + return nil + } + for _, ev := range events { + when := "—" + if ev.OccurredAt != nil { + when = ev.OccurredAt.Format(time.RFC3339) + } + line := fmt.Sprintf("%s %-8s %s", when, ev.Kind, ev.IssueID) + if ev.Ref != nil { + refKind := "" + if ev.RefKind != nil { + refKind = *ev.RefKind + ":" + } + line += fmt.Sprintf(" %s%s", refKind, *ev.Ref) + } + if ev.Actor != nil { + line += fmt.Sprintf(" by %s", *ev.Actor) + } + line += fmt.Sprintf(" (%s)", ev.Source) + fmt.Println(line) + } + return nil +} + +func init() { + provenanceRecordCmd.Flags().StringVar(&provIssue, "issue", "", "issue id (required)") + provenanceRecordCmd.Flags().StringVar(&provKind, "kind", "", "event kind: cut|claim|suspend|resume|handoff|commit|land|used (required)") + provenanceRecordCmd.Flags().StringVar(&provSource, "source", "", "producer of the event, e.g. git-hook, orchestrator (required)") + provenanceRecordCmd.Flags().StringVar(&provActor, "actor", "", "opaque actor identifier (optional)") + provenanceRecordCmd.Flags().StringVar(&provRef, "ref", "", "opaque external reference, e.g. a SHA or PR url (optional)") + provenanceRecordCmd.Flags().StringVar(&provRefKind, "ref-kind", "", "ref kind: git-sha|pr|work-id|transcript|branch (optional)") + provenanceRecordCmd.Flags().StringVar(&provAt, "at", "", "event-time as RFC3339 (required for ref-less kinds)") + provenanceRecordCmd.Flags().StringVar(&provPayload, "payload", "", "opaque payload, e.g. JSON (optional)") + _ = provenanceRecordCmd.MarkFlagRequired("issue") + _ = provenanceRecordCmd.MarkFlagRequired("kind") + _ = provenanceRecordCmd.MarkFlagRequired("source") + + provenanceLogCmd.Flags().StringVar(&provLogKind, "kind", "", "filter by kind (optional)") + provenanceLogCmd.ValidArgsFunction = issueIDCompletion + + provenanceCmd.AddCommand(provenanceRecordCmd) + provenanceCmd.AddCommand(provenanceLogCmd) + provenanceCmd.AddCommand(provenanceByRefCmd) + rootCmd.AddCommand(provenanceCmd) +} diff --git a/cmd/bd/provenance_test.go b/cmd/bd/provenance_test.go new file mode 100644 index 000000000..445f21ff0 --- /dev/null +++ b/cmd/bd/provenance_test.go @@ -0,0 +1,392 @@ +//go:build cgo + +package main + +import ( + "context" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/steveyegge/beads/internal/types" +) + +// newProvenanceTestIssue creates an issue the provenance events can hang off +// (the FK requires a real issue row). +func newProvenanceTestIssue(t *testing.T, ctx context.Context, s interface { + CreateIssue(context.Context, *types.Issue, string) error +}) string { + t.Helper() + issue := &types.Issue{ + Title: "Provenance subject", + Status: types.StatusOpen, + Priority: 1, + IssueType: types.TypeTask, + CreatedAt: time.Now(), + } + if err := s.CreateIssue(ctx, issue, "test"); err != nil { + t.Fatalf("create issue: %v", err) + } + return issue.ID +} + +func TestProvenanceRecordLogByRefRoundTrip(t *testing.T) { + t.Parallel() + ctx := context.Background() + s := newTestStore(t, filepath.Join(t.TempDir(), ".beads", "beads.db")) + issueID := newProvenanceTestIssue(t, ctx, s) + + sha := "0123456789abcdef0123456789abcdef01234567" + refKind := "git-sha" + actor := "git-hook" + ev := types.ProvenanceEvent{ + IssueID: issueID, + Kind: types.ProvCommit, + Source: "git-hook", + Actor: &actor, + Ref: &sha, + RefKind: &refKind, + } + id, inserted, err := s.RecordProvenanceEvent(ctx, ev) + if err != nil { + t.Fatalf("record: %v", err) + } + if !inserted { + t.Fatal("first record should report inserted=true") + } + if id == "" { + t.Fatal("record returned empty id") + } + + logged, err := s.GetProvenanceEvents(ctx, issueID, "") + if err != nil { + t.Fatalf("log: %v", err) + } + if len(logged) != 1 { + t.Fatalf("log returned %d events, want 1", len(logged)) + } + got := logged[0] + if got.Kind != types.ProvCommit || got.Ref == nil || *got.Ref != sha || got.Actor == nil || *got.Actor != actor { + t.Fatalf("logged event mismatch: %+v", got) + } + + byRef, err := s.GetProvenanceByRef(ctx, sha) + if err != nil { + t.Fatalf("by-ref: %v", err) + } + if len(byRef) != 1 || byRef[0].ID != id { + t.Fatalf("by-ref returned %d events (ids mismatch)", len(byRef)) + } +} + +func TestProvenanceRecordIsIdempotent(t *testing.T) { + t.Parallel() + ctx := context.Background() + s := newTestStore(t, filepath.Join(t.TempDir(), ".beads", "beads.db")) + issueID := newProvenanceTestIssue(t, ctx, s) + + sha := "abcabcabcabcabcabcabcabcabcabcabcabcabca" + refKind := "git-sha" + ev := types.ProvenanceEvent{ + IssueID: issueID, + Kind: types.ProvLand, + Source: "orchestrator", + Ref: &sha, + RefKind: &refKind, + } + + id1, inserted1, err := s.RecordProvenanceEvent(ctx, ev) + if err != nil { + t.Fatalf("first record: %v", err) + } + id2, inserted2, err := s.RecordProvenanceEvent(ctx, ev) + if err != nil { + t.Fatalf("second record: %v", err) + } + if !inserted1 { + t.Fatal("first record should be inserted") + } + if inserted2 { + t.Fatal("second record should report inserted=false (idempotent no-op)") + } + if id1 != id2 { + t.Fatalf("deterministic id changed: %s vs %s", id1, id2) + } + + logged, err := s.GetProvenanceEvents(ctx, issueID, "") + if err != nil { + t.Fatalf("log: %v", err) + } + if len(logged) != 1 { + t.Fatalf("idempotent re-record left %d rows, want 1", len(logged)) + } +} + +func TestProvenanceRejectsUnknownKind(t *testing.T) { + t.Parallel() + ctx := context.Background() + s := newTestStore(t, filepath.Join(t.TempDir(), ".beads", "beads.db")) + issueID := newProvenanceTestIssue(t, ctx, s) + + _, _, err := s.RecordProvenanceEvent(ctx, types.ProvenanceEvent{ + IssueID: issueID, + Kind: types.ProvKind("teleport"), + Source: "orchestrator", + }) + if err == nil || !strings.Contains(err.Error(), "unknown kind") { + t.Fatalf("expected unknown-kind rejection, got %v", err) + } +} + +func TestProvenanceRejectsReservedSourceCaseInsensitive(t *testing.T) { + t.Parallel() + ctx := context.Background() + s := newTestStore(t, filepath.Join(t.TempDir(), ".beads", "beads.db")) + issueID := newProvenanceTestIssue(t, ctx, s) + + at := time.Now().UTC() + for _, src := range []string{"ingest-backfill", "Ingest-Backfill", "INGEST-BACKFILL"} { + _, _, err := s.RecordProvenanceEvent(ctx, types.ProvenanceEvent{ + IssueID: issueID, + Kind: types.ProvClaim, + Source: src, + OccurredAt: &at, + }) + if err == nil || !strings.Contains(err.Error(), "reserved") { + t.Fatalf("source %q: expected reserved-source rejection, got %v", src, err) + } + } +} + +func TestProvenanceRejectsNon40HexGitSHA(t *testing.T) { + t.Parallel() + ctx := context.Background() + s := newTestStore(t, filepath.Join(t.TempDir(), ".beads", "beads.db")) + issueID := newProvenanceTestIssue(t, ctx, s) + + badRef := "not-a-sha" + refKind := "git-sha" + _, _, err := s.RecordProvenanceEvent(ctx, types.ProvenanceEvent{ + IssueID: issueID, + Kind: types.ProvCommit, + Source: "git-hook", + Ref: &badRef, + RefKind: &refKind, + }) + if err == nil || !strings.Contains(err.Error(), "git-sha") { + t.Fatalf("expected git-sha shape rejection, got %v", err) + } +} + +// TestProvenanceRefLessKindIsDeterministicWithOccurredAt verifies the property +// the CLI's --at requirement protects: a ref-less kind keyed by occurred_at gets +// distinct ids for distinct event-times and a stable id for the same one. +func TestProvenanceRefLessKindIsDeterministicWithOccurredAt(t *testing.T) { + t.Parallel() + ctx := context.Background() + s := newTestStore(t, filepath.Join(t.TempDir(), ".beads", "beads.db")) + issueID := newProvenanceTestIssue(t, ctx, s) + + t1 := time.Date(2026, 6, 19, 12, 0, 0, 0, time.UTC) + t2 := time.Date(2026, 6, 19, 13, 0, 0, 0, time.UTC) + + id1a, ins1a, err := s.RecordProvenanceEvent(ctx, types.ProvenanceEvent{ + IssueID: issueID, Kind: types.ProvClaim, Source: "orchestrator", OccurredAt: &t1, + }) + if err != nil { + t.Fatalf("record t1: %v", err) + } + id1b, ins1b, err := s.RecordProvenanceEvent(ctx, types.ProvenanceEvent{ + IssueID: issueID, Kind: types.ProvClaim, Source: "orchestrator", OccurredAt: &t1, + }) + if err != nil { + t.Fatalf("re-record t1: %v", err) + } + id2, ins2, err := s.RecordProvenanceEvent(ctx, types.ProvenanceEvent{ + IssueID: issueID, Kind: types.ProvClaim, Source: "orchestrator", OccurredAt: &t2, + }) + if err != nil { + t.Fatalf("record t2: %v", err) + } + + if !ins1a || ins1b || !ins2 { + t.Fatalf("inserted flags wrong: %v %v %v", ins1a, ins1b, ins2) + } + if id1a != id1b { + t.Fatal("same event-time should yield same id") + } + if id1a == id2 { + t.Fatal("different event-times should yield different ids") + } + + // Kind filter narrows results. + claims, err := s.GetProvenanceEvents(ctx, issueID, string(types.ProvClaim)) + if err != nil { + t.Fatalf("filtered log: %v", err) + } + if len(claims) != 2 { + t.Fatalf("kind-filtered log returned %d, want 2", len(claims)) + } +} + +// TestProvenanceRejectsRefLessWithoutOccurredAt covers the store-boundary guard: +// an event with neither a ref nor an occurred_at has no stable-id discriminator, +// so distinct events would collapse to one id. This must be rejected for any +// kind (handoff here — the CLI's old ref-less kind list missed cut/handoff/used). +func TestProvenanceRejectsRefLessWithoutOccurredAt(t *testing.T) { + t.Parallel() + ctx := context.Background() + s := newTestStore(t, filepath.Join(t.TempDir(), ".beads", "beads.db")) + issueID := newProvenanceTestIssue(t, ctx, s) + + _, _, err := s.RecordProvenanceEvent(ctx, types.ProvenanceEvent{ + IssueID: issueID, + Kind: types.ProvHandoff, + Source: "orchestrator", + }) + if err == nil || !strings.Contains(err.Error(), "occurred_at") { + t.Fatalf("expected ref-less-without-occurred_at rejection, got %v", err) + } +} + +// TestProvenanceRejectsRefKindWithoutRef covers the low-severity guard: a +// ref_kind set with no ref is meaningless and must be rejected. +func TestProvenanceRejectsRefKindWithoutRef(t *testing.T) { + t.Parallel() + ctx := context.Background() + s := newTestStore(t, filepath.Join(t.TempDir(), ".beads", "beads.db")) + issueID := newProvenanceTestIssue(t, ctx, s) + + refKind := "git-sha" + _, _, err := s.RecordProvenanceEvent(ctx, types.ProvenanceEvent{ + IssueID: issueID, + Kind: types.ProvCut, + Source: "orchestrator", + RefKind: &refKind, + }) + if err == nil || !strings.Contains(err.Error(), "requires a ref") { + t.Fatalf("expected ref-kind-without-ref rejection, got %v", err) + } +} + +// TestProvenanceSubSecondOccurredAtIsIdempotent verifies that recording the same +// ref-less event twice with a sub-second --at is idempotent: occurred_at is +// stored at second precision (bare DATETIME), and the id basis truncates to whole +// seconds to match, so the second record is a no-op and exactly one row exists. +func TestProvenanceSubSecondOccurredAtIsIdempotent(t *testing.T) { + t.Parallel() + ctx := context.Background() + s := newTestStore(t, filepath.Join(t.TempDir(), ".beads", "beads.db")) + issueID := newProvenanceTestIssue(t, ctx, s) + + // 123456789ns fractional seconds — must not affect the id. + at := time.Date(2026, 6, 20, 12, 0, 0, 123456789, time.UTC) + ev := types.ProvenanceEvent{ + IssueID: issueID, + Kind: types.ProvClaim, + Source: "orchestrator", + OccurredAt: &at, + } + + id1, ins1, err := s.RecordProvenanceEvent(ctx, ev) + if err != nil { + t.Fatalf("first record: %v", err) + } + id2, ins2, err := s.RecordProvenanceEvent(ctx, ev) + if err != nil { + t.Fatalf("second record: %v", err) + } + if !ins1 || ins2 { + t.Fatalf("sub-second --at not idempotent: inserted flags %v %v", ins1, ins2) + } + if id1 != id2 { + t.Fatalf("id changed across truncated re-record: %s vs %s", id1, id2) + } + + logged, err := s.GetProvenanceEvents(ctx, issueID, "") + if err != nil { + t.Fatalf("log: %v", err) + } + if len(logged) != 1 { + t.Fatalf("sub-second re-record left %d rows, want 1", len(logged)) + } + // Stored occurred_at is truncated to whole seconds (matches the id basis). + if logged[0].OccurredAt == nil || logged[0].OccurredAt.Nanosecond() != 0 { + t.Fatalf("stored occurred_at not truncated to whole seconds: %v", logged[0].OccurredAt) + } +} + +// TestProvenanceOpaqueFieldsRoundTripExactly verifies the NullStringPtr swap did +// not regress opaque-field handling: a non-empty actor round-trips byte-for-byte, +// and an absent (nil) actor reads back as nil (NULL). +func TestProvenanceOpaqueFieldsRoundTripExactly(t *testing.T) { + t.Parallel() + ctx := context.Background() + s := newTestStore(t, filepath.Join(t.TempDir(), ".beads", "beads.db")) + issueID := newProvenanceTestIssue(t, ctx, s) + + sha := "0123456789abcdef0123456789abcdef01234567" + refKind := "git-sha" + actor := "agent/worker-7 (session=abc)" + if _, _, err := s.RecordProvenanceEvent(ctx, types.ProvenanceEvent{ + IssueID: issueID, Kind: types.ProvCommit, Source: "git-hook", + Actor: &actor, Ref: &sha, RefKind: &refKind, + }); err != nil { + t.Fatalf("record with actor: %v", err) + } + + // Absent actor must read back as nil (NULL), not empty string. + sha2 := "fedcba9876543210fedcba9876543210fedcba98" + if _, _, err := s.RecordProvenanceEvent(ctx, types.ProvenanceEvent{ + IssueID: issueID, Kind: types.ProvLand, Source: "git-hook", + Ref: &sha2, RefKind: &refKind, + }); err != nil { + t.Fatalf("record without actor: %v", err) + } + + logged, err := s.GetProvenanceEvents(ctx, issueID, "") + if err != nil { + t.Fatalf("log: %v", err) + } + var withActor, withoutActor *types.ProvenanceEvent + for i := range logged { + switch logged[i].Kind { + case types.ProvCommit: + withActor = &logged[i] + case types.ProvLand: + withoutActor = &logged[i] + } + } + if withActor == nil || withActor.Actor == nil || *withActor.Actor != actor { + t.Fatalf("actor did not round-trip exactly: %+v", withActor) + } + if withoutActor == nil || withoutActor.Actor != nil { + t.Fatalf("absent actor should read back nil (NULL), got %+v", withoutActor) + } +} + +// TestProvenanceNonGitShaRefKindWithRefWorks verifies a non-git-sha ref-kind +// with a ref is accepted (the ref-kind/ref guards do not over-reject). +func TestProvenanceNonGitShaRefKindWithRefWorks(t *testing.T) { + t.Parallel() + ctx := context.Background() + s := newTestStore(t, filepath.Join(t.TempDir(), ".beads", "beads.db")) + issueID := newProvenanceTestIssue(t, ctx, s) + + ref := "https://github.com/example/repo/pull/42" + refKind := "pr" + _, inserted, err := s.RecordProvenanceEvent(ctx, types.ProvenanceEvent{ + IssueID: issueID, + Kind: types.ProvLand, + Source: "orchestrator", + Ref: &ref, + RefKind: &refKind, + }) + if err != nil { + t.Fatalf("record pr ref: %v", err) + } + if !inserted { + t.Fatal("first record should report inserted=true") + } +} diff --git a/engdocs/DOC_INVENTORY.md b/engdocs/DOC_INVENTORY.md index 855faaf49..d5a2efa2d 100644 --- a/engdocs/DOC_INVENTORY.md +++ b/engdocs/DOC_INVENTORY.md @@ -51,6 +51,7 @@ Follow-up automation should replace marker-only checks with generated or | `AGENT_SIGNING.md` | Keep | Maintainer/operator convention for agent comment and commit signatures. | | `adr/0001-multi-remote-approach.md` | Keep | ADR; historical decision record, not a live reference table. | | `adr/0002-init-safety-invariants.md` | Keep | ADR backing `RECOVERY.md` and init safety code. | +| `adr/0003-provenance-event-log.md` | Keep | ADR; historical decision record for the `provenance_events` table and `bd provenance` command group. Status: Proposed pending a maintainer post on issue 4460. | | `ADVANCED.md` | Keep/revise as needed | User-facing advanced workflows; mixed command examples should defer to generated CLI reference when expanded. | | `AIDER_INTEGRATION.md` | Keep | User-facing integration guide; evidence is setup/integration behaviour. | | `ANTIVIRUS.md` | Keep | User-facing operational note; review vendor/version claims when touched. | diff --git a/engdocs/adr/0003-provenance-event-log.md b/engdocs/adr/0003-provenance-event-log.md new file mode 100644 index 000000000..f2906a4e3 --- /dev/null +++ b/engdocs/adr/0003-provenance-event-log.md @@ -0,0 +1,144 @@ +# ADR-0003: Provenance Event Log + +## Status + +Proposed + +## Date + +2026-06-19 + +## Decision Drivers + +- **Outcome linkage**: a runtime (orchestrator, git hook, CI) needs a durable + place to record that work on an issue was *claimed*, *handed off*, *committed*, + or *landed*, and to bind that fact to a structured external artifact (a git + SHA, a PR, a work-id, a transcript, a branch). +- **Primitive, not policy**: the recording surface must be usable by *any* + runtime without bd knowing anything about sessions, agents, or a specific + orchestrator's vocabulary. +- **Honesty under reconstruction**: a consumer that later backfills derived + events must be able to distinguish them from first-party recordings, so a + "read-first" honesty filter can exclude reconstructed rows. +- **Append-only auditability**: provenance is a log of things that happened. + Individual events are immutable — there is no UPDATE or DELETE operation on a + single event after it is recorded. + +## Context + +bd already has an `events` table (migration 0005): a two-value field-mutation +audit trail (`old_value` / `new_value`, plus an `event_type` like +`status_changed`). It answers "what field changed on this issue, when, by whom." + +It does *not* answer "what external artifact is this issue bound to" — there is +no typed place to record that commit `abc…` landed work for `bd-123`, or that a +PR was opened, or that a transcript captured the working session. Producers today +have to overload labels, comments, or `external_ref`, none of which carry a +typed `kind` / `ref_kind` pair or an event-time distinct from ingest-time. + +This primitive was first proven in a TypeScript prototype; this ADR records the +durable upstream version. + +## Decision + +Add a dedicated, append-only `provenance_events` table (migration 0063) and a +`bd provenance` command group with exactly three verbs: `record`, `log`, +`by-ref`. There is deliberately **no** update or delete verb. + +**Append-only is at the event level.** There is no UPDATE or DELETE operation on +an individual event. An event's lifecycle is bound to its issue: it is removed +only if the issue itself is deleted (`ON DELETE CASCADE`), identical to the +`events` audit table (migration 0005). In bd, `issues` is the source of truth, so +a provenance event for a deleted issue is meaningless; cascading on issue delete +matches the existing audit-log precedent rather than leaving orphaned rows. + +**Wisp demotion is a delete for this purpose.** Demoting a permanent issue to a +wisp (`bd update --ephemeral`) deletes the row from `issues` — a wisp is a +separate table, not a status on the same row — so it drops that issue's +provenance events for the same `ON DELETE CASCADE` reason as any other issue +deletion. This is an accepted consequence, not a gap: wisps are cheap, +short-lived, ephemeral-by-design records, and there is deliberately no wisp +counterpart to `provenance_events` to preserve history across the demotion. + +Key design points: + +- **Opaque identifiers.** `actor` and `ref` are opaque strings that bd never + interprets. Only `kind` and `ref_kind` are structurally validated against + closed sets (`kind` ∈ {cut, claim, suspend, resume, handoff, commit, land, + used}; `ref_kind` ∈ {git-sha, pr, work-id, transcript, branch}). When + `ref_kind = git-sha`, the `ref` must match `^[0-9a-f]{40}$`. This is the only + shape bd asserts — consistent with ZFC: bd validates structure, never meaning. + +- **occurred_at vs created_at.** `occurred_at` (event-time) is separate from + `created_at` (ingest-time, `DEFAULT CURRENT_TIMESTAMP`), because a producer + such as a git hook may record a fact *after* it happened. Reads order by + `occurred_at` (nulls last) then `id`. + +- **Idempotent recording.** `bd provenance record` computes a deterministic id + from `source:issue:kind:(ref or occurred_at)` and inserts with `INSERT + IGNORE`, so a producer firing twice is a harmless no-op (`inserted=false` on + the second call). The id is always content-addressed — a caller-supplied `id` + is never honored, so idempotency cannot be bypassed. Any event recorded + *without* a `ref` requires `--at` so the id is caller-owned rather than minted + from the wall clock — otherwise two distinct ref-less events would collapse to + one id. This is enforced at the store boundary (in `ValidateProvenanceEvent`), + not as a CLI kind-list, so every caller is covered. + +- **Reserved source for backfill.** The source value `ingest-backfill` is + rejected by the record path (case-insensitively). It is reserved for + derived/reconstructed events written by consumers, so a read-first honesty + filter can exclude them. Real producers must name their own source. + +- **`ref` is `VARCHAR(255)`, indexed.** A SHA or a PR URL fits comfortably, and + the column is directly indexable in dolt/MySQL (unlike `TEXT`). Refs longer + than 255 chars are out of scope for this column. + +## Considered Alternatives + +### Extend the existing `events` table + +Rejected. `events` is the hot field-mutation audit path: a `(old_value, +new_value, event_type)` shape answering "what changed." Provenance is a different +concern — a typed binding from an issue to a *structured external artifact*, with +its own closed `kind`/`ref_kind` vocabulary, an event-time/ingest-time split, and +idempotent deterministic ids. Bolting these columns onto `events` would (a) +widen and complicate the most frequently written table, (b) mix two unrelated +reasons-to-change in one schema (SRP violation), and (c) force every `events` +reader to reason about NULL provenance columns. + +bd already has a precedent for purpose-specific event tables: `wisp_events` +(migration 0021) is a parallel event log for ephemeral wisps rather than a column +set grafted onto `events`. A separate `provenance_events` table follows the same +established pattern and keeps the hot audit path untouched. + +### Make `actor` / `ref` typed and interpreted by bd + +Rejected. If bd parsed `actor` into a session/agent identity or `ref` into a +known orchestrator's work-id format, the table would only be usable by that one +runtime. Keeping them opaque makes provenance a *primitive*: a git hook, a CI +job, and an orchestrator can all record into the same log, and consumers attach +whatever semantics they need at read time. bd's only job is to store the binding +and validate its structural shape. + +### Allow update/delete + +Rejected. Provenance is a log of facts that occurred; mutating it would destroy +auditability and the idempotency guarantee. The append-only constraint is +enforced at the event level by simply not providing UPDATE or DELETE operations +on individual events. (An event is still removed when its owning issue is deleted +via `ON DELETE CASCADE` — see the Decision section. This is not an event-level +delete path; it is the same issue-bound lifecycle the `events` audit table has.) + +## Distributed merge + +`provenance_events` is **not** in the merge auto-resolve allowlist today. A +`bd pull` that brings in concurrent inserts from two clones will leave the +conflict for the operator to resolve manually, like any other non-allowlisted +table. + +This is safe to auto-resolve in a future change: the table is append-only, its +ids are deterministic and content-addressed, and recording uses `INSERT IGNORE`, +so concurrent inserts from independent clones are commutative (the union of rows +is the correct merge regardless of order, and duplicate ids are idempotent +no-ops). Adding `provenance_events` to the auto-resolve allowlist is deferred as +a separate change so the merge-policy surface is reviewed on its own. diff --git a/internal/jira/tracker_test.go b/internal/jira/tracker_test.go index f1b52789b..c9dd98eb2 100644 --- a/internal/jira/tracker_test.go +++ b/internal/jira/tracker_test.go @@ -889,6 +889,16 @@ func (s *configStore) IterWisps(_ context.Context, _ types.WispFilter) (storage. func (s *configStore) Close() error { return nil } +func (s *configStore) RecordProvenanceEvent(_ context.Context, _ types.ProvenanceEvent) (string, bool, error) { + return "", false, nil +} +func (s *configStore) GetProvenanceEvents(_ context.Context, _, _ string) ([]types.ProvenanceEvent, error) { + return nil, nil +} +func (s *configStore) GetProvenanceByRef(_ context.Context, _ string) ([]types.ProvenanceEvent, error) { + return nil, nil +} + func TestFetchIssuesIncludesPullJQLInQuery(t *testing.T) { var capturedJQL string diff --git a/internal/storage/dolt/ephemeral_routing.go b/internal/storage/dolt/ephemeral_routing.go index 86c7fab39..104ee62a3 100644 --- a/internal/storage/dolt/ephemeral_routing.go +++ b/internal/storage/dolt/ephemeral_routing.go @@ -12,7 +12,7 @@ import ( "github.com/steveyegge/beads/internal/types" ) -var permanentIssueAuxTables = []string{"issues", "labels", "dependencies", "events", "comments"} +var permanentIssueAuxTables = []string{"issues", "labels", "dependencies", "events", "comments", "provenance_events"} // IsEphemeralID returns true if the ID belongs to an ephemeral issue. func IsEphemeralID(id string) bool { diff --git a/internal/storage/dolt/issues.go b/internal/storage/dolt/issues.go index c447fd350..6b356bfcc 100644 --- a/internal/storage/dolt/issues.go +++ b/internal/storage/dolt/issues.go @@ -667,7 +667,7 @@ func (s *DoltStore) deleteIssue(ctx context.Context, id string) error { commitMsg := fmt.Sprintf("bd: delete %s", id) return s.doltAddAndCommitInTx(ctx, tx, - []string{"issues", "dependencies", "labels", "comments", "events", "child_counters", "issue_snapshots", "compaction_snapshots"}, + []string{"issues", "dependencies", "labels", "comments", "events", "provenance_events", "child_counters", "issue_snapshots", "compaction_snapshots"}, commitMsg) }); err != nil { return s.recordDoltPublicationFailure(ctx, err) @@ -749,7 +749,7 @@ func (s *DoltStore) deleteIssues(ctx context.Context, ids []string, cascade bool commitMsg := fmt.Sprintf("bd: delete %d issue(s)", result.DeletedCount) return s.doltAddAndCommitInTx(ctx, tx, - []string{"issues", "dependencies", "labels", "comments", "events", "child_counters", "issue_snapshots", "compaction_snapshots"}, + []string{"issues", "dependencies", "labels", "comments", "events", "provenance_events", "child_counters", "issue_snapshots", "compaction_snapshots"}, commitMsg) }); err != nil { // Preserve partial result (e.g., OrphanedIssues) on error. diff --git a/internal/storage/dolt/provenance.go b/internal/storage/dolt/provenance.go new file mode 100644 index 000000000..8c4c39566 --- /dev/null +++ b/internal/storage/dolt/provenance.go @@ -0,0 +1,57 @@ +package dolt + +import ( + "context" + "database/sql" + "fmt" + + "github.com/steveyegge/beads/internal/storage/issueops" + "github.com/steveyegge/beads/internal/types" +) + +// RecordProvenanceEvent validates and appends a provenance event, then commits +// the provenance_events table. The id is always derived deterministically from +// the event's idempotency basis, so a duplicate insert is ignored: inserted is +// false when the id already existed. Append-only — there is no update or delete +// path. +func (s *DoltStore) RecordProvenanceEvent(ctx context.Context, ev types.ProvenanceEvent) (id string, inserted bool, err error) { + err = s.withRetryTx(ctx, func(tx *sql.Tx) error { + var txErr error + id, inserted, txErr = issueops.RecordProvenanceEventInTx(ctx, tx, ev) + return txErr + }) + if err != nil { + return "", false, err + } + if inserted { + if err := s.doltAddAndCommit(ctx, []string{"provenance_events"}, + fmt.Sprintf("bd: provenance %s %s", ev.Kind, ev.IssueID)); err != nil { + return "", false, err + } + } + return id, inserted, nil +} + +// GetProvenanceEvents returns the provenance events for an issue, ordered by +// occurred_at (nulls last) then id. A non-empty kindFilter restricts to one kind. +func (s *DoltStore) GetProvenanceEvents(ctx context.Context, issueID, kindFilter string) ([]types.ProvenanceEvent, error) { + var result []types.ProvenanceEvent + err := s.withReadTx(ctx, func(tx *sql.Tx) error { + var err error + result, err = issueops.GetProvenanceEventsInTx(ctx, tx, issueID, kindFilter) + return err + }) + return result, err +} + +// GetProvenanceByRef returns every provenance event bound to the opaque ref, +// ordered by occurred_at (nulls last) then id. +func (s *DoltStore) GetProvenanceByRef(ctx context.Context, ref string) ([]types.ProvenanceEvent, error) { + var result []types.ProvenanceEvent + err := s.withReadTx(ctx, func(tx *sql.Tx) error { + var err error + result, err = issueops.GetProvenanceByRefInTx(ctx, tx, ref) + return err + }) + return result, err +} diff --git a/internal/storage/dolt/sweeper.go b/internal/storage/dolt/sweeper.go index 352000bae..3bfce12c8 100644 --- a/internal/storage/dolt/sweeper.go +++ b/internal/storage/dolt/sweeper.go @@ -89,6 +89,6 @@ func (s *sweeper) Sweep(ctx context.Context, req issueops.SweepRequest) (issueop // commit. It is the same list DeleteIssues stages, because a sweep IS a // delete of a selected set. var sweptTables = []string{ - "issues", "dependencies", "labels", "comments", "events", + "issues", "dependencies", "labels", "comments", "events", "provenance_events", "child_counters", "issue_snapshots", "compaction_snapshots", } diff --git a/internal/storage/embeddeddolt/store.go b/internal/storage/embeddeddolt/store.go index 6a835c2f8..28e2b2057 100644 --- a/internal/storage/embeddeddolt/store.go +++ b/internal/storage/embeddeddolt/store.go @@ -617,6 +617,40 @@ func (s *EmbeddedDoltStore) EventsSince(ctx context.Context, cursor storage.Even return result, err } +// RecordProvenanceEvent appends a provenance event idempotently. inserted is +// false when the deterministic id already existed. Append-only — no update path. +func (s *EmbeddedDoltStore) RecordProvenanceEvent(ctx context.Context, ev types.ProvenanceEvent) (id string, inserted bool, err error) { + err = s.withConn(ctx, true, func(tx *sql.Tx) error { + var txErr error + id, inserted, txErr = issueops.RecordProvenanceEventInTx(ctx, tx, ev) + return txErr + }) + if err != nil { + return "", false, err + } + return id, inserted, nil +} + +func (s *EmbeddedDoltStore) GetProvenanceEvents(ctx context.Context, issueID, kindFilter string) ([]types.ProvenanceEvent, error) { + var result []types.ProvenanceEvent + err := s.withConn(ctx, false, func(tx *sql.Tx) error { + var err error + result, err = issueops.GetProvenanceEventsInTx(ctx, tx, issueID, kindFilter) + return err + }) + return result, err +} + +func (s *EmbeddedDoltStore) GetProvenanceByRef(ctx context.Context, ref string) ([]types.ProvenanceEvent, error) { + var result []types.ProvenanceEvent + err := s.withConn(ctx, false, func(tx *sql.Tx) error { + var err error + result, err = issueops.GetProvenanceByRefInTx(ctx, tx, ref) + return err + }) + return result, err +} + // RunInTransaction is implemented in transaction.go. // Close decrements the reference count if this store was opened via Open (the diff --git a/internal/storage/issueops/provenance.go b/internal/storage/issueops/provenance.go new file mode 100644 index 000000000..59c15a38f --- /dev/null +++ b/internal/storage/issueops/provenance.go @@ -0,0 +1,223 @@ +package issueops + +import ( + "context" + "crypto/sha256" + "database/sql" + "encoding/hex" + "fmt" + "regexp" + "strings" + "time" + + "github.com/steveyegge/beads/internal/types" +) + +// Provenance event log on types.ProvenanceEvent (migration 0063; see ADR-0003). + +// knownProvKinds is the closed set of structurally-valid provenance kinds. +var knownProvKinds = map[types.ProvKind]struct{}{ + types.ProvCut: {}, + types.ProvClaim: {}, + types.ProvSuspend: {}, + types.ProvResume: {}, + types.ProvHandoff: {}, + types.ProvCommit: {}, + types.ProvLand: {}, + types.ProvUsed: {}, +} + +// knownProvRefKinds is the closed set of structurally-valid ref kinds. +var knownProvRefKinds = map[string]struct{}{ + "git-sha": {}, + "pr": {}, + "work-id": {}, + "transcript": {}, + "branch": {}, +} + +// ReservedProvSource is reserved for derived/reconstructed events so a consumer's +// read-first honesty filter can exclude backfilled rows. The record path rejects +// it (case-insensitively): real producers must name their own source. +const ReservedProvSource = "ingest-backfill" + +var gitSHARE = regexp.MustCompile(`^[0-9a-f]{40}$`) + +// ValidateProvenanceEvent checks the structural fields of a provenance event +// before it is recorded: kind, ref_kind (when present), the git-sha ref shape, +// and the reserved source. It never interprets the opaque actor/ref values. It +// is exported so the CLI can fail early with the same rules the store enforces. +func ValidateProvenanceEvent(ev types.ProvenanceEvent) error { + if strings.TrimSpace(ev.IssueID) == "" { + return fmt.Errorf("provenance: issue id is required") + } + if _, ok := knownProvKinds[ev.Kind]; !ok { + return fmt.Errorf("provenance: unknown kind %q", ev.Kind) + } + if strings.TrimSpace(ev.Source) == "" { + return fmt.Errorf("provenance: source is required") + } + if strings.EqualFold(strings.TrimSpace(ev.Source), ReservedProvSource) { + return fmt.Errorf("provenance: source %q is reserved for ingest backfill and cannot be recorded directly", ReservedProvSource) + } + if ev.RefKind != nil { + if _, ok := knownProvRefKinds[*ev.RefKind]; !ok { + return fmt.Errorf("provenance: unknown ref-kind %q", *ev.RefKind) + } + if ev.Ref == nil || *ev.Ref == "" { + return fmt.Errorf("provenance: ref-kind %q requires a ref", *ev.RefKind) + } + if *ev.RefKind == "git-sha" { + if !gitSHARE.MatchString(*ev.Ref) { + return fmt.Errorf("provenance: ref-kind git-sha requires a 40-character lowercase hex ref") + } + } + } + // A ref-less event is keyed by occurred_at for its stable id; without either, + // two distinct events would collapse to the same content-addressed id. Guard + // at the store boundary so every caller (CLI or library) is covered. + if (ev.Ref == nil || *ev.Ref == "") && ev.OccurredAt == nil { + return fmt.Errorf("provenance: event with no ref requires occurred_at (--at) for a stable id") + } + return nil +} + +// ProvenanceEventID computes the deterministic, idempotent id for a provenance +// event from source:issue:kind:(ref or occurred_at). A producer firing twice +// with the same facts yields the same id, so the INSERT IGNORE in +// RecordProvenanceEventInTx is a harmless no-op the second time. The +// discriminator is the ref when present, otherwise the fixed-width occurred_at, +// which is why a ref-less event requires --at (so the id is caller-owned, never +// clock-derived at insert time). ValidateProvenanceEvent enforces that. +func ProvenanceEventID(ev types.ProvenanceEvent) string { + disc := "" + switch { + case ev.Ref != nil && *ev.Ref != "": + disc = *ev.Ref + case ev.OccurredAt != nil: + // Truncate to whole seconds: occurred_at is stored as bare DATETIME + // (second precision in dolt/MySQL), so the id basis must match the stored + // value or an id re-derived from a stored row would diverge from the input. + // Fixed-width .000000000 (not RFC3339Nano's variable width) on purpose: + // a stable, deterministic discriminator regardless of trailing zeros. + disc = ev.OccurredAt.UTC().Truncate(time.Second).Format("2006-01-02T15:04:05.000000000Z07:00") + } + key := strings.Join([]string{ev.Source, ev.IssueID, string(ev.Kind), disc}, ":") + sum := sha256.Sum256([]byte(key)) + // Format the digest as a UUID-shaped CHAR(36) so it fits the id column. + h := hex.EncodeToString(sum[:16]) + return fmt.Sprintf("%s-%s-%s-%s-%s", h[0:8], h[8:12], h[12:16], h[16:20], h[20:32]) +} + +// RecordProvenanceEventInTx validates and appends a provenance event. The id is +// always derived deterministically (content-addressed) — a caller-supplied +// ev.ID is ignored so the idempotency invariant cannot be bypassed. A duplicate +// insert is a no-op: inserted is false when the id already existed. Append-only +// at the event level — there is no UPDATE or DELETE of an individual event; an +// event is removed only if its issue is (ON DELETE CASCADE), like the events table. +func RecordProvenanceEventInTx(ctx context.Context, tx *sql.Tx, ev types.ProvenanceEvent) (id string, inserted bool, err error) { + if err := ValidateProvenanceEvent(ev); err != nil { + return "", false, err + } + id = ProvenanceEventID(ev) + + var occurredAt any + if ev.OccurredAt != nil { + // Truncate to whole seconds so the stored value matches the id basis + // (the occurred_at column is bare DATETIME, second precision). + occurredAt = ev.OccurredAt.UTC().Truncate(time.Second) + } + + result, err := tx.ExecContext(ctx, ` + INSERT IGNORE INTO provenance_events + (id, issue_id, kind, actor, ref, ref_kind, payload, source, occurred_at) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?) + `, id, ev.IssueID, string(ev.Kind), NullStringPtr(ev.Actor), NullStringPtr(ev.Ref), + NullStringPtr(ev.RefKind), NullStringPtr(ev.Payload), ev.Source, occurredAt) + if err != nil { + return "", false, fmt.Errorf("recording provenance event: %w", err) + } + affected, err := result.RowsAffected() + if err != nil { + return "", false, fmt.Errorf("provenance rows affected: %w", err) + } + return id, affected == 1, nil +} + +// GetProvenanceEventsInTx returns the provenance events for an issue, ordered by +// occurred_at (nulls last) then id. When kindFilter is non-empty, only events of +// that kind are returned. +func GetProvenanceEventsInTx(ctx context.Context, tx *sql.Tx, issueID, kindFilter string) ([]types.ProvenanceEvent, error) { + query := ` + SELECT id, issue_id, kind, actor, ref, ref_kind, payload, source, occurred_at, created_at + FROM provenance_events + WHERE issue_id = ? + ` + args := []any{issueID} + if kindFilter != "" { + query += " AND kind = ?" + args = append(args, kindFilter) + } + query += " ORDER BY occurred_at IS NULL, occurred_at ASC, id ASC" + + rows, err := tx.QueryContext(ctx, query, args...) + if err != nil { + return nil, fmt.Errorf("querying provenance events: %w", err) + } + defer rows.Close() + return scanProvenanceEvents(rows) +} + +// GetProvenanceByRefInTx returns every provenance event bound to the opaque ref, +// ordered by occurred_at (nulls last) then id. +func GetProvenanceByRefInTx(ctx context.Context, tx *sql.Tx, ref string) ([]types.ProvenanceEvent, error) { + rows, err := tx.QueryContext(ctx, ` + SELECT id, issue_id, kind, actor, ref, ref_kind, payload, source, occurred_at, created_at + FROM provenance_events + WHERE ref = ? + ORDER BY occurred_at IS NULL, occurred_at ASC, id ASC + `, ref) + if err != nil { + return nil, fmt.Errorf("querying provenance by ref: %w", err) + } + defer rows.Close() + return scanProvenanceEvents(rows) +} + +func scanProvenanceEvents(rows *sql.Rows) ([]types.ProvenanceEvent, error) { + var events []types.ProvenanceEvent + for rows.Next() { + var ( + ev types.ProvenanceEvent + kind string + actor sql.NullString + ref sql.NullString + refKind sql.NullString + payload sql.NullString + occurredAt sql.NullTime + ) + if err := rows.Scan(&ev.ID, &ev.IssueID, &kind, &actor, &ref, &refKind, + &payload, &ev.Source, &occurredAt, &ev.CreatedAt); err != nil { + return nil, fmt.Errorf("scanning provenance event: %w", err) + } + ev.Kind = types.ProvKind(kind) + if actor.Valid { + ev.Actor = &actor.String + } + if ref.Valid { + ev.Ref = &ref.String + } + if refKind.Valid { + ev.RefKind = &refKind.String + } + if payload.Valid { + ev.Payload = &payload.String + } + if occurredAt.Valid { + t := occurredAt.Time + ev.OccurredAt = &t + } + events = append(events, ev) + } + return events, rows.Err() +} diff --git a/internal/storage/issueops/wisp_routing.go b/internal/storage/issueops/wisp_routing.go index 7d84619ff..729a5ffd9 100644 --- a/internal/storage/issueops/wisp_routing.go +++ b/internal/storage/issueops/wisp_routing.go @@ -240,6 +240,7 @@ func DeleteCascadeTables(isWisp bool) []string { "labels", "comments", "events", + "provenance_events", "child_counters", "issue_snapshots", "compaction_snapshots", diff --git a/internal/storage/schema/lock_test.go b/internal/storage/schema/lock_test.go index 9850b37b6..c518794ae 100644 --- a/internal/storage/schema/lock_test.go +++ b/internal/storage/schema/lock_test.go @@ -293,11 +293,11 @@ func expectOnePendingMigration(t *testing.T, mock sqlmock.Sqlmock) { // Per-step commit (#4566) snapshots the working set before the migration // runs so it can force-stage only the tables this step newly dirties. expectDoltStatusRows(mock) - // The pending (latest) migration is 0062_events_dolt_ignore, whose body - // contains a bare CALL DOLT_COMMIT — execMigrationBody routes such bodies - // through DrainCall (QueryContext), not ExecContext. - mock.ExpectQuery("(?s).*"). - WillReturnRows(sqlmock.NewRows([]string{"status"})) + // The pending (latest) migration is 0063_create_provenance_events, a + // plain CREATE TABLE body with no bare CALL — execMigrationBody routes + // such bodies through ExecContext, not DrainCall. + mock.ExpectExec("(?s).*"). + WillReturnResult(sqlmock.NewResult(0, 0)) mock.ExpectExec(regexp.QuoteMeta("INSERT IGNORE INTO schema_migrations (version, content_hash) VALUES (?, ?)")). WithArgs(latest, sqlmock.AnyArg()). WillReturnResult(sqlmock.NewResult(0, 1)) diff --git a/internal/storage/schema/migrations/0063_create_provenance_events.down.sql b/internal/storage/schema/migrations/0063_create_provenance_events.down.sql new file mode 100644 index 000000000..47ffa8313 --- /dev/null +++ b/internal/storage/schema/migrations/0063_create_provenance_events.down.sql @@ -0,0 +1 @@ +DROP TABLE IF EXISTS provenance_events; diff --git a/internal/storage/schema/migrations/0063_create_provenance_events.up.sql b/internal/storage/schema/migrations/0063_create_provenance_events.up.sql new file mode 100644 index 000000000..91fafbe67 --- /dev/null +++ b/internal/storage/schema/migrations/0063_create_provenance_events.up.sql @@ -0,0 +1,35 @@ +-- provenance_events: a log binding an issue to a structured external artifact +-- (a git SHA, a PR, a work-id, a transcript, a branch). Append-only at the event +-- level — there is no UPDATE or DELETE operation on individual events. An event's +-- lifecycle is bound to its issue: it is removed only if the issue itself is +-- deleted (ON DELETE CASCADE), identical to the `events` audit table (0005). +-- +-- This is a separate table from `events` on purpose. `events` is the hot +-- field-mutation audit trail (old_value/new_value pairs); provenance_events is +-- a typed binding to an opaque external reference produced by any runtime +-- (git hook, orchestrator, CI). bd never interprets `actor` or `ref` — they are +-- opaque identifiers — so the table stays a primitive usable without any +-- specific orchestrator. Only `kind` and `ref_kind` are structurally validated. +-- +-- occurred_at (event-time) is separate from created_at (ingest-time): a producer +-- may record a fact after it happened (a git hook firing on a past commit). +-- +-- `ref` is VARCHAR(255) rather than TEXT so it is directly indexable in +-- dolt/MySQL; a 40-char SHA or a PR URL fits comfortably. Refs longer than 255 +-- chars are not supported by this column and should not be recorded here. +CREATE TABLE IF NOT EXISTS provenance_events ( + id CHAR(36) NOT NULL PRIMARY KEY, + issue_id VARCHAR(255) NOT NULL, + kind VARCHAR(32) NOT NULL, + actor VARCHAR(255), + ref VARCHAR(255), + ref_kind VARCHAR(32), + payload TEXT, + source VARCHAR(64) NOT NULL, + occurred_at DATETIME, + created_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP, + INDEX idx_provenance_events_issue_occurred (issue_id, occurred_at), + INDEX idx_provenance_events_ref (ref), + INDEX idx_provenance_events_kind (kind), + CONSTRAINT fk_provenance_events_issue FOREIGN KEY (issue_id) REFERENCES issues(id) ON DELETE CASCADE ON UPDATE CASCADE +); diff --git a/internal/storage/storage.go b/internal/storage/storage.go index 4b260dea2..603b5194e 100644 --- a/internal/storage/storage.go +++ b/internal/storage/storage.go @@ -404,6 +404,13 @@ type Storage interface { GetEvents(ctx context.Context, issueID string, limit int) ([]*types.Event, error) GetAllEventsSince(ctx context.Context, since time.Time) ([]*types.Event, error) + // Provenance is an append-only event log binding issues to opaque external + // artifacts (git SHAs, PRs, work-ids). Record is idempotent on a + // deterministic id; there is deliberately no update or delete operation. + RecordProvenanceEvent(ctx context.Context, ev types.ProvenanceEvent) (id string, inserted bool, err error) + GetProvenanceEvents(ctx context.Context, issueID string, kindFilter string) ([]types.ProvenanceEvent, error) + GetProvenanceByRef(ctx context.Context, ref string) ([]types.ProvenanceEvent, error) + // Aggregate counts — cheaper than materializing rows when only cardinality is needed. // Filter.Limit and Filter.Offset are ignored by CountIssues; all others apply. diff --git a/internal/telemetry/storage.go b/internal/telemetry/storage.go index 6338c91ee..dbbf9c7c3 100644 --- a/internal/telemetry/storage.go +++ b/internal/telemetry/storage.go @@ -490,6 +490,30 @@ func (s *InstrumentedStorage) GetAllEventsSince(ctx context.Context, since time. return v, err } +func (s *InstrumentedStorage) RecordProvenanceEvent(ctx context.Context, ev types.ProvenanceEvent) (string, bool, error) { + attrs := []attribute.KeyValue{attribute.String("bd.issue.id", ev.IssueID), attribute.String("bd.prov.kind", string(ev.Kind))} + ctx, span, t := s.op(ctx, "RecordProvenanceEvent", attrs...) + id, inserted, err := s.inner.RecordProvenanceEvent(ctx, ev) + s.done(ctx, span, t, err, attrs...) + return id, inserted, err +} + +func (s *InstrumentedStorage) GetProvenanceEvents(ctx context.Context, issueID, kindFilter string) ([]types.ProvenanceEvent, error) { + attrs := []attribute.KeyValue{attribute.String("bd.issue.id", issueID)} + ctx, span, t := s.op(ctx, "GetProvenanceEvents", attrs...) + v, err := s.inner.GetProvenanceEvents(ctx, issueID, kindFilter) + s.done(ctx, span, t, err, attrs...) + return v, err +} + +func (s *InstrumentedStorage) GetProvenanceByRef(ctx context.Context, ref string) ([]types.ProvenanceEvent, error) { + attrs := []attribute.KeyValue{attribute.String("bd.prov.ref", ref)} + ctx, span, t := s.op(ctx, "GetProvenanceByRef", attrs...) + v, err := s.inner.GetProvenanceByRef(ctx, ref) + s.done(ctx, span, t, err, attrs...) + return v, err +} + // ── Statistics ─────────────────────────────────────────────────────────────── func (s *InstrumentedStorage) GetStatistics(ctx context.Context) (*types.Statistics, error) { diff --git a/internal/types/types.go b/internal/types/types.go index 0fb174fab..3ee300245 100644 --- a/internal/types/types.go +++ b/internal/types/types.go @@ -1529,6 +1529,47 @@ const ( EventLeaseReclaimed EventType = "lease_reclaimed" ) +// ProvenanceEvent is one entry in the append-only provenance log: a typed +// binding from an issue to a structured external artifact (a git SHA, PR, +// work-id, transcript, or branch). +// +// Unlike Event (a field-mutation audit record), a ProvenanceEvent records that +// something happened in the world — a commit landed, a claim was made, work was +// handed off — and ties it to an opaque external Ref. bd never interprets Actor +// or Ref; only Kind and RefKind are structurally validated. This keeps the log +// a primitive usable by any runtime without baking in orchestrator semantics. +// +// OccurredAt (event-time) is distinct from CreatedAt (ingest-time): a producer +// may record a fact after it happened. +type ProvenanceEvent struct { + ID string `json:"id"` + IssueID string `json:"issue_id"` + Kind ProvKind `json:"kind"` + Actor *string `json:"actor,omitempty"` + Ref *string `json:"ref,omitempty"` + RefKind *string `json:"ref_kind,omitempty"` + Payload *string `json:"payload,omitempty"` + Source string `json:"source"` + OccurredAt *time.Time `json:"occurred_at,omitempty"` + CreatedAt time.Time `json:"created_at"` +} + +// ProvKind categorizes a provenance event. +type ProvKind string + +// Provenance event kind constants. These are the only structurally-valid kinds; +// the record path rejects anything outside this set. +const ( + ProvCut ProvKind = "cut" + ProvClaim ProvKind = "claim" + ProvSuspend ProvKind = "suspend" + ProvResume ProvKind = "resume" + ProvHandoff ProvKind = "handoff" + ProvCommit ProvKind = "commit" + ProvLand ProvKind = "land" + ProvUsed ProvKind = "used" +) + // BlockedIssue extends Issue with blocking information type BlockedIssue struct { Issue