diff --git a/CHANGELOG.md b/CHANGELOG.md index 2b0209e93..580099cca 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -459,6 +459,23 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 `--all`: the same rows the raw filter selected, on both planes. `--limit 0` is unlimited, as before. The bead-scoped form (`bd gate list `) is unchanged. +- **`bd export` works over a connected HTTP workspace** (S21). It read + through the raw `SearchIssues` method and the bulk label, dependency and + comment reads, which the HTTP backend refuses, so gc's 15-minute JSONL + export order (`gc bd export --all`) failed on every remote scope. On a remote + backend the read set now comes from one library entry, `issueops.ReadExport`, + composed over role reads every deployed server publishes: an unlimited + `Reader.List` for the rows, their labels and counts, `EdgeReader.ReadEdges` + for the dependency records, and `Reader.Get` with comments for each row + that has any. The JSONL is written by the same code as on every other + route. Memories (`--all`, `--include-memories`) come from the served memory + plane (`listMemories`), not the settings listing, which withholds every + `kv.*` row and so exported none. A failed read fails the export, never + shortens it. A remote export does not yet stamp the `wisp_plane` marker, + because the server publishes no plane membership. So a current `bd import` + of a remote export puts genuine no-history wisps on the durable issues + plane instead of the wisps plane, and gc's JSONL archive keeps them. + Nothing is dropped. - **An update of a template is now refused for every caller, not only `bd update`.** The template guard used to be a pre-read in `cmd/bd`, so an update through `bd serve` (`PATCH /v0/beads/issues/{id}`), an `update` item diff --git a/backend/http/BUILD.bazel b/backend/http/BUILD.bazel index dc812c067..045b68860 100644 --- a/backend/http/BUILD.bazel +++ b/backend/http/BUILD.bazel @@ -32,6 +32,7 @@ go_test( srcs = [ "attach_test.go", "e2e_cli_test.go", + "e2e_export_test.go", "e2e_list_filters_test.go", "e2e_mol_test.go", "e2e_recompute_blocked_test.go", diff --git a/backend/http/corpus/testdata/corpus.yaml b/backend/http/corpus/testdata/corpus.yaml index 7eb95caeb..9c3d2e485 100644 --- a/backend/http/corpus/testdata/corpus.yaml +++ b/backend/http/corpus/testdata/corpus.yaml @@ -434,7 +434,6 @@ shapes: source: gc core pack reaper (gc bd export --all -o F) args: [export, --all, -o, "{file:export.jsonl}"] compare: exit - pending: {slice: "S21", reason: "export --all is not wired over http (B3 #16)"} # ================================================================ writes # ---------------------------------------------------------------- create @@ -686,7 +685,7 @@ shapes: source: mc mol-normalized-issue-pr-v1 (promote --reason) args: [promote, e2e-wisp-p, --reason, "keep it"] compare: exit - pending: {slice: "S21", reason: "promote is not wired over http (B3 #16)"} + pending: {slice: "S21b", reason: "promote has no served operation (B3 #16): the http Store stubs PromoteFromEphemeral, updateIssue's patch does not publish persistence (W-IssuePatch.Persistence), and no server has a promote route; it needs a new additive capability, then SRV-1"} - id: mc-mol-burn source: mc review fixtures (mol burn --force) args: [mol, burn, e2e-wisp-m, --force] diff --git a/backend/http/e2e_export_test.go b/backend/http/e2e_export_test.go new file mode 100644 index 000000000..6b5a7e804 --- /dev/null +++ b/backend/http/e2e_export_test.go @@ -0,0 +1,203 @@ +//go:build cgo + +package bdhttp_test + +import ( + "encoding/json" + "maps" + "os" + "path/filepath" + "strconv" + "strings" + "testing" +) + +type exportLine struct { + Type string `json:"_type"` + ID string `json:"id"` + Ephemeral bool `json:"ephemeral"` + NoHistory bool `json:"no_history"` + WispPlane *bool `json:"wisp_plane"` + Labels []string `json:"labels"` + CommentCount int `json:"comment_count"` + DepCount int `json:"dependency_count"` + Dependencies []struct { + IssueID string `json:"issue_id"` + DependsOnID string `json:"depends_on_id"` + Type string `json:"type"` + } `json:"dependencies"` + Comments []struct { + Text string `json:"text"` + } `json:"comments"` + // Key and Value are a memory record's fields. + Key string `json:"key"` + Value string `json:"value"` +} + +// plane renders the explicit wisps-plane marker: "absent", "true" or "false". +func (l exportLine) plane() string { + if l.WispPlane == nil { + return "absent" + } + return strconv.FormatBool(*l.WispPlane) +} + +func readExportRecords(t *testing.T, path string) []exportLine { + t.Helper() + data, err := os.ReadFile(path) + if err != nil { + t.Fatalf("read export: %v", err) + } + var out []exportLine + for _, line := range strings.Split(strings.TrimSpace(string(data)), "\n") { + if line == "" { + continue + } + var rec exportLine + if err := json.Unmarshal([]byte(line), &rec); err != nil { + t.Fatalf("parse export line %q: %v", line, err) + } + out = append(out, rec) + } + return out +} + +func readExportLines(t *testing.T, path string) map[string]exportLine { + t.Helper() + out := map[string]exportLine{} + for _, rec := range readExportRecords(t, path) { + if rec.Type == "issue" { + out[rec.ID] = rec + } + } + return out +} + +// readExportMemories returns an export's memory records, value by key. +func readExportMemories(t *testing.T, path string) map[string]string { + t.Helper() + out := map[string]string{} + for _, rec := range readExportRecords(t, path) { + if rec.Type == "memory" { + out[rec.Key] = rec.Value + } + } + return out +} + +func wantExportMemories(t *testing.T, export string, got, want map[string]string) { + t.Helper() + if !maps.Equal(got, want) { + t.Errorf("%s memories = %v, want %v", export, got, want) + } +} + +// TestE2E_ExportOverHTTP is S21. `bd export` read through the raw SearchIssues +// method and the bulk relation reads, which the http backend refuses, so gc's +// JSONL-export order failed on every remote scope. It now composes served role +// reads (issueops.ReadExport): the rows with their labels and counts, every +// row's dependency records and the comment bodies. Memories come from the +// served memory plane (listMemories): the settings enumeration withholds every +// kv.* row, so a remote export that read them there carried none. +// +// The no-history row pins the one place this route still differs from the +// local ones. No served read publishes plane membership, so the row is +// exported without the wisp_plane marker, and a current import pins it to the +// durable issues plane: the data-safe side, since a wrong marker would drop a +// durable row's relations (bd-r9uce). Serving the membership (bd-1lbvas) +// flips those assertions: the row is then marked and re-imports as a wisp. +func TestE2E_ExportOverHTTP(t *testing.T) { + bd := connectE2EWorkspace(t, "worker") + + blocked := createE2EIssue(t, bd, "blocked work") + blocker := createE2EIssue(t, bd, "blocker") + mustRun(t, bd, "dep", "add", blocked, blocker, "--json") + mustRun(t, bd, "label", "add", blocked, "lane:x", "--json") + mustRun(t, bd, "comments", "add", blocked, "an export comment") + closed := createE2EIssue(t, bd, "closed work") + mustRun(t, bd, "close", closed, "--json") + wisp := firstJSONField(t, mustRun(t, bd, "create", "a wisp", "--ephemeral", "--json").stdout, "id") + noHistory := firstJSONField(t, mustRun(t, bd, "create", "a no-history wisp", "--no-history", "--json").stdout, "id") + memories := map[string]string{ + "export-alpha": "remembered over http", + "export-beta": "a second memory", + } + for key, content := range memories { + mustRun(t, bd, "remember", content, "--key", key) + } + + dir := t.TempDir() + all := filepath.Join(dir, "all.jsonl") + mustRun(t, bd, "export", "--all", "-o", all) + wantExportMemories(t, "bd export --all", readExportMemories(t, all), memories) + got := readExportLines(t, all) + for _, id := range []string{blocked, blocker, closed, wisp, noHistory} { + if _, ok := got[id]; !ok { + t.Errorf("bd export --all is missing %s; got %d issues", id, len(got)) + } + } + rec := got[blocked] + if len(rec.Dependencies) != 1 || rec.Dependencies[0].DependsOnID != blocker || rec.DepCount != 1 { + t.Errorf("%s dependencies = %+v (count %d), want one edge onto %s", blocked, rec.Dependencies, rec.DepCount, blocker) + } + if len(rec.Labels) != 1 || rec.Labels[0] != "lane:x" { + t.Errorf("%s labels = %v, want [lane:x]", blocked, rec.Labels) + } + if rec.CommentCount != 1 || len(rec.Comments) != 1 || rec.Comments[0].Text != "an export comment" { + t.Errorf("%s comments = %+v (count %d), want the one comment", blocked, rec.Comments, rec.CommentCount) + } + if !got[wisp].Ephemeral { + t.Errorf("%s exported without ephemeral: %+v", wisp, got[wisp]) + } + if rec := got[noHistory]; !rec.NoHistory || rec.plane() != "absent" { + t.Errorf("%s exported as no_history=%v wisp_plane=%s, want no_history with no marker", noHistory, rec.NoHistory, rec.plane()) + } + + // The default export keeps every status and the no-history row, and drops + // the true wisp. + def := filepath.Join(dir, "default.jsonl") + mustRun(t, bd, "export", "-o", def) + gotDef := readExportLines(t, def) + if _, ok := gotDef[wisp]; ok { + t.Errorf("bd export (default) exported the wisp %s", wisp) + } + for _, id := range []string{blocked, blocker, closed, noHistory} { + if _, ok := gotDef[id]; !ok { + t.Errorf("bd export (default) is missing %s", id) + } + } + if rec := gotDef[noHistory]; rec.plane() != "absent" { + t.Errorf("bd export (default) stamped %s wisp_plane=%s, want no marker", noHistory, rec.plane()) + } + // Memories are opt-in: absent by default, present with --include-memories. + wantExportMemories(t, "bd export (default)", readExportMemories(t, def), map[string]string{}) + withMemories := filepath.Join(dir, "memories.jsonl") + mustRun(t, bd, "export", "--include-memories", "-o", withMemories) + wantExportMemories(t, "bd export --include-memories", readExportMemories(t, withMemories), memories) + + // Import the remote export into a fresh local workspace and export it + // again there. The local route stamps wisp_plane from table membership, + // which the locally created control row proves, so the imported row's + // missing marker means it sits in the durable issues table. + bin := buildBD(t) + localDir := t.TempDir() + local := func(args ...string) bdResult { + t.Helper() + return runBD(t, bin, localDir, []string{"BEADS_ACTOR=worker"}, args...) + } + mustRun(t, local, "init", "--prefix", "e2e", "--quiet") + mustRun(t, local, "import", "-i", all) + control := firstJSONField(t, mustRun(t, local, "create", "a local no-history wisp", "--no-history", "--json").stdout, "id") + back := filepath.Join(dir, "reexport.jsonl") + mustRun(t, local, "export", "--all", "-o", back) + gotBack := readExportLines(t, back) + if rec, ok := gotBack[control]; !ok || rec.plane() != "true" { + t.Fatalf("local control %s exported with wisp_plane=%s (present=%v), want true", control, rec.plane(), ok) + } + wantExportMemories(t, "the local re-export", readExportMemories(t, back), memories) + if rec, ok := gotBack[noHistory]; !ok { + t.Errorf("the local re-export is missing the imported %s", noHistory) + } else if !rec.NoHistory || rec.plane() != "absent" { + t.Errorf("imported %s re-exported as no_history=%v wisp_plane=%s, want no_history on the durable plane (no marker)", noHistory, rec.NoHistory, rec.plane()) + } +} diff --git a/cmd/bd/BUILD.bazel b/cmd/bd/BUILD.bazel index 4a5e6605a..043395ee2 100644 --- a/cmd/bd/BUILD.bazel +++ b/cmd/bd/BUILD.bazel @@ -780,6 +780,7 @@ go_test( "export_exclude_owner_test.go", "export_max_rows_gate_test.go", "export_proxied_integration_test.go", + "export_source_role_test.go", "export_source_wisp_subset_test.go", "export_test.go", "external_policy_direct_integration_test.go", diff --git a/cmd/bd/export.go b/cmd/bd/export.go index 69bba6405..e616b45bc 100644 --- a/cmd/bd/export.go +++ b/cmd/bd/export.go @@ -17,6 +17,7 @@ import ( "github.com/steveyegge/beads/internal/storage/domain" "github.com/steveyegge/beads/internal/storage/uow" "github.com/steveyegge/beads/internal/types" + "github.com/steveyegge/beads/issueops" ) var exportCmd = &cobra.Command{ @@ -100,6 +101,11 @@ func runExport(cmd *cobra.Command, args []string) error { return err } + if storeIsRemoteBackend(store) { + // A remote backend serves no raw search or bulk relation read; its + // export is composed from served role reads (issueops.ReadExport). + return runExportFromSource(ctx, &roleExportSource{}) + } return runExportFromSource(ctx, storeExportSource{}) } @@ -129,49 +135,9 @@ func runExportFromSource(ctx context.Context, src exportSource) error { w = os.Stdout } - // Build filter for issues table. Export all statuses by default. - // Opt out of BEADS_MAX_ROWS (designer §4.1) — export is a data-integrity - // path and must never abort partway through an export run. - filter := types.IssueFilter{ - Limit: 0, - MaxRows: 0, - MaxRowsSource: "", - } - - // Exclude infra types by default (agents, roles, messages). - if !exportAll && !exportIncludeInfra { - var infraTypes []string - infraSet := src.GetInfraTypes(ctx) - if len(infraSet) > 0 { - for t := range infraSet { - infraTypes = append(infraTypes, t) - } - } - if len(infraTypes) == 0 { - infraTypes = domain.DefaultInfraTypes() - } - for _, t := range infraTypes { - filter.ExcludeTypes = append(filter.ExcludeTypes, types.IssueType(t)) - } - } - - // Exclude templates by default - if !exportAll { - isTemplate := false - filter.IsTemplate = &isTemplate - } - - // Exclude ephemeral wisps by default — they are private/transient and - // must not reach git history or external integrations (GH#3649). - // --all overrides to include everything. - if !exportAll { - persistentOnly := false - filter.Ephemeral = &persistentOnly - } - - issues, err := src.SearchIssues(ctx, "", filter) + issues, err := src.ExportIssues(ctx, issueops.ExportRequest{All: exportAll, IncludeInfra: exportIncludeInfra}) if err != nil { - return HandleErrorRespectJSON("failed to search issues: %v", err) + return HandleErrorRespectJSON("failed to read the export set: %v", err) } // Scrub test/pollution records if requested @@ -271,26 +237,21 @@ func runExportFromSource(ctx context.Context, src exportSource) error { // Memories may contain sensitive agent context and are excluded by default. memoryCount := 0 if (exportIncludeMemories || exportAll) && !exportNoMemories { - allConfig, err := src.GetAllConfig(ctx) + memories, err := src.ExportMemories(ctx) if err != nil { - return HandleErrorRespectJSON("failed to read config for memories: %v", err) + return HandleErrorRespectJSON("failed to read memories: %v", err) } - fullPrefix := kvPrefix + memoryPrefix // Sort keys for deterministic output order (GH#3474). - var memKeys []string - for k := range allConfig { - if strings.HasPrefix(k, fullPrefix) { - memKeys = append(memKeys, k) - } + memKeys := make([]string, 0, len(memories)) + for userKey := range memories { + memKeys = append(memKeys, userKey) } sort.Strings(memKeys) - for _, k := range memKeys { - v := allConfig[k] - userKey := strings.TrimPrefix(k, fullPrefix) + for _, userKey := range memKeys { record := map[string]string{ "_type": "memory", "key": userKey, - "value": v, + "value": memories[userKey], } data, err := json.Marshal(record) if err != nil { @@ -328,6 +289,54 @@ func runExportFromSource(ctx context.Context, src exportSource) error { return nil } +// exportIssueFilter is the storage filter the store-shaped export sources +// search with: every status, infra types excluded unless --all or +// --include-infra, templates and true wisps excluded unless --all. The +// role-backed source asks the same question through issueops.ReadExport. +func exportIssueFilter(ctx context.Context, src exportSource, req issueops.ExportRequest) types.IssueFilter { + // Build filter for issues table. Export all statuses by default. + // Opt out of BEADS_MAX_ROWS (designer §4.1) — export is a data-integrity + // path and must never abort partway through an export run. + filter := types.IssueFilter{ + Limit: 0, + MaxRows: 0, + MaxRowsSource: "", + } + + // Exclude infra types by default (agents, roles, messages). + if !req.All && !req.IncludeInfra { + var infraTypes []string + infraSet := src.GetInfraTypes(ctx) + if len(infraSet) > 0 { + for t := range infraSet { + infraTypes = append(infraTypes, t) + } + } + if len(infraTypes) == 0 { + infraTypes = domain.DefaultInfraTypes() + } + for _, t := range infraTypes { + filter.ExcludeTypes = append(filter.ExcludeTypes, types.IssueType(t)) + } + } + + // Exclude templates by default + if !req.All { + isTemplate := false + filter.IsTemplate = &isTemplate + } + + // Exclude ephemeral wisps by default — they are private/transient and + // must not reach git history or external integrations (GH#3649). + // --all overrides to include everything. + if !req.All { + persistentOnly := false + filter.Ephemeral = &persistentOnly + } + + return filter +} + // exportIssueRecord wraps IssueWithCounts with a _type discriminator so that // every line in the JSONL export is self-describing. Memory lines already // carry "_type":"memory"; this gives issue lines "_type":"issue". (GH#3271) diff --git a/cmd/bd/export_source.go b/cmd/bd/export_source.go index df4e4acfd..8b28b6eaa 100644 --- a/cmd/bd/export_source.go +++ b/cmd/bd/export_source.go @@ -4,11 +4,14 @@ import ( "context" "errors" "fmt" + "strings" "github.com/steveyegge/beads/internal/storage" "github.com/steveyegge/beads/internal/storage/dberrors" "github.com/steveyegge/beads/internal/storage/uow" "github.com/steveyegge/beads/internal/types" + "github.com/steveyegge/beads/issueops" + "github.com/steveyegge/beads/memoryops" ) // exportSource abstracts the storage reads `bd export` performs, so the one @@ -30,16 +33,17 @@ type exportSource interface { // hardcoded defaults). A nil/empty result makes the caller fall back to // domain.DefaultInfraTypes(), mirroring the pre-seam behavior. GetInfraTypes(ctx context.Context) map[string]bool - // SearchIssues returns the full, untruncated result set for the export - // filter (Limit=0, MaxRows=0). Implementations MUST fail rather than - // return a truncated page — export is a data-integrity path. - SearchIssues(ctx context.Context, query string, filter types.IssueFilter) ([]*types.Issue, error) + // ExportIssues returns the full, untruncated export set for req. + // Implementations MUST fail rather than return a truncated or shortened + // set — export is a data-integrity path. + ExportIssues(ctx context.Context, req issueops.ExportRequest) ([]*types.Issue, error) // GetConfig reads one config key from the database (used for the // export.exclude_owners owner-filter keys). GetConfig(ctx context.Context, key string) (string, error) - // GetAllConfig reads the whole config table (used to extract kv.memory.* - // rows when memories are exported). - GetAllConfig(ctx context.Context) (map[string]string, error) + // ExportMemories returns every persistent memory (`bd remember`), keyed + // by the user key without the kv.memory. prefix. Like ExportIssues, it + // MUST fail rather than return a partial set. + ExportMemories(ctx context.Context) (map[string]string, error) // LoadExportRelations bulk-loads labels, dependency records, comments, // comment counts, and dependency counts for the searched issues. LoadExportRelations(ctx context.Context, issues []*types.Issue) (exportRelations, error) @@ -89,8 +93,8 @@ func (storeExportSource) GetInfraTypes(ctx context.Context) map[string]bool { return store.GetInfraTypes(ctx) } -func (storeExportSource) SearchIssues(ctx context.Context, query string, filter types.IssueFilter) ([]*types.Issue, error) { - return store.SearchIssues(ctx, query, filter) +func (src storeExportSource) ExportIssues(ctx context.Context, req issueops.ExportRequest) ([]*types.Issue, error) { + return store.SearchIssues(ctx, "", exportIssueFilter(ctx, src, req)) } func (storeExportSource) GetConfig(ctx context.Context, key string) (string, error) { @@ -100,8 +104,24 @@ func (storeExportSource) GetConfig(ctx context.Context, key string) (string, err return store.GetConfig(ctx, key) } -func (storeExportSource) GetAllConfig(ctx context.Context) (map[string]string, error) { - return store.GetAllConfig(ctx) +func (storeExportSource) ExportMemories(ctx context.Context) (map[string]string, error) { + allConfig, err := store.GetAllConfig(ctx) + if err != nil { + return nil, err + } + return memoriesFromConfig(allConfig), nil +} + +// memoriesFromConfig extracts the persistent memories from a whole config +// table: its kv.memory.* rows, keyed by the user key. +func memoriesFromConfig(allConfig map[string]string) map[string]string { + memories := map[string]string{} + for k, v := range allConfig { + if userKey, ok := strings.CutPrefix(k, kvPrefix+memoryPrefix); ok { + memories[userKey] = v + } + } + return memories } func (storeExportSource) LoadExportRelations(ctx context.Context, issues []*types.Issue) (exportRelations, error) { @@ -177,8 +197,9 @@ func (s *uowExportSource) GetInfraTypes(ctx context.Context) map[string]bool { return infra } -func (s *uowExportSource) SearchIssues(ctx context.Context, query string, filter types.IssueFilter) ([]*types.Issue, error) { - page, err := s.uw.IssueUseCase().SearchIssues(ctx, query, filter) +func (s *uowExportSource) ExportIssues(ctx context.Context, req issueops.ExportRequest) ([]*types.Issue, error) { + filter := exportIssueFilter(ctx, s, req) + page, err := s.uw.IssueUseCase().SearchIssues(ctx, "", filter) if err != nil { return nil, err } @@ -196,8 +217,12 @@ func (s *uowExportSource) GetConfig(ctx context.Context, key string) (string, er return s.uw.ConfigUseCase().GetConfig(ctx, key) } -func (s *uowExportSource) GetAllConfig(ctx context.Context) (map[string]string, error) { - return s.uw.ConfigUseCase().GetAllConfig(ctx) +func (s *uowExportSource) ExportMemories(ctx context.Context) (map[string]string, error) { + allConfig, err := s.uw.ConfigUseCase().GetAllConfig(ctx) + if err != nil { + return nil, err + } + return memoriesFromConfig(allConfig), nil } // LoadExportRelations reproduces the classic bulk loaders over the @@ -405,3 +430,106 @@ func mergeExportMap[V any](dst, src map[string]V) { dst[k] = v } } + +// roleExportSource is the export read set of a remote backend, which serves +// no raw search and no bulk relation read. The issues and every relation come +// from ONE library entry, issueops.ReadExport, composed over role reads every +// deployed server publishes (listIssues, listDependencies, getIssue). The +// JSONL is then built by the same runExportFromSource as on every other route. +type roleExportSource struct { + rows map[string]issueops.ExportRow +} + +// GetInfraTypes is unused on this route: only exportIssueFilter, the filter of +// the store-shaped sources, reads it. ReadExport's listing excludes the infra +// set the server resolves for its own workspace. +func (s *roleExportSource) GetInfraTypes(ctx context.Context) map[string]bool { + return store.GetInfraTypes(ctx) +} + +func (s *roleExportSource) ExportIssues(ctx context.Context, req issueops.ExportRequest) ([]*types.Issue, error) { + rd, err := store.IssueReader() + if err != nil { + return nil, err + } + edges, err := store.EdgeReader() + if err != nil { + return nil, err + } + rows, err := issueops.ReadExport(ctx, rd, edges, req) + if err != nil { + return nil, err + } + s.rows = make(map[string]issueops.ExportRow, len(rows)) + issues := make([]*types.Issue, 0, len(rows)) + for _, row := range rows { + s.rows[row.Issue.ID] = row + issues = append(issues, row.Issue) + } + return issues, nil +} + +func (s *roleExportSource) GetConfig(ctx context.Context, key string) (string, error) { + return store.GetConfig(ctx, key) +} + +// ExportMemories reads the served memory plane (listMemories). The config +// table cannot stand in for it on this route: the server's settings +// enumeration withholds every kv.* row, memories included, so reading it +// exported no memory at all. +func (s *roleExportSource) ExportMemories(ctx context.Context) (map[string]string, error) { + memories, err := store.Memories() + if err != nil { + return nil, err + } + res, err := memories.List(ctx, memoryops.ListRequest{}) + if err != nil { + return nil, err + } + return res.Memories, nil +} + +// LoadExportRelations hands back what ReadExport already read; it issues no +// request of its own. +func (s *roleExportSource) LoadExportRelations(_ context.Context, issues []*types.Issue) (exportRelations, error) { + rel := exportRelations{ + labels: map[string][]string{}, + deps: map[string][]*types.Dependency{}, + comments: map[string][]*types.Comment{}, + commentCounts: map[string]int{}, + depCounts: map[string]*types.DependencyCounts{}, + } + for _, issue := range issues { + row, ok := s.rows[issue.ID] + if !ok { + continue + } + if len(row.Issue.Labels) > 0 { + rel.labels[issue.ID] = row.Issue.Labels + } + if len(row.Issue.Dependencies) > 0 { + rel.deps[issue.ID] = row.Issue.Dependencies + } + if len(row.Issue.Comments) > 0 { + rel.comments[issue.ID] = row.Issue.Comments + } + rel.commentCounts[issue.ID] = row.CommentCount + rel.depCounts[issue.ID] = &types.DependencyCounts{DependencyCount: row.DependencyCount, DependentCount: row.DependentCount} + } + return rel, nil +} + +// WispPlaneIDs answers nothing: no served read publishes plane membership +// (listIssues rows carry no plane field, however the listing is scoped), so a +// remote export stamps no row wisp_plane. A current import pins an +// unmarked no-history row to the durable plane (applyImportWispPlane). That +// is right for a durable row that still carries the flag (a legacy promotion, +// or an earlier unmarked import) and wrong for a genuine no-history wisp, +// which imports onto the Dolt-versioned issues plane. It is the same data-safe +// degradation as a store without storeWispPartitioner: a missing marker +// mis-planes a row, while a wrong one re-planes a durable row and drops its +// relations (bd-r9uce), so the marker is never inferred from flags. Serving +// the membership is bd-1lbvas. +func (s *roleExportSource) WispPlaneIDs(context.Context, []string) (map[string]bool, error) { + return nil, nil +} diff --git a/cmd/bd/export_source_role_test.go b/cmd/bd/export_source_role_test.go new file mode 100644 index 000000000..2f389d36c --- /dev/null +++ b/cmd/bd/export_source_role_test.go @@ -0,0 +1,64 @@ +package main + +import ( + "context" + "errors" + "testing" + + "github.com/steveyegge/beads/internal/storage" + "github.com/steveyegge/beads/memoryops" +) + +// memoriesOnlyStore is a remote backend reduced to the one read +// roleExportSource.ExportMemories makes: it embeds a nil DoltStorage and +// answers only the Memories role accessor. +type memoriesOnlyStore struct { + storage.DoltStorage + memories memoryops.Memories + err error +} + +func (s *memoriesOnlyStore) Memories() (memoryops.Memories, error) { return s.memories, s.err } + +// failingMemoryList is a memory plane whose listing (listMemories) fails. +type failingMemoryList struct { + memoryops.Memories + err error +} + +func (l failingMemoryList) List(context.Context, memoryops.ListRequest) (memoryops.ListResult, error) { + return memoryops.ListResult{}, l.err +} + +// TestRoleExportSourceExportMemoriesFailsTheExport pins exportSource's "fail +// rather than return a partial set" on the remote route: a memory plane that +// cannot be reached or listed ends the export with that error, never with an +// empty map. Degrading to empty would bring back what reading memories from +// the settings listing did, a remote `bd export --all` that exits 0 with no +// memory in it. +func TestRoleExportSourceExportMemoriesFailsTheExport(t *testing.T) { + oldStore := store + t.Cleanup(func() { store = oldStore }) + + unreachable := errors.New("memories: no transport") + listFailed := errors.New("listMemories: 503 Service Unavailable") + for _, tc := range []struct { + name string + store *memoriesOnlyStore + want error + }{ + {"the role is unavailable", &memoriesOnlyStore{err: unreachable}, unreachable}, + {"the listing fails", &memoriesOnlyStore{memories: failingMemoryList{err: listFailed}}, listFailed}, + } { + t.Run(tc.name, func(t *testing.T) { + store = tc.store + got, err := (&roleExportSource{}).ExportMemories(context.Background()) + if !errors.Is(err, tc.want) { + t.Fatalf("ExportMemories error = %v, want %v", err, tc.want) + } + if got != nil { + t.Fatalf("ExportMemories returned %v with its error, want no map", got) + } + }) + } +} diff --git a/internal/httpclient/memories.go b/internal/httpclient/memories.go index 986f6a8f4..28778cde7 100644 --- a/internal/httpclient/memories.go +++ b/internal/httpclient/memories.go @@ -5,6 +5,7 @@ package httpclient import ( "context" "errors" + "fmt" "strings" "github.com/steveyegge/beads/internal/httpapi/apigen" @@ -95,11 +96,20 @@ func (m *httpMemories) Forget(ctx context.Context, req memoryops.ForgetRequest) // The folding is the ROLE's, so it happens server-side: the raw term goes on the // wire as `search` and the server's own role lowercases both sides. Folding it // here as well would be a second implementation of a rule that has one. +// +// A TRUNCATED PAGE IS REFUSED. The envelope is paginated but ListResult has no +// cursor, so a page reporting has_more would pass a partial plane off as the +// whole one, and `bd export` would drop the rest without a word. No v0 server +// truncates; this keeps the day one does from becoming a silent loss. func (m *httpMemories) List(ctx context.Context, req memoryops.ListRequest) (memoryops.ListResult, error) { page, err := m.wire.ListMemories(ctx, req.Search) if err != nil { return memoryops.ListResult{}, err } + if page.HasMore { + return memoryops.ListResult{}, fmt.Errorf("listMemories over http: the server returned %d memories with more remaining; "+ + "refusing a truncated answer (this client cannot page the memory plane)", len(page.Items)) + } // Empty map, never nil: a caller ranges over it without a nil check, and // "the plane is empty" is not something to spell as an absent answer. out := make(map[string]string, len(page.Items)) diff --git a/internal/httpclient/write_roles_test.go b/internal/httpclient/write_roles_test.go index a4456dab6..f4645ef0a 100644 --- a/internal/httpclient/write_roles_test.go +++ b/internal/httpclient/write_roles_test.go @@ -1802,6 +1802,24 @@ func TestMemoryListAnswersAnEmptyMapNeverNil(t *testing.T) { } } +// TestMemoryListRefusesATruncatedPage: ListResult has no cursor, so a page that +// reports more memories than it carries is refused rather than returned as the +// whole plane (`bd export` would drop the rest silently). +func TestMemoryListRefusesATruncatedPage(t *testing.T) { + w := &stubWire{page: &apigen.MemoriesPage{HasMore: true, Items: []apigen.Memory{{Key: "a", Value: "1"}}}} + memories, err := stubStore(t, w).Memories() + if err != nil { + t.Fatalf("Memories(): %v", err) + } + res, err := memories.List(t.Context(), memoryops.ListRequest{}) + if err == nil { + t.Fatalf("List of a has_more page = %+v, want a refusal", res) + } + if !strings.Contains(err.Error(), "listMemories") { + t.Errorf("refusal %q does not name listMemories", err) + } +} + // TestWriteAccessorsRefuseWithoutATransport: a build that registered the backend // without linking a wire client is a wiring fault, and it must be named rather // than reached as a nil dereference three frames into a role. diff --git a/issueops/BUILD.bazel b/issueops/BUILD.bazel index b4ca8d266..e23cdd54a 100644 --- a/issueops/BUILD.bazel +++ b/issueops/BUILD.bazel @@ -19,6 +19,7 @@ go_library( "dependencyeditor.go", "edgereader.go", "errors.go", + "export.go", "gates.go", "graphcounter.go", "importer.go", @@ -57,6 +58,7 @@ go_test( srcs = [ "claimer_external_test.go", "current_test.go", + "export_test.go", "gates_test.go", "issueops_external_test.go", "molecule_internal_test.go", diff --git a/issueops/export.go b/issueops/export.go new file mode 100644 index 000000000..242509513 --- /dev/null +++ b/issueops/export.go @@ -0,0 +1,130 @@ +package issueops + +import ( + "context" + "errors" + "fmt" + + "github.com/steveyegge/beads/internal/types" +) + +// ExportRequest describes which issues `bd export` writes. +type ExportRequest struct { + // All exports every row: every status, every type (infra, templates, + // gates), and both planes, wisps included. + All bool + // IncludeInfra admits the workspace's infrastructure types (agents, roles, + // messages by default) without lifting the other default exclusions. + IncludeInfra bool +} + +// ExportRow is one exported issue, fully hydrated: its labels, its stored +// outgoing dependency records and its comment bodies, plus the three +// cardinalities the export line carries. +type ExportRow struct { + Issue *types.Issue + DependencyCount int + DependentCount int + CommentCount int +} + +// ReadExport answers `bd export`'s read set by composing three role reads that +// every backend serves (a remote one as listIssues, listDependencies and +// getIssue): +// +// - Reader.List, unlimited, for the rows, their labels and their counts; +// - EdgeReader.ReadEdges, for every row's stored dependency records; +// - Reader.Get with IncludeComments, for the comment bodies of each row whose +// comment count is nonzero (a row with no comments costs no read). +// +// THE SELECTION IS EXPORT'S, not the listing's. Every status is exported +// (AllFlag), and gates are always exported. Without All, templates and the +// workspace's infra types are excluded (IncludeInfra admits the latter), and +// TRUE wisps are excluded while no-history rows, which live on the wisps plane +// with ephemeral unset, are kept (GH#3649). A listing cannot say "the wisps +// plane minus its ephemeral rows", so the plane is admitted and the ephemeral +// rows are dropped here. That is exact because the listing is unlimited: no +// row is dropped before a limit could have counted it. +// +// A FAILED READ IS AN ERROR. Export is a data-integrity path, so a refusal or +// a missing relation never degrades to a shorter export. An anchor the edge +// read reports missing (a row deleted between the reads) is an error too, and +// so is a listing page marked HasMore: the listing is unlimited, so a +// truncated page means the reader did not honor that. +func ReadExport(ctx context.Context, rd Reader, edges EdgeReader, req ExportRequest) ([]ExportRow, error) { + if rd == nil || edges == nil { + return nil, &ErrUnsupported{Op: "ReadExport", Backend: "nil"} + } + unlimited := 0 + list := ListRequest{ + AllFlag: true, + IncludeGates: true, + IncludeEphemeral: true, + Limit: &unlimited, + } + // --all is EXPORT's scope, spelled as export defines it (infra, + // templates, gates and wisps), not ListRequest.IncludeAllTypes: that + // union is the listing's "hide nothing" contract, which a remote reader + // refuses to restate, and export has always named its own four. + list.IncludeInfra = req.All || req.IncludeInfra + list.IncludeTemplates = req.All + page, err := rd.List(ctx, list) + if err != nil { + return nil, fmt.Errorf("export: listing issues: %w", err) + } + if page.HasMore { + return nil, errors.New("export: listing issues: the listing returned a truncated page; export must never truncate") + } + + rows := make([]ExportRow, 0, len(page.Items)) + ids := make([]string, 0, len(page.Items)) + for _, item := range page.Items { + if item == nil || item.Issue == nil { + continue + } + if !req.All && item.Ephemeral { + continue + } + rows = append(rows, ExportRow{ + Issue: item.Issue, + DependencyCount: item.DependencyCount, + DependentCount: item.DependentCount, + CommentCount: item.CommentCount, + }) + ids = append(ids, item.ID) + } + if len(rows) == 0 { + return rows, nil + } + + edgeResult, err := edges.ReadEdges(ctx, EdgeReadRequest{IDs: ids}) + if err != nil { + return nil, fmt.Errorf("export: reading dependencies: %w", err) + } + deps := make(map[string][]*types.Dependency, len(edgeResult.Anchors)) + for _, anchor := range edgeResult.Anchors { + if anchor.Missing { + return nil, fmt.Errorf("export: reading dependencies: %s is no longer in the store", anchor.ID) + } + if len(anchor.Edges) > 0 { + deps[anchor.ID] = anchor.Edges + } + } + + for i := range rows { + issue := rows[i].Issue + issue.Dependencies = deps[issue.ID] + issue.Comments = nil + if rows[i].CommentCount == 0 { + continue + } + detail, err := rd.Get(ctx, GetRequest{ID: issue.ID, IncludeComments: true}) + if err != nil { + return nil, fmt.Errorf("export: reading comments of %s: %w", issue.ID, err) + } + if detail != nil { + issue.Comments = detail.Comments + } + } + return rows, nil +} diff --git a/issueops/export_test.go b/issueops/export_test.go new file mode 100644 index 000000000..d6024b3ab --- /dev/null +++ b/issueops/export_test.go @@ -0,0 +1,160 @@ +package issueops_test + +import ( + "context" + "errors" + "reflect" + "testing" + + "github.com/steveyegge/beads/internal/types" + "github.com/steveyegge/beads/issueops" +) + +type exportReader struct { + page []*types.IssueWithCounts + hasMore bool + lists []issueops.ListRequest + gets []issueops.GetRequest + err error + getErr error +} + +func (r *exportReader) Ready(context.Context, issueops.ReadyRequest) (issueops.IssuePage, error) { + return issueops.IssuePage{}, errors.New("ready is not export's read") +} + +func (r *exportReader) List(_ context.Context, req issueops.ListRequest) (issueops.IssuePage, error) { + r.lists = append(r.lists, req) + if r.err != nil { + return issueops.IssuePage{}, r.err + } + return issueops.IssuePage{Items: r.page, HasMore: r.hasMore}, nil +} + +func (r *exportReader) Get(_ context.Context, req issueops.GetRequest) (*issueops.IssueDetails, error) { + r.gets = append(r.gets, req) + if r.getErr != nil { + return nil, r.getErr + } + return &issueops.IssueDetails{Comments: []*types.Comment{{IssueID: req.ID, Text: "c of " + req.ID}}}, nil +} + +type exportEdges struct { + calls []issueops.EdgeReadRequest + missing string +} + +func (e *exportEdges) ReadEdges(_ context.Context, req issueops.EdgeReadRequest) (issueops.EdgeReadResult, error) { + e.calls = append(e.calls, req) + out := issueops.EdgeReadResult{Anchors: []issueops.AnchorEdges{}} + for _, id := range req.IDs { + a := issueops.AnchorEdges{ID: id, Edges: []*types.Dependency{}, Missing: id == e.missing} + if id == "x-1" { + a.Edges = []*types.Dependency{{IssueID: "x-1", DependsOnID: "x-2", Type: types.DepBlocks}} + } + out.Anchors = append(out.Anchors, a) + } + return out, nil +} + +func exportPage() []*types.IssueWithCounts { + return []*types.IssueWithCounts{ + {Issue: &types.Issue{ID: "x-1", Labels: []string{"l"}}, DependencyCount: 1, CommentCount: 2}, + {Issue: &types.Issue{ID: "x-2"}, DependentCount: 1}, + {Issue: &types.Issue{ID: "x-wisp", Ephemeral: true}}, + {Issue: &types.Issue{ID: "x-nh", NoHistory: true}}, + nil, + } +} + +// TestReadExportAllExportsEveryRowThroughRoles is S21: `bd export --all` +// composes served role reads, so a remote backend exports without the raw +// search and bulk relation reads it does not serve. +func TestReadExportAllExportsEveryRowThroughRoles(t *testing.T) { + rd := &exportReader{page: exportPage()} + edges := &exportEdges{} + rows, err := issueops.ReadExport(context.Background(), rd, edges, issueops.ExportRequest{All: true}) + if err != nil { + t.Fatalf("ReadExport: %v", err) + } + zero := 0 + wantList := issueops.ListRequest{AllFlag: true, IncludeGates: true, IncludeEphemeral: true, IncludeInfra: true, IncludeTemplates: true, Limit: &zero} + if len(rd.lists) != 1 || !reflect.DeepEqual(rd.lists[0], wantList) { + t.Fatalf("List calls = %#v, want one %#v", rd.lists, wantList) + } + var ids []string + for _, r := range rows { + ids = append(ids, r.Issue.ID) + } + if want := []string{"x-1", "x-2", "x-wisp", "x-nh"}; !reflect.DeepEqual(ids, want) { + t.Fatalf("rows = %v, want %v (every row, in list order)", ids, want) + } + if len(edges.calls) != 1 || !reflect.DeepEqual(edges.calls[0].IDs, ids) { + t.Fatalf("ReadEdges calls = %#v, want one over %v", edges.calls, ids) + } + if got := rows[0].Issue.Dependencies; len(got) != 1 || got[0].DependsOnID != "x-2" { + t.Fatalf("x-1 dependencies = %#v, want the stored edge onto x-2", got) + } + if rows[1].Issue.Dependencies != nil { + t.Fatalf("x-2 dependencies = %#v, want none", rows[1].Issue.Dependencies) + } + // Only the row with comments is read for them. + if want := []issueops.GetRequest{{ID: "x-1", IncludeComments: true}}; !reflect.DeepEqual(rd.gets, want) { + t.Fatalf("Get calls = %#v, want %#v", rd.gets, want) + } + if c := rows[0].Issue.Comments; len(c) != 1 || c[0].Text != "c of x-1" { + t.Fatalf("x-1 comments = %#v", c) + } + if rows[0].CommentCount != 2 || rows[0].DependencyCount != 1 || rows[1].DependentCount != 1 { + t.Fatalf("counts lost: %+v %+v", rows[0], rows[1]) + } + if !reflect.DeepEqual(rows[0].Issue.Labels, []string{"l"}) { + t.Fatalf("labels lost: %v", rows[0].Issue.Labels) + } +} + +// TestReadExportDefaultDropsTrueWispsKeepsNoHistory pins the default scope: +// gates and the wisps plane are read, templates and infra are the listing's to +// exclude, and the true wisps are dropped while no-history rows stay (GH#3649). +func TestReadExportDefaultDropsTrueWispsKeepsNoHistory(t *testing.T) { + for _, infra := range []bool{false, true} { + rd := &exportReader{page: exportPage()} + rows, err := issueops.ReadExport(context.Background(), rd, &exportEdges{}, issueops.ExportRequest{IncludeInfra: infra}) + if err != nil { + t.Fatalf("ReadExport: %v", err) + } + zero := 0 + wantList := issueops.ListRequest{AllFlag: true, IncludeGates: true, IncludeEphemeral: true, IncludeInfra: infra, Limit: &zero} + if !reflect.DeepEqual(rd.lists[0], wantList) { + t.Fatalf("List = %#v, want %#v", rd.lists[0], wantList) + } + var ids []string + for _, r := range rows { + ids = append(ids, r.Issue.ID) + } + if want := []string{"x-1", "x-2", "x-nh"}; !reflect.DeepEqual(ids, want) { + t.Fatalf("rows = %v, want %v", ids, want) + } + } +} + +func TestReadExportNeverShortensOnAFailure(t *testing.T) { + refusal := &issueops.ErrUnsupported{Op: "Reader.List", Backend: "http"} + if rows, err := issueops.ReadExport(context.Background(), &exportReader{err: refusal}, &exportEdges{}, issueops.ExportRequest{All: true}); !errors.Is(err, refusal) || rows != nil { + t.Fatalf("ReadExport = (%v, %v), want (nil, the refusal)", rows, err) + } + if _, err := issueops.ReadExport(context.Background(), &exportReader{page: exportPage()}, &exportEdges{missing: "x-2"}, issueops.ExportRequest{All: true}); err == nil { + t.Fatal("ReadExport with a vanished anchor succeeded, want an error") + } + if rows, err := issueops.ReadExport(context.Background(), &exportReader{page: exportPage(), hasMore: true}, &exportEdges{}, issueops.ExportRequest{All: true}); err == nil || rows != nil { + t.Fatalf("ReadExport over a truncated listing = (%v, %v), want (nil, an error)", rows, err) + } + notFound := errors.New("x-1 not found") + if rows, err := issueops.ReadExport(context.Background(), &exportReader{page: exportPage(), getErr: notFound}, &exportEdges{}, issueops.ExportRequest{All: true}); !errors.Is(err, notFound) || rows != nil { + t.Fatalf("ReadExport with a failed comment read = (%v, %v), want (nil, the failure)", rows, err) + } + var unsupported *issueops.ErrUnsupported + if _, err := issueops.ReadExport(context.Background(), nil, nil, issueops.ExportRequest{}); !errors.As(err, &unsupported) { + t.Fatalf("ReadExport(nil) = %v, want ErrUnsupported", err) + } +}