diff --git a/BUILD.bazel b/BUILD.bazel index 104409817..7b10becbc 100644 --- a/BUILD.bazel +++ b/BUILD.bazel @@ -300,6 +300,7 @@ filegroup( "//backend:repo_doc_files", "//backend/conformance:repo_doc_files", "//backend/http:repo_doc_files", + "//backend/http/corpus:repo_doc_files", "//beadserrors:repo_doc_files", "//cmd/bd:repo_doc_files", "//cmd/bd/doctor:repo_doc_files", @@ -533,6 +534,7 @@ filegroup( "//backend:repo_go_srcs", "//backend/conformance:repo_go_srcs", "//backend/http:repo_go_srcs", + "//backend/http/corpus:repo_go_srcs", "//beadserrors:repo_go_srcs", "//cmd/bd:repo_go_srcs", "//cmd/bd/doctor:repo_go_srcs", @@ -751,6 +753,7 @@ filegroup( "//backend:repo_go_test_srcs", "//backend/conformance:repo_go_test_srcs", "//backend/http:repo_go_test_srcs", + "//backend/http/corpus:repo_go_test_srcs", "//beadserrors:repo_go_test_srcs", "//cmd/bd:repo_go_test_srcs", "//cmd/bd/doctor:repo_go_test_srcs", @@ -978,6 +981,7 @@ filegroup( "//backend:repo_other_files", "//backend/conformance:repo_other_files", "//backend/http:repo_other_files", + "//backend/http/corpus:repo_other_files", "//beadserrors:repo_other_files", "//cmd/bd:repo_other_files", "//cmd/bd/doctor:repo_other_files", diff --git a/backend/http/corpus/BUILD.bazel b/backend/http/corpus/BUILD.bazel new file mode 100644 index 000000000..7a6f87f43 --- /dev/null +++ b/backend/http/corpus/BUILD.bazel @@ -0,0 +1,123 @@ +load("@rules_go//go:def.bzl", "go_test") +load("@rules_shell//shell:sh_test.bzl", "sh_test") + +go_test( + name = "corpus_test", + # The corpus runs every shape through the real bd (bazeltest.PrebuiltBD), + # served and embedded, under BEADS_TEST_EMBEDDED_DOLT=1 (corpus_served_test + # below); it skips otherwise. + size = "large", + timeout = "long", + srcs = [ + "corpus_test.go", + "harness_test.go", + "reference_proxy_test.go", + "required_corpus_tier_test.go", + ], + data = glob(["testdata/**"]) + ["//cmd/bd:bd_for_tests"], + env = {"BEADS_TEST_BD_BINARY": "$(rlocationpath //cmd/bd:bd_for_tests)"}, + deps = [ + "//internal/httpapi", + "//internal/httpapi/apigen", + "//internal/storage", + "//internal/storage/domain", + "//internal/storage/embeddeddolt", + "//internal/testutil/bazeltest", + "@in_gopkg_yaml_v3//:yaml_v3", + ], +) + +# corpus_test again, with the corpus tier on: BEADS_TEST_EMBEDDED_DOLT=1 runs +# TestServedCorpus and BEADS_HTTP_TEST_REQUIRED=1 turns a skip of it into a +# failure. corpus_test alone skips TestServedCorpus in every lane, because no +# lane sets those variables for it. It repeats corpus_test's `env` +# (go_test_variant.sh). +# +# Tags: +# embedded: the embedded-Dolt tier; excluded from --config=prcore/ci, +# selected by --config=embedded. +sh_test( + name = "corpus_served_test", + size = "large", + # -test.timeout=19m fires (with a goroutine dump) before Bazel's 1200s + # action limit (eternal under --config=embedded). + timeout = "eternal", + srcs = ["//tools/bazel:go_test_variant.sh"], + args = [ + "$(rootpath :corpus_test)", + "-test.v", + "-test.count=1", + "-test.timeout=19m", + ], + data = [ + ":corpus_test", + "//cmd/bd:bd_for_tests", + ], + env = { + "BEADS_HTTP_TEST_REQUIRED": "1", + "BEADS_TEST_BD_BINARY": "$(rlocationpath //cmd/bd:bd_for_tests)", + "BEADS_TEST_EMBEDDED_DOLT": "1", + }, + tags = ["embedded"], +) + +# --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- + +filegroup( + name = "repo_doc_files", + srcs = glob( + [ + "**/*.md", + "**/*.mdx", + ], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_srcs", + srcs = glob( + ["**/*.go"], + allow_empty = True, + exclude = [ + "**/*_test.go", + "**/node_modules/**", + ], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_test_srcs", + srcs = glob( + ["**/*_test.go"], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_other_files", + srcs = glob( + ["**"], + exclude = [ + "**/*.db", + "**/*.exe", + "**/*.go", + "**/*.md", + "**/*.mdx", + "**/*.out", + "**/*.prof", + "**/*.pyc", + "**/*.test", + "**/__pycache__/**", + "**/node_modules/**", + ], + ), + visibility = ["//:__subpackages__"], +) + +# --- end repo_files --- diff --git a/backend/http/corpus/README.md b/backend/http/corpus/README.md new file mode 100644 index 000000000..57e17550b --- /dev/null +++ b/backend/http/corpus/README.md @@ -0,0 +1,120 @@ +# Served command corpus + +This suite is the working definition of "bd fully supports a remote HTTP +backend". It runs every distinct `bd` command shape that an orchestrator (gc) +and its packs and formulas issue against a work store. Each shape runs +against a served workspace and is compared with the same command on an +embedded workspace. + +## What runs + +`TestServedCorpus` does the following: + +1. Builds `bd` from this tree. `BEADS_TEST_BD_BINARY=` reuses a prebuilt + binary instead. +2. Starts an in-process `bd serve` (`httpapi.Listen` over embedded Dolt) the + way a deployed server is bound: `AllowNonLoopback` plus a bearer token + file. The socket is still `127.0.0.1`, but the bind mode decides every + behaviour the server keys on, including its refusal of unlimited reads. +3. Puts a reference proxy in front of the server. The proxy makes it answer + like the deployed reference server described in + `testdata/reference_server.json`: + - the context handshake publishes exactly that server's capability set and + its `ContextResponse` members (so no `wire_revision`); + - an operation the reference server does not publish gets the server's own + 404; + - a query parameter or top-level request member it does not declare gets + the server's own 400 `unknown_parameter`. +4. Runs `bd connect` on one workspace and `bd init` on another. Both get the + `seed` from `testdata/corpus.yaml`: one graph of named rows, plus bulk + graphs that put more than one server page of ready rows and wisps in the + store. Both sides use `BEADS_ACTOR=worker` and set `CLAUDE_SESSION_ID`. +5. Runs each shape as a subtest, on both sides. The embedded run must exit + with the shape's `expect_exit` and match its `expect_stderr`. If it does + not, the shape is wrong or bd changed, and the subtest fails as a corpus + bug that no pending mark or allowlist entry hides. The served run must: + - exit like the embedded run; + - carry no refusal (`ErrUnsupported`, a v0-wire refusal, the + loopback-only 400, or a reference-profile 404/400); + - match the embedded output under the shape's `compare` mode, after + timestamps and revision tokens are dropped (but not `defer_until` or + `due_at`: a shape sets those itself, as absolute dates, and they must + match exactly); + - for writes, give the same JSON from every `verify` read on both sides + afterwards. + +The run takes about five minutes. + +``` +CGO_ENABLED=1 BEADS_TEST_EMBEDDED_DOLT=1 BEADS_HTTP_TEST_REQUIRED=1 \ + go test -tags gms_pure_go -run 'TestServedCorpus|TestCorpusFile' ./backend/http/corpus/ +``` + +In CI, `//backend/http/corpus:corpus_served_test` runs the suite with both +variables set, in the Embedded Dolt lane (`--config=embedded`; see +[engdocs/TESTING.md](../../../engdocs/TESTING.md)). `corpus_test` on its own +sets neither, so it skips `TestServedCorpus`. + +Two environment variables change the run: +- `BEADS_CORPUS_REPORT=` writes a JSON summary with per-shape status + and request counts. +- `BEADS_CORPUS_SKIP_PENDING=1` skips pending shapes without running them. + +## The data + +| File | Holds | +|---|---| +| `testdata/corpus.yaml` | assets, the seed, and the shapes | +| `testdata/allowlist.yaml` | permanent refusals, each with its reason and the refusal text the served run must match | +| `testdata/reference_server.json` | the reference server's wire surface | + +Each shape in `corpus.yaml` has these fields: + +| Field | Meaning | +|---|---| +| `id` | `gc-*` for orchestrator subprocess shapes, `mc-*` for pack/formula/prompt shapes | +| `source` | where the shape comes from | +| `args` | the argv; `{file:NAME}` expands to an asset path and `{rev:ID}` to that side's current revision of `ID` | +| `stdin` | text piped to `bd` on both sides | +| `setup` | commands run on both sides before the shape; one that fails fails the shape | +| `compare` | `json` (default), `ids`, `count`, `text` or `exit` | +| `ordered` | under `json` and `ids`, array order is significant; by default arrays are sorted before comparing | +| `prefix_of` | for a limited `ids` listing: its unlimited form. Each side's limited rows must be a prefix, in order, of that side's own unlimited listing, both sides return as many rows, and the two unlimited listings hold the same ids. A limit that cuts through a tie set (same priority, `created_at` in the same second) lands on different rows on two workspaces seeded seconds apart, so the limited rows are not compared across sides directly | +| `verify` | reads run after the shape | +| `ignore_fields` | JSON keys dropped, at any depth, before a `json` comparison and before `verify` output is compared; for example comment ids, which each workspace mints itself | +| `expect_exit`, `expect_stderr` | expected outcome on both sides (exit 0 by default). An allowlisted shape must state `expect_exit`; only its embedded run is held to these | +| `pending` | a known failure | + +`pending: {slice, reason}` marks a known failure and names the slice or +slices that fix it, for example `S6c+S15`. A pending shape **still runs**: +- while it fails, the subtest is skipped and the skip records what was + observed; +- once it passes, the subtest fails with "now passes; remove the pending + mark". + +So removing a mark is how a slice proves itself, and any other failure is +unexpected and fails the suite. + +## Updating the reference profile + +`reference_server.json` was derived from two things: +- the deployed server's live `GET /v0/beads/context` capabilities; +- its published OpenAPI document (bd 1.3.0, no `wire_revision`). For each + operation the profile keeps the method, the path template, the query + parameter names, and the top-level request members. + +When the deployed server is upgraded (SRV-1), regenerate the file the same +way. Then remove the `SRV-1` pending marks that now pass. + +## Known fidelity limits + +- Responses pass through unfiltered. A response member the reference server + would not send still reaches the client. +- Only top-level request members are checked; nested members (for example + `batchApply` items) are not. +- The served backend underneath is embedded Dolt, not the reference + deployment's storage backend. +- Out of scope, because they are local by design or never touch a remote + store: `init`, `bootstrap`, `migrate`, `doctor`, `flatten`, `gc`, and the + `dolt` subcommands. `context`, `version`, `backup status` and `prime` + are in the corpus, but only checked for exit codes. diff --git a/backend/http/corpus/corpus_test.go b/backend/http/corpus/corpus_test.go new file mode 100644 index 000000000..96a3b095e --- /dev/null +++ b/backend/http/corpus/corpus_test.go @@ -0,0 +1,817 @@ +//go:build cgo + +// Package corpus_test is the served command-corpus suite: every distinct bd +// command shape an orchestrator (gc) and its packs/formulas issue, run by the +// real bd binary against an in-process bd serve masked to a reference +// deployment's wire surface, and compared with the same command on an embedded +// workspace. See README.md. +package corpus_test + +import ( + "encoding/json" + "fmt" + "os" + "path/filepath" + "reflect" + "regexp" + "sort" + "strconv" + "strings" + "testing" + + "gopkg.in/yaml.v3" +) + +// corpusFile is testdata/corpus.yaml. +type corpusFile struct { + Files []corpusAsset `yaml:"files"` + Seed [][]string `yaml:"seed"` + Shapes []shape `yaml:"shapes"` +} + +// corpusAsset is a file the shapes reference as {file:NAME}. Exactly one of +// Content and Bulk is set. +type corpusAsset struct { + Name string `yaml:"name"` + Content string `yaml:"content"` + Bulk *bulkGraph `yaml:"bulk"` +} + +// bulkGraph generates a `bd create --graph` plan of Count independent tasks +// with ids IDPrefix-001.. so the suite can hold more rows than one server page +// without a 100-line literal in the corpus. +type bulkGraph struct { + IDPrefix string `yaml:"id_prefix"` + Count int `yaml:"count"` + Priority int `yaml:"priority"` + Ephemeral bool `yaml:"ephemeral"` + Metadata map[string]string `yaml:"metadata"` +} + +type shape struct { + ID string `yaml:"id"` + Source string `yaml:"source"` + // Args is the bd argv. Tokens may carry {file:NAME} (the side's copy of a + // corpus asset) and {rev:ID} (that side's current revision of ID). + Args []string `yaml:"args"` + Stdin string `yaml:"stdin"` + Setup [][]string `yaml:"setup"` + // Compare is how the served stdout is held to the embedded stdout: + // json (default), ids, count, text, or exit (exit code only). + Compare string `yaml:"compare"` + // Ordered keeps array order significant under json/ids. + Ordered bool `yaml:"ordered"` + // PrefixOf is the unlimited form of a limited listing (compare: ids). A + // limit cut through a tie set (same priority, created_at within one + // second) lands at a different row on two workspaces seeded seconds + // apart, so the limited rows are not compared across sides. Instead each + // side's limited rows must be, in order, a prefix of that SAME side's + // PrefixOf listing, both sides must return as many rows, and the two + // PrefixOf listings must hold the same ids. + PrefixOf []string `yaml:"prefix_of"` + // IgnoreFields are extra JSON keys dropped before comparing. + IgnoreFields []string `yaml:"ignore_fields"` + // ExpectExit is the exit code both sides must produce (default 0). An + // allowlisted shape must state it: the embedded run is held to it even + // though the served run is only held to the allowlisted refusal. + ExpectExit *int `yaml:"expect_exit"` + // ExpectStderr, when set, must match both sides' stderr (only the + // embedded side's on an allowlisted shape). + ExpectStderr string `yaml:"expect_stderr"` + // Verify are reads run on both sides after the shape and compared as json: + // how a write is checked for having done the same thing on both backends. + Verify [][]string `yaml:"verify"` + // Pending marks a known failure and names the slice that fixes it. + Pending *pending `yaml:"pending"` +} + +// wantExit is the exit code the embedded run must produce. +func (s shape) wantExit() int { + if s.ExpectExit == nil { + return 0 + } + return *s.ExpectExit +} + +type pending struct { + Slice string `yaml:"slice"` + Reason string `yaml:"reason"` +} + +// allowEntry is testdata/allowlist.yaml: a shape that is EXPECTED to be +// refused over http, permanently, for a stated reason. The served run must +// fail and match Refusal; if it ever succeeds the entry is stale and fails. +// The embedded run is still held to the shape's expect_exit and +// expect_stderr, and nothing else of the shape is checked. +type allowEntry struct { + ID string `yaml:"id"` + Reason string `yaml:"reason"` + Refusal string `yaml:"refusal"` +} + +// refusalMarkers are the texts a refusal of an operation over http carries: +// issueops.ErrUnsupported, encode.RefusedError, the client's capability +// preflight (wire.CapabilityError, which never reaches the proxy), the +// server's loopback-only unlimited-read 400, an unrouted 404, and a strict +// decoder's unknown member. +var refusalMarkers = regexp.MustCompile(`(?i)not supported by the|not expressible on the v0 wire|loopback-only|capability "[^"]+" not advertised|does not advertise capability|no such route on this server|unknown_parameter|unknown query parameter|json: unknown field`) + +// volatileKey names JSON members that legitimately differ between two +// workspaces fed the same commands: timestamps and revision tokens. +var volatileKey = regexp.MustCompile(`(^|_)at$|^revision$|^row_version$|^expected_revision$|^current_revision$`) + +// callerTimeKey names the timestamps a shape's own arguments set (bd create +// --defer, --due). They are compared as given, neither dropped as volatile +// nor scrubbed to , so a client that sends the wrong one fails; a shape +// must give them as absolute dates, since a relative one resolves against +// each side's own clock. +var callerTimeKey = map[string]bool{"defer_until": true, "due_at": true} + +var timestampText = regexp.MustCompile(`\d{4}-\d{2}-\d{2}[T ]\d{2}:\d{2}(:\d{2}(\.\d+)?)?(Z|[+-]\d{2}:?\d{2})?`) + +func loadCorpus(t *testing.T) (*corpusFile, map[string]allowEntry) { + t.Helper() + raw, err := os.ReadFile("testdata/corpus.yaml") + if err != nil { + t.Fatalf("read corpus: %v", err) + } + var c corpusFile + dec := yaml.NewDecoder(strings.NewReader(string(raw))) + dec.KnownFields(true) + if err := dec.Decode(&c); err != nil { + t.Fatalf("parse testdata/corpus.yaml: %v", err) + } + rawAllow, err := os.ReadFile("testdata/allowlist.yaml") + if err != nil { + t.Fatalf("read allowlist: %v", err) + } + var entries []allowEntry + dec = yaml.NewDecoder(strings.NewReader(string(rawAllow))) + dec.KnownFields(true) + if err := dec.Decode(&entries); err != nil { + t.Fatalf("parse testdata/allowlist.yaml: %v", err) + } + allow := map[string]allowEntry{} + for _, e := range entries { + if e.ID == "" || strings.TrimSpace(e.Reason) == "" || e.Refusal == "" { + t.Fatalf("allowlist entry %+v needs id, reason and refusal", e) + } + allow[e.ID] = e + } + return &c, allow +} + +// TestCorpusFileIsWellFormed runs without the embedded engine: it keeps the +// data honest (unique ids, a source and a slice for every pending mark, no +// allowlist entry for a shape that does not exist, no shape both pending and +// allowlisted, an allowlisted shape states its embedded exit code and +// carries no check that the allowlist would skip, every expect_stderr and +// refusal compiles, and the reference profile loads). +func TestCorpusFileIsWellFormed(t *testing.T) { + c, allow := loadCorpus(t) + seen := map[string]bool{} + for _, s := range c.Shapes { + if s.ID == "" || len(s.Args) == 0 { + t.Errorf("shape %+v needs an id and args", s) + continue + } + if seen[s.ID] { + t.Errorf("duplicate shape id %q", s.ID) + } + seen[s.ID] = true + if strings.TrimSpace(s.Source) == "" { + t.Errorf("shape %s names no source", s.ID) + } + if s.Pending != nil && (s.Pending.Slice == "" || strings.TrimSpace(s.Pending.Reason) == "") { + t.Errorf("shape %s is pending without a slice and a reason", s.ID) + } + if _, ok := allow[s.ID]; ok { + if s.Pending != nil { + t.Errorf("shape %s is both pending and allowlisted; pick one", s.ID) + } + if s.ExpectExit == nil { + t.Errorf("shape %s is allowlisted without expect_exit; state the exit code the embedded run must produce", s.ID) + } + if modeOf(s) != "exit" || len(s.Verify) > 0 { + t.Errorf("shape %s is allowlisted, so only its exit code and stderr are checked; give it compare: exit and no verify", s.ID) + } + } + switch s.Compare { + case "", "json", "ids", "count", "text", "exit": + default: + t.Errorf("shape %s: unknown compare mode %q", s.ID, s.Compare) + } + if len(s.PrefixOf) > 0 && s.Compare != "ids" { + t.Errorf("shape %s: prefix_of needs compare: ids", s.ID) + } + if _, err := regexp.Compile(s.ExpectStderr); err != nil { + t.Errorf("shape %s: expect_stderr: %v", s.ID, err) + } + } + for id, entry := range allow { + if !seen[id] { + t.Errorf("allowlist names %q, which is not a corpus shape", id) + } + if _, err := regexp.Compile(entry.Refusal); err != nil { + t.Errorf("allowlist entry %s: refusal: %v", id, err) + } + } + for _, a := range c.Files { + if (a.Content == "") == (a.Bulk == nil) { + t.Errorf("asset %s needs exactly one of content and bulk", a.Name) + } + } + loadReferenceProfile(t) +} + +// side is one workspace the corpus runs in. +type side struct { + name string + dir string + files string + env []string +} + +type outcome struct { + id, status, detail string + requests int64 +} + +// TestServedCorpus is the suite. Each shape is a subtest; a pending shape is +// still RUN and then skipped with what it did, so a slice that fixes it sees +// the subtest fail with "now passes; remove the pending mark". +// BEADS_CORPUS_SKIP_PENDING=1 skips pending shapes without running them. +// BEADS_CORPUS_REPORT= writes a JSON summary. +func TestServedCorpus(t *testing.T) { + skipUnlessEmbeddedDolt(t) + c, allow := loadCorpus(t) + profile := loadReferenceProfile(t) + bin := buildBD(t) + + serverAddr := startCorpusServer(t) + proxy := newReferenceProxy(t, profile, serverAddr) + proxyAddr := startProxy(t, proxy) + + common := []string{ + "BEADS_ACTOR=worker", + "CLAUDE_SESSION_ID=corpus-session-s23", + "BD_BACKUP_ENABLED=false", + "BD_EXPORT_AUTO=false", + } + served := &side{name: "served", dir: t.TempDir(), files: t.TempDir(), + env: append([]string{"BEADS_HTTP_TOKEN=" + proxyAddr + "=" + corpusToken}, common...)} + local := &side{name: "embedded", dir: t.TempDir(), files: t.TempDir(), env: common} + + if r := runBD(t, bin, served.dir, served.env, "", "connect", "http://"+proxyAddr, "--expect-project-id", corpusProjectID, "--json"); r.code != 0 { + t.Fatalf("bd connect: %s", r) + } + if r := runBD(t, bin, local.dir, local.env, "", "init", "-p", corpusPrefix, "--non-interactive", "--skip-hooks", "--skip-agents", "-q"); r.code != 0 { + t.Fatalf("bd init (embedded): %s", r) + } + for _, sd := range []*side{served, local} { + writeAssets(t, c.Files, sd.files) + } + for _, args := range c.Seed { + for _, sd := range []*side{served, local} { + argv := expand(t, bin, sd, args) + if r := runBD(t, bin, sd.dir, sd.env, "", argv...); r.code != 0 { + t.Fatalf("seed %v on %s failed; the seed must use shapes that work on the reference profile: %s", args, sd.name, r) + } + } + } + proxy.takeRefusals() + + skipPending := os.Getenv("BEADS_CORPUS_SKIP_PENDING") == "1" + var outcomes []outcome + for _, s := range c.Shapes { + s := s + t.Run(s.ID, func(t *testing.T) { + o := outcome{id: s.ID} + defer func() { outcomes = append(outcomes, o) }() + if s.Pending != nil && skipPending { + o.status = "pending:" + s.Pending.Slice + t.Skipf("pending %s: %s", s.Pending.Slice, s.Pending.Reason) + } + before := proxy.requests.Load() + failures := runShape(t, bin, s, served, local, allow, proxy) + o.requests = proxy.requests.Load() - before + entry, allowed := allow[s.ID] + switch { + case hasCorpusBug(failures): + // The embedded side itself disagrees with the corpus: no pending + // mark or allowlist entry may hide that, or a pending shape that + // can never pass would sit in the corpus forever. + o.status = "FAIL(corpus)" + o.detail = failures[0] + for _, f := range failures { + t.Error(f) + } + case s.Pending != nil && len(failures) == 0: + o.status = "FAIL(pending-now-passes)" + t.Errorf("shape is marked pending %s but now passes on the reference profile; remove the pending mark from testdata/corpus.yaml", s.Pending.Slice) + case s.Pending != nil: + o.status = "pending:" + s.Pending.Slice + o.detail = failures[0] + t.Skipf("pending %s: %s\nobserved:\n%s", s.Pending.Slice, s.Pending.Reason, strings.Join(failures, "\n")) + case len(failures) > 0: + o.status = "FAIL" + o.detail = failures[0] + for _, f := range failures { + t.Error(f) + } + case allowed: + o.status = "allowed" + o.detail = entry.Reason + default: + o.status = "pass" + } + }) + } + + summarize(t, outcomes) +} + +// runShape runs one shape on both sides and returns every way the served run +// fell short. It never calls t.Error itself, so the caller can turn a pending +// shape's failures into a skip. +func runShape(t *testing.T, bin string, s shape, served, local *side, allow map[string]allowEntry, proxy *referenceProxy) []string { + t.Helper() + var fails []string + failf := func(format string, a ...any) { fails = append(fails, fmt.Sprintf(format, a...)) } + wantStderr, err := regexp.Compile(s.ExpectStderr) + if err != nil { + failf("CORPUS BUG: expect_stderr %q: %v", s.ExpectStderr, err) + return fails + } + + for _, args := range s.Setup { + for _, sd := range []*side{served, local} { + if r := runBD(t, bin, sd.dir, sd.env, "", expand(t, bin, sd, args)...); r.code != 0 { + failf("setup %v failed on %s: %s", args, sd.name, r) + return fails + } + } + } + proxy.takeRefusals() + sr := runBD(t, bin, served.dir, served.env, s.Stdin, expand(t, bin, served, s.Args)...) + wireRefusals := proxy.takeRefusals() + lr := runBD(t, bin, local.dir, local.env, s.Stdin, expand(t, bin, local, s.Args)...) + t.Logf("bd %s\nserved: %s\nembedded: exit=%d", strings.Join(s.Args, " "), sr, lr.code) + if len(wireRefusals) > 0 { + t.Logf("reference-profile refusals on the wire: %v", wireRefusals) + } + + // The embedded run is the oracle for every shape, an allowlisted one + // included: when it no longer does what the shape says, the shape is + // wrong or bd changed, whatever the served run did. + if lr.code != s.wantExit() { + failf("CORPUS BUG: the embedded run exited %d, want %d; the shape itself is wrong or bd changed: %s", lr.code, s.wantExit(), lr) + return fails + } + if s.ExpectStderr != "" && !wantStderr.MatchString(lr.stderr) { + failf("CORPUS BUG: embedded stderr does not match %q: %s", s.ExpectStderr, lr.stderr) + return fails + } + + if entry, ok := allow[s.ID]; ok { + re, err := regexp.Compile(entry.Refusal) + if err != nil { + failf("allowlist refusal regexp for %s: %v", s.ID, err) + return fails + } + if sr.code == 0 { + failf("allowlisted (%s) but the served run SUCCEEDED; drop it from testdata/allowlist.yaml and give it a comparison", entry.Reason) + } else if !re.MatchString(sr.stderr + sr.stdout) { + failf("allowlisted refusal did not match %q (a different failure than the one allowed): %s", entry.Refusal, sr) + } + return fails + } + + if m := refusalMarkers.FindString(sr.stderr + sr.stdout); m != "" { + failf("served run carries a refusal (%q): %s", m, sr) + } else if len(wireRefusals) > 0 && sr.code != 0 { + failf("served run failed on a reference-profile refusal %v: %s", wireRefusals, sr) + } + if sr.code != lr.code { + failf("exit code: served %d, embedded %d: %s", sr.code, lr.code, sr) + } + if s.ExpectStderr != "" && !wantStderr.MatchString(sr.stderr) { + failf("served stderr does not match %q (embedded does): %s", s.ExpectStderr, sr) + } + if len(fails) > 0 { + return fails + } + if len(s.PrefixOf) > 0 { + vs := runBD(t, bin, served.dir, served.env, "", expand(t, bin, served, s.PrefixOf)...) + vl := runBD(t, bin, local.dir, local.env, "", expand(t, bin, local, s.PrefixOf)...) + if vs.code != 0 || vl.code != 0 { + failf("prefix_of %v: exit served %d, embedded %d: served %s", s.PrefixOf, vs.code, vl.code, vs) + } else if d := comparePrefix(sr.stdout, lr.stdout, vs.stdout, vl.stdout); d != "" { + failf("output parity (ids, prefix_of): %s", d) + } + } else if d := compareOutputs(s, sr.stdout, lr.stdout); d != "" { + failf("output parity (%s): %s", modeOf(s), d) + } + for _, args := range s.Verify { + vs := runBD(t, bin, served.dir, served.env, "", expand(t, bin, served, args)...) + vl := runBD(t, bin, local.dir, local.env, "", expand(t, bin, local, args)...) + if vs.code != vl.code { + failf("verify %v: exit served %d, embedded %d: served %s", args, vs.code, vl.code, vs) + continue + } + if vl.code != 0 { + continue // both refused the read (e.g. a deleted id): equal is the check + } + if d := compareJSON(vs.stdout, vl.stdout, s.IgnoreFields, false); d != "" { + failf("verify %v diverged after the write: %s", args, d) + } + } + return fails +} + +func hasCorpusBug(failures []string) bool { + for _, f := range failures { + if strings.HasPrefix(f, "CORPUS BUG") { + return true + } + } + return false +} + +func modeOf(s shape) string { + if s.Compare == "" { + return "json" + } + return s.Compare +} + +func compareOutputs(s shape, served, local string) string { + switch modeOf(s) { + case "exit": + return "" + case "text": + a, b := scrubText(served), scrubText(local) + if a != b { + return fmt.Sprintf("text differs\nserved:\n%s\nembedded:\n%s", clip(a, 3000), clip(b, 3000)) + } + return "" + case "count": + a, errA := topLevelLen(served) + b, errB := topLevelLen(local) + if errA != nil || errB != nil { + return fmt.Sprintf("count: served err=%v embedded err=%v\nserved:\n%s", errA, errB, clip(served, 2000)) + } + if a != b { + return fmt.Sprintf("row count: served %d, embedded %d", a, b) + } + return "" + case "ids": + a, errA := idList(served, s.Ordered) + b, errB := idList(local, s.Ordered) + if errA != nil || errB != nil { + return fmt.Sprintf("ids: served err=%v embedded err=%v\nserved:\n%s", errA, errB, clip(served, 2000)) + } + if !reflect.DeepEqual(a, b) { + return fmt.Sprintf("ids differ\nserved (%d): %v\nembedded (%d): %v", len(a), a, len(b), b) + } + return "" + default: + return compareJSON(served, local, s.IgnoreFields, s.Ordered) + } +} + +// comparePrefix holds a limited listing to its unlimited form on each side: +// see shape.PrefixOf. +func comparePrefix(served, local, servedAll, localAll string) string { + lists := make([][]string, 4) + for i, out := range []string{served, local, servedAll, localAll} { + ids, err := idList(out, true) + if err != nil { + return fmt.Sprintf("listing %d: %v\n%s", i, err, clip(out, 2000)) + } + lists[i] = ids + } + s, l, sAll, lAll := lists[0], lists[1], lists[2], lists[3] + if len(s) != len(l) { + return fmt.Sprintf("row count: served %d, embedded %d", len(s), len(l)) + } + for _, c := range []struct { + name string + rows, all []string + }{{"served", s, sAll}, {"embedded", l, lAll}} { + if len(c.rows) > len(c.all) || !reflect.DeepEqual(c.rows, c.all[:len(c.rows)]) { + return fmt.Sprintf("%s limited rows are not a prefix of its own unlimited listing\nlimited: %v\nunlimited: %v", c.name, c.rows, c.all) + } + } + sSet := append([]string(nil), sAll...) + lSet := append([]string(nil), lAll...) + sort.Strings(sSet) + sort.Strings(lSet) + if !reflect.DeepEqual(sSet, lSet) { + return fmt.Sprintf("unlimited ids differ\nserved (%d): %v\nembedded (%d): %v", len(sSet), sSet, len(lSet), lSet) + } + return "" +} + +func scrubText(s string) string { + return strings.TrimSpace(timestampText.ReplaceAllString(s, "")) +} + +func decode(s string) (any, error) { + var v any + if err := json.Unmarshal([]byte(strings.TrimSpace(s)), &v); err != nil { + return nil, fmt.Errorf("not JSON: %w", err) + } + return v, nil +} + +// rows finds the row array in a bd --json payload: the payload itself, or the +// first array member of an envelope object. +func rows(v any) ([]any, bool) { + switch x := v.(type) { + case []any: + return x, true + case map[string]any: + for _, k := range []string{"issues", "data", "items", "results"} { + if arr, ok := x[k].([]any); ok { + return arr, true + } + } + } + return nil, false +} + +func topLevelLen(s string) (int, error) { + v, err := decode(s) + if err != nil { + return 0, err + } + arr, ok := rows(v) + if !ok { + return 0, fmt.Errorf("no row array in output") + } + return len(arr), nil +} + +func idList(s string, ordered bool) ([]string, error) { + v, err := decode(s) + if err != nil { + return nil, err + } + arr, ok := rows(v) + if !ok { + return nil, fmt.Errorf("no row array in output") + } + out := []string{} + for _, r := range arr { + if m, ok := r.(map[string]any); ok { + if id, ok := m["id"].(string); ok { + out = append(out, id) + } + } + } + if !ordered { + sort.Strings(out) + } + return out, nil +} + +func compareJSON(served, local string, ignore []string, ordered bool) string { + a, errA := decode(served) + b, errB := decode(local) + if errA != nil || errB != nil { + return fmt.Sprintf("served err=%v embedded err=%v\nserved:\n%s\nembedded:\n%s", errA, errB, clip(served, 2000), clip(local, 2000)) + } + drop := map[string]bool{} + for _, k := range ignore { + drop[k] = true + } + a, b = normalize(a, drop, ordered), normalize(b, drop, ordered) + if path, x, y, same := firstDiff("$", a, b); !same { + return fmt.Sprintf("first difference at %s: served=%s embedded=%s", path, brief(x), brief(y)) + } + return "" +} + +func normalize(v any, drop map[string]bool, ordered bool) any { + switch x := v.(type) { + case map[string]any: + out := make(map[string]any, len(x)) + for k, val := range x { + if drop[k] { + continue + } + if callerTimeKey[k] { + out[k] = val + continue + } + if volatileKey.MatchString(k) { + continue + } + out[k] = normalize(val, drop, ordered) + } + return out + case []any: + out := make([]any, len(x)) + for i, e := range x { + out[i] = normalize(e, drop, ordered) + } + if !ordered { + sort.SliceStable(out, func(i, j int) bool { return sortKey(out[i]) < sortKey(out[j]) }) + } + return out + case string: + return timestampText.ReplaceAllString(x, "") + default: + return v + } +} + +func sortKey(v any) string { + if m, ok := v.(map[string]any); ok { + for _, k := range []string{"id", "issue_id", "key"} { + if s, ok := m[k].(string); ok { + return k + "=" + s + } + } + if inner, ok := m["issue"].(map[string]any); ok { + if s, ok := inner["id"].(string); ok { + return "issue.id=" + s + } + } + } + raw, _ := json.Marshal(v) + return string(raw) +} + +func firstDiff(path string, a, b any) (string, any, any, bool) { + switch x := a.(type) { + case map[string]any: + y, ok := b.(map[string]any) + if !ok { + return path, a, b, false + } + keys := map[string]bool{} + for k := range x { + keys[k] = true + } + for k := range y { + keys[k] = true + } + sorted := make([]string, 0, len(keys)) + for k := range keys { + sorted = append(sorted, k) + } + sort.Strings(sorted) + for _, k := range sorted { + xv, xok := x[k] + yv, yok := y[k] + if !xok || !yok { + return path + "." + k, xv, yv, false + } + if p, l, r, same := firstDiff(path+"."+k, xv, yv); !same { + return p, l, r, false + } + } + return "", nil, nil, true + case []any: + y, ok := b.([]any) + if !ok { + return path, a, b, false + } + if len(x) != len(y) { + return path + ".length", len(x), len(y), false + } + for i := range x { + if p, l, r, same := firstDiff(path+"["+strconv.Itoa(i)+"]", x[i], y[i]); !same { + return p, l, r, false + } + } + return "", nil, nil, true + default: + if reflect.DeepEqual(a, b) { + return "", nil, nil, true + } + return path, a, b, false + } +} + +func brief(v any) string { + if v == nil { + return "" + } + raw, _ := json.Marshal(v) + return clip(string(raw), 400) +} + +var placeholder = regexp.MustCompile(`\{(file|rev):([^}]+)\}`) + +// expand substitutes {file:NAME} and {rev:ID} in args for side sd. +func expand(t *testing.T, bin string, sd *side, args []string) []string { + t.Helper() + out := make([]string, len(args)) + for i, a := range args { + out[i] = placeholder.ReplaceAllStringFunc(a, func(m string) string { + parts := placeholder.FindStringSubmatch(m) + switch parts[1] { + case "file": + return filepath.Join(sd.files, parts[2]) + default: + r := runBD(t, bin, sd.dir, sd.env, "", "show", parts[2], "--json") + if r.code != 0 { + t.Logf("{rev:%s} on %s: show failed: %s", parts[2], sd.name, r) + return "0" + } + v, err := decode(r.stdout) + if err != nil { + return "0" + } + if arr, ok := rows(v); ok && len(arr) > 0 { + v = arr[0] + } + if m, ok := v.(map[string]any); ok { + if rev, ok := m["revision"].(string); ok { + return rev + } + } + return "0" + } + }) + } + return out +} + +func writeAssets(t *testing.T, assets []corpusAsset, dir string) { + t.Helper() + for _, a := range assets { + content := a.Content + if a.Bulk != nil { + content = bulkPlan(*a.Bulk) + } + if err := os.WriteFile(filepath.Join(dir, a.Name), []byte(content), 0o600); err != nil { + t.Fatalf("write asset %s: %v", a.Name, err) + } + } +} + +func bulkPlan(b bulkGraph) string { + type node struct { + Key string `json:"key"` + ID string `json:"id"` + Title string `json:"title"` + Type string `json:"type"` + Priority int `json:"priority"` + Ephemeral bool `json:"ephemeral,omitempty"` + Metadata map[string]string `json:"metadata,omitempty"` + } + plan := struct { + CommitMessage string `json:"commit_message"` + Nodes []node `json:"nodes"` + }{CommitMessage: "corpus bulk " + b.IDPrefix} + for i := 1; i <= b.Count; i++ { + id := fmt.Sprintf("%s-%03d", b.IDPrefix, i) + plan.Nodes = append(plan.Nodes, node{Key: id, ID: id, Title: "bulk " + id, Type: "task", + Priority: b.Priority, Ephemeral: b.Ephemeral, Metadata: b.Metadata}) + } + raw, _ := json.MarshalIndent(plan, "", " ") + return string(raw) +} + +func summarize(t *testing.T, outcomes []outcome) { + t.Helper() + counts := map[string]int{} + var lines []string + for _, o := range outcomes { + key := o.status + if strings.HasPrefix(key, "pending:") { + key = "pending" + } + counts[key]++ + lines = append(lines, fmt.Sprintf("%-26s %-48s req=%-3d %s", o.status, o.id, o.requests, firstLine(o.detail))) + } + t.Logf("corpus summary: %v\n%s", counts, strings.Join(lines, "\n")) + if path := os.Getenv("BEADS_CORPUS_REPORT"); path != "" { + type row struct { + ID string `json:"id"` + Status string `json:"status"` + Requests int64 `json:"requests"` + Detail string `json:"detail,omitempty"` + } + var out []row + for _, o := range outcomes { + out = append(out, row{o.id, o.status, o.requests, o.detail}) + } + raw, _ := json.MarshalIndent(out, "", " ") + if err := os.WriteFile(path, raw, 0o600); err != nil { + t.Errorf("write BEADS_CORPUS_REPORT: %v", err) + } + } +} + +func firstLine(s string) string { + s = strings.TrimSpace(s) + if i := strings.IndexByte(s, '\n'); i >= 0 { + s = s[:i] + } + return clip(s, 160) +} diff --git a/backend/http/corpus/harness_test.go b/backend/http/corpus/harness_test.go new file mode 100644 index 000000000..3b93d0473 --- /dev/null +++ b/backend/http/corpus/harness_test.go @@ -0,0 +1,312 @@ +//go:build cgo + +package corpus_test + +import ( + "context" + "errors" + "fmt" + "io" + "net" + "net/http" + "os" + "os/exec" + "path/filepath" + "runtime" + "strings" + "sync" + "testing" + "time" + + "github.com/steveyegge/beads/internal/httpapi" + "github.com/steveyegge/beads/internal/storage" + "github.com/steveyegge/beads/internal/storage/domain" + "github.com/steveyegge/beads/internal/storage/embeddeddolt" + "github.com/steveyegge/beads/internal/testutil/bazeltest" +) + +const ( + corpusProjectID = "proj-corpus-s23" + corpusDatabase = "corpus" + corpusPrefix = "e2e" + // corpusToken is the bearer the in-process server accepts. The reference + // deployment authenticates every request, so the suite does too. + // #nosec G101 -- a throwaway test credential for an httptest-scoped server. + corpusToken = "corpus-s23-test-token" + // bdTimeout bounds one bd subprocess so a hung shape fails rather than + // stalling the whole suite. + bdTimeout = 90 * time.Second +) + +// startCorpusServer binds the in-process server the way the reference +// deployment is bound: AllowNonLoopback with a bearer token file. The socket is +// still 127.0.0.1 (this suite never leaves the host), but every behavior the +// server keys off its bind mode — the loopback-only refusal of an unlimited +// read above all — is the non-loopback one, because that is decided by +// Config.AllowNonLoopback and not by the address. +func startCorpusServer(t *testing.T) string { + t.Helper() + beadsDir := t.TempDir() + ctx := context.Background() + store, err := embeddeddolt.Open(ctx, beadsDir, corpusDatabase, "main") + if err != nil { + t.Fatalf("open the served store: %v", err) + } + t.Cleanup(func() { _ = store.Close() }) + if err := store.SetConfig(ctx, "issue_prefix", corpusPrefix); err != nil { + t.Fatalf("set the issue prefix: %v", err) + } + + tokenFile := filepath.Join(t.TempDir(), "tokens") + if err := os.WriteFile(tokenFile, []byte(corpusToken+"\n"), 0o600); err != nil { + t.Fatalf("write the token file: %v", err) + } + auth, err := httpapi.NewTokenFileAuth(tokenFile) + if err != nil { + t.Fatalf("load the token file: %v", err) + } + + cfg := corpusServeConfig(t, store) + cfg.AllowNonLoopback = true + cfg.Auth = auth + srv, err := httpapi.Listen(cfg) + if err != nil { + t.Fatalf("bind the in-process server: %v", err) + } + serveCtx, cancel := context.WithCancel(context.Background()) + done := make(chan struct{}) + go func() { defer close(done); _ = srv.Serve(serveCtx) }() + t.Cleanup(func() { + cancel() + select { + case <-done: + case <-time.After(30 * time.Second): + t.Error("the in-process server did not shut down") + } + }) + return srv.Addr() +} + +// startProxy serves h on an ephemeral loopback port. +func startProxy(t *testing.T, h http.Handler) string { + t.Helper() + srv := &http.Server{Handler: h, ReadHeaderTimeout: 10 * time.Second} + ln, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatalf("listen for the reference proxy: %v", err) + } + go func() { _ = srv.Serve(ln) }() + t.Cleanup(func() { _ = srv.Close() }) + return ln.Addr().String() +} + +// corpusServeConfig takes every role off the store, as bd serve does. It +// mirrors backend/http's e2eServeConfig (a sibling test package, so it cannot +// be imported). +func corpusServeConfig(t *testing.T, s storage.DoltStorage) httpapi.Config { + t.Helper() + cfg := httpapi.Config{ + Addr: "127.0.0.1:0", + Stdout: io.Discard, + Stderr: io.Discard, + Workspace: domain.ContextInfo{ProjectID: corpusProjectID, Database: corpusDatabase}, + } + var err error + must := func(name string) { + t.Helper() + if err != nil { + t.Fatalf("%s(): %v", name, err) + } + } + cfg.Reader, err = s.IssueReader() + must("IssueReader") + cfg.Claimer, err = s.IssueClaimer() + must("IssueClaimer") + cfg.ReadyClaimer, err = s.ReadyClaimer() + must("ReadyClaimer") + cfg.Releaser, err = s.Releaser() + must("Releaser") + cfg.Lifecycle, err = s.IssueLifecycle() + must("IssueLifecycle") + cfg.Settings, err = s.WorkspaceConfig() + must("WorkspaceConfig") + cfg.Stats, err = s.StatsReporter() + must("StatsReporter") + cfg.CycleDetector, err = s.CycleDetector() + must("CycleDetector") + cfg.EdgeReader, err = s.EdgeReader() + must("EdgeReader") + cfg.GraphCounter, err = s.GraphCounter() + must("GraphCounter") + cfg.BatchGetter, err = s.BatchGetter() + must("BatchGetter") + cfg.LeaseReclaimer, err = s.LeaseReclaimer() + must("LeaseReclaimer") + cfg.Relations, err = s.IssueRelations() + must("IssueRelations") + cfg.Commenter, err = s.Commenter() + must("Commenter") + cfg.BlockingAnnotator, err = s.BlockingAnnotator() + must("BlockingAnnotator") + cfg.TreeWalker, err = s.TreeWalker() + must("TreeWalker") + cfg.ReadyCounter, err = s.ReadyCounter() + must("ReadyCounter") + cfg.Counter, err = s.Counter() + must("Counter") + cfg.Querier, err = s.Querier() + must("Querier") + cfg.Sweeper, err = s.Sweeper() + must("Sweeper") + cfg.Deleter, err = s.Deleter() + must("Deleter") + cfg.BatchCreator, err = s.BatchCreator() + must("BatchCreator") + cfg.BatchCloser, err = s.BatchCloser() + must("BatchCloser") + cfg.DependencyEditor, err = s.DependencyEditor() + must("DependencyEditor") + cfg.MetadataCAS, err = s.MetadataCAS() + must("MetadataCAS") + cfg.BatchApplier, err = s.BatchApplier() + must("BatchApplier") + cfg.Memories, err = s.Memories() + must("Memories") + return cfg +} + +// This file is the corpus tier's cgo half; see TestCorpusTierIsLinkedWhenRequired. +func init() { corpusTierLinked = true } + +// skipUnlessEmbeddedDolt is the served tier's gate, as in backend/http. +func skipUnlessEmbeddedDolt(t *testing.T) { + t.Helper() + if os.Getenv("BEADS_TEST_EMBEDDED_DOLT") == "1" { + return + } + if os.Getenv("BEADS_HTTP_TEST_REQUIRED") == "1" { + t.Fatal("BEADS_HTTP_TEST_REQUIRED=1 but BEADS_TEST_EMBEDDED_DOLT is not 1; the http corpus tier is not enforced") + } + t.Skip("set BEADS_TEST_EMBEDDED_DOLT=1 to run the http command-corpus tier") +} + +type bdResult struct { + stdout, stderr string + code int +} + +func (r bdResult) String() string { + return fmt.Sprintf("exit=%d\nstdout:\n%s\nstderr:\n%s", r.code, clip(r.stdout, 4000), clip(r.stderr, 4000)) +} + +func clip(s string, n int) string { + if len(s) <= n { + return s + } + return s[:n] + fmt.Sprintf("\n... (%d more bytes)", len(s)-n) +} + +// runBD runs the built bd binary in dir with a hermetic environment. +func runBD(t *testing.T, bin, dir string, env []string, stdin string, args ...string) bdResult { + t.Helper() + ctx, cancel := context.WithTimeout(context.Background(), bdTimeout) + defer cancel() + cmd := exec.CommandContext(ctx, bin, args...) + cmd.Dir = dir + cmd.Env = minimalBDEnv(dir, env...) + if stdin != "" { + cmd.Stdin = strings.NewReader(stdin) + } + var o, e strings.Builder + cmd.Stdout, cmd.Stderr = &o, &e + err := cmd.Run() + code := 0 + if err != nil { + var ee *exec.ExitError + switch { + case ctx.Err() != nil: + code = -1 + e.WriteString(fmt.Sprintf("\n[corpus harness] bd timed out after %s", bdTimeout)) + case errors.As(err, &ee): + code = ee.ExitCode() + default: + t.Fatalf("running bd %s: %v", strings.Join(args, " "), err) + } + } + return bdResult{stdout: o.String(), stderr: e.String(), code: code} +} + +// minimalBDEnv is backend/http's hermetic bd environment: PATH and the few +// variables a Go binary needs, HOME repointed at dir, nothing BEADS_* carried +// over from the invoking shell. extra is applied last. +func minimalBDEnv(dir string, extra ...string) []string { + env := []string{ + "HOME=" + dir, + "BD_DISABLE_METRICS=1", + "BD_DISABLE_EVENT_FLUSH=1", + } + for _, key := range []string{"PATH", "TMPDIR", "SYSTEMROOT", "WINDIR"} { + if v, ok := os.LookupEnv(key); ok { + env = append(env, key+"="+v) + } + } + return append(env, extra...) +} + +var ( + buildBDOnce sync.Once + bdBinPath string + bdBuildErr error + bdBinOwnBuild bool +) + +// buildBD builds bd from this tree once per test process (or takes Bazel's +// prebuilt binary). TestMain deletes the build. +func buildBD(t *testing.T) string { + t.Helper() + buildBDOnce.Do(func() { + if bazeltest.IsBazel() { + bdBinPath, bdBuildErr = bazeltest.PrebuiltBD() + return + } + if prebuilt, err := bazeltest.PrebuiltBD(); err != nil || prebuilt != "" { + bdBinPath, bdBuildErr = prebuilt, err + return + } + dir, err := os.MkdirTemp(os.Getenv("TMPDIR"), "bd-corpus") + if err != nil { + bdBuildErr = fmt.Errorf("mkdir temp for bd binary: %w", err) + return + } + bin := filepath.Join(dir, "bd-corpus") + cmd := exec.Command("go", "build", "-tags", "gms_pure_go", "-o", bin, "./cmd/bd") + cmd.Dir = repoRoot() + if out, err := cmd.CombinedOutput(); err != nil { + bdBuildErr = fmt.Errorf("build bd: %w\n%s", err, out) + _ = os.RemoveAll(dir) + return + } + bdBinPath = bin + bdBinOwnBuild = true + }) + if bdBuildErr != nil { + t.Fatal(bdBuildErr) + } + if bdBinPath == "" { + t.Fatal("no bd binary: Bazel did not provide one") + } + return bdBinPath +} + +func TestMain(m *testing.M) { + code := m.Run() + if bdBinOwnBuild && bdBinPath != "" { + _ = os.RemoveAll(filepath.Dir(bdBinPath)) + } + os.Exit(code) +} + +func repoRoot() string { + _, file, _, _ := runtime.Caller(0) // /backend/http/corpus/harness_test.go + return filepath.Dir(filepath.Dir(filepath.Dir(filepath.Dir(file)))) +} diff --git a/backend/http/corpus/reference_proxy_test.go b/backend/http/corpus/reference_proxy_test.go new file mode 100644 index 000000000..0142b1ad4 --- /dev/null +++ b/backend/http/corpus/reference_proxy_test.go @@ -0,0 +1,279 @@ +//go:build cgo + +package corpus_test + +import ( + "bytes" + "encoding/json" + "fmt" + "io" + "maps" + "net/http" + "net/http/httputil" + "net/url" + "os" + "regexp" + "slices" + "sort" + "strings" + "sync" + "sync/atomic" + "testing" + + "github.com/steveyegge/beads/internal/httpapi" + "github.com/steveyegge/beads/internal/httpapi/apigen" +) + +// referenceProfile is testdata/reference_server.json: the wire surface of the +// deployed server the corpus must work against. See README.md. +type referenceProfile struct { + Capabilities []string `json:"capabilities"` + ContextMembers []string `json:"context_members"` + Operations []referenceOperation `json:"operations"` +} + +type referenceOperation struct { + Method string `json:"method"` + Path string `json:"path"` + OperationID string `json:"operation_id"` + Query []string `json:"query"` + Body []string `json:"body"` + + pattern *regexp.Regexp + query map[string]bool + body map[string]bool +} + +func loadReferenceProfile(t *testing.T) *referenceProfile { + t.Helper() + raw, err := os.ReadFile("testdata/reference_server.json") + if err != nil { + t.Fatalf("read the reference server profile: %v", err) + } + var p referenceProfile + if err := json.Unmarshal(raw, &p); err != nil { + t.Fatalf("parse testdata/reference_server.json: %v", err) + } + if len(p.Capabilities) == 0 || len(p.Operations) == 0 || len(p.ContextMembers) == 0 { + t.Fatal("testdata/reference_server.json is missing capabilities, operations or context_members") + } + for i := range p.Operations { + op := &p.Operations[i] + pattern, err := templatePattern(op.Path) + if err != nil { + t.Fatalf("testdata/reference_server.json: operation %s path %q: %v", op.OperationID, op.Path, err) + } + op.pattern = pattern + op.query = setOf(op.Query) + op.body = setOf(op.Body) + } + return &p +} + +// templatePattern turns an OpenAPI path template into an anchored regexp. A +// path parameter never spans a "/" or a ":" (AIP-136 custom methods hang off +// the last segment as {id}:verb, and no issue id, config key or memory key +// carries a raw colon), so /issues/{id} does not swallow /issues/{id}:claim. +// A "{" with no "}" is an error: the profile is edited by hand, and without the +// check the loop would never advance past it. +func templatePattern(tmpl string) (*regexp.Regexp, error) { + var b strings.Builder + b.WriteString("^") + for { + open := strings.IndexByte(tmpl, '{') + if open < 0 { + b.WriteString(regexp.QuoteMeta(tmpl)) + break + } + closeAt := strings.IndexByte(tmpl[open:], '}') + if closeAt < 0 { + return nil, fmt.Errorf("unclosed { in %q", tmpl[open:]) + } + b.WriteString(regexp.QuoteMeta(tmpl[:open])) + b.WriteString(`[^/:]+`) + tmpl = tmpl[open+closeAt+1:] + } + b.WriteString("$") + return regexp.Compile(b.String()) +} + +func setOf(xs []string) map[string]bool { + m := make(map[string]bool, len(xs)) + for _, x := range xs { + m[x] = true + } + return m +} + +// referenceProxy fronts the in-process server and makes it answer the way the +// reference server answers, at the wire level, for everything this suite can +// see from a client: +// +// - GET /v0/beads/context publishes only the reference ContextResponse +// members (so no wire_revision: the reference server predates it) and +// exactly the reference capability set, whatever this build advertises; +// - a request for an operation the reference document does not publish gets +// the server's own unrouted 404 problem; +// - a query parameter or top-level JSON request member the reference +// operation does not declare gets the server's own 400 unknown_parameter +// problem naming it — which is how the real reference server answers, +// since its decoder is strict. +// +// Responses are otherwise passed through untouched. That is deliberately +// generous in one direction (a response member the reference server would not +// send still reaches the client) and is the residual fidelity gap README.md +// records. +type referenceProxy struct { + profile *referenceProfile + upstream *url.URL + rp *httputil.ReverseProxy + + requests atomic.Int64 + + mu sync.Mutex + refused []string // "METHOD path: why", for the per-shape log +} + +func newReferenceProxy(t *testing.T, profile *referenceProfile, upstreamAddr string) *referenceProxy { + t.Helper() + u, err := url.Parse("http://" + upstreamAddr) + if err != nil { + t.Fatalf("parse upstream address: %v", err) + } + p := &referenceProxy{profile: profile, upstream: u} + p.rp = &httputil.ReverseProxy{ + Rewrite: func(r *httputil.ProxyRequest) { + r.SetURL(u) // also rewrites Host, so the server's Host gate sees its own bind + }, + FlushInterval: -1, // events:watch streams + } + return p +} + +func (p *referenceProxy) takeRefusals() []string { + p.mu.Lock() + defer p.mu.Unlock() + out := p.refused + p.refused = nil + return out +} + +func (p *referenceProxy) noteRefusal(r *http.Request, why string) { + p.mu.Lock() + defer p.mu.Unlock() + p.refused = append(p.refused, fmt.Sprintf("%s %s: %s", r.Method, r.URL.Path, why)) +} + +func (p *referenceProxy) match(r *http.Request) *referenceOperation { + path := r.URL.EscapedPath() + var best *referenceOperation + for i := range p.profile.Operations { + op := &p.profile.Operations[i] + if op.Method != r.Method || !op.pattern.MatchString(path) { + continue + } + // Prefer the most literal template when two match (none do today). + if best == nil || len(op.Path) > len(best.Path) { + best = op + } + } + return best +} + +func (p *referenceProxy) ServeHTTP(w http.ResponseWriter, r *http.Request) { + p.requests.Add(1) + if r.Method == http.MethodGet && r.URL.Path == "/v0/beads/context" { + p.serveContext(w, r) + return + } + op := p.match(r) + if op == nil { + p.noteRefusal(r, "operation not published by the reference server") + detail := "no such route on this server" + httpapi.Write(w, httpapi.Result{Problem: apigen.Problem{ + Status: http.StatusNotFound, + Title: http.StatusText(http.StatusNotFound), + Code: string(httpapi.CodeNotFound), + Detail: &detail, + }}) + return + } + if offender := firstUnknown(slices.Collect(maps.Keys(r.URL.Query())), op.query); offender != "" { + p.noteRefusal(r, "query parameter "+offender+" unknown to "+op.OperationID) + httpapi.Write(w, httpapi.InvalidArgument(offender, httpapi.ReasonUnknownParameter, + "unknown query parameter "+offender)) + return + } + if r.Body != nil && r.Body != http.NoBody { + raw, err := io.ReadAll(r.Body) + _ = r.Body.Close() + if err != nil { + http.Error(w, "reference proxy: read body: "+err.Error(), http.StatusBadGateway) + return + } + r.Body = io.NopCloser(bytes.NewReader(raw)) + r.ContentLength = int64(len(raw)) + var members map[string]json.RawMessage + if len(bytes.TrimSpace(raw)) > 0 && json.Unmarshal(raw, &members) == nil { + if offender := firstUnknown(slices.Collect(maps.Keys(members)), op.body); offender != "" { + p.noteRefusal(r, "request member "+offender+" unknown to "+op.OperationID) + httpapi.Write(w, httpapi.InvalidArgument(offender, httpapi.ReasonUnknownParameter, + "json: unknown field \""+offender+"\"")) + return + } + } + } + p.rp.ServeHTTP(w, r) +} + +// firstUnknown names the smallest key not in known, the same deterministic +// choice the server makes, or "" when every key is known. +func firstUnknown(keys []string, known map[string]bool) string { + sort.Strings(keys) + for _, k := range keys { + if !known[k] { + return k + } + } + return "" +} + +func (p *referenceProxy) serveContext(w http.ResponseWriter, r *http.Request) { + out := r.Clone(r.Context()) + out.URL.Scheme, out.URL.Host, out.Host = p.upstream.Scheme, p.upstream.Host, p.upstream.Host + out.RequestURI = "" + resp, err := http.DefaultTransport.RoundTrip(out) + if err != nil { + http.Error(w, "reference proxy: "+err.Error(), http.StatusBadGateway) + return + } + defer resp.Body.Close() + raw, err := io.ReadAll(resp.Body) + if err != nil { + http.Error(w, "reference proxy: "+err.Error(), http.StatusBadGateway) + return + } + if resp.StatusCode != http.StatusOK { + maps.Copy(w.Header(), resp.Header) + w.WriteHeader(resp.StatusCode) + _, _ = w.Write(raw) + return + } + var body map[string]json.RawMessage + if err := json.Unmarshal(raw, &body); err != nil { + http.Error(w, "reference proxy: decode context: "+err.Error(), http.StatusBadGateway) + return + } + keep := setOf(p.profile.ContextMembers) + for k := range body { + if !keep[k] { + delete(body, k) + } + } + caps, _ := json.Marshal(p.profile.Capabilities) + body["capabilities"] = caps + masked, _ := json.Marshal(body) + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusOK) + _, _ = w.Write(masked) +} diff --git a/backend/http/corpus/required_corpus_tier_test.go b/backend/http/corpus/required_corpus_tier_test.go new file mode 100644 index 000000000..055afeb79 --- /dev/null +++ b/backend/http/corpus/required_corpus_tier_test.go @@ -0,0 +1,23 @@ +package corpus_test + +import ( + "os" + "testing" +) + +// corpusTierLinked reports whether this test binary links the corpus tier. +// harness_test.go, which builds only under cgo like every other file in this +// package, sets it from its init. +var corpusTierLinked bool + +// TestCorpusTierIsLinkedWhenRequired is the one corpus check that builds +// without cgo. skipUnlessEmbeddedDolt turns a missing BEADS_TEST_EMBEDDED_DOLT +// into a failure under BEADS_HTTP_TEST_REQUIRED=1, but it lives in a cgo file: +// with cgo off the go tool drops every other file in this package, so no +// corpus test exists to fail, and a required run would pass having run nothing. +func TestCorpusTierIsLinkedWhenRequired(t *testing.T) { + if os.Getenv("BEADS_HTTP_TEST_REQUIRED") == "1" && !corpusTierLinked { + t.Fatal("BEADS_HTTP_TEST_REQUIRED=1 but this test binary was built without cgo, which drops " + + "the http command corpus (every other file in this package); build it with CGO_ENABLED=1") + } +} diff --git a/backend/http/corpus/testdata/allowlist.yaml b/backend/http/corpus/testdata/allowlist.yaml new file mode 100644 index 000000000..090577cc9 --- /dev/null +++ b/backend/http/corpus/testdata/allowlist.yaml @@ -0,0 +1,29 @@ +# Shapes that are EXPECTED to be refused on a served workspace, permanently. +# The served run must fail and its output must match `refusal`; if it ever +# succeeds, the entry is stale and the suite fails until it is removed. The +# embedded run is still held to the shape's expect_exit, which an allowlisted +# shape must state, and to its expect_stderr: a shape that stops doing what it +# says on the embedded backend fails as a corpus bug, not as allowed. +# A gap some slice will close is NOT listed here: it is a `pending` mark on the +# shape in corpus.yaml instead. + +- id: gc-sql-ready-projection + reason: >- + bd sql needs a Dolt sql-server workspace: the embedded backend refuses it + too (the shape's expect_exit and expect_stderr pin that), and an http + workspace refuses it with the same "embedded mode" text. gc must not issue + it against a remote workspace (DESIGN B3 #10 and B6: gc reads the ready + projection through roles there). + refusal: "'bd sql' is not yet supported" +- id: gc-sql-release-fallback + reason: >- + bd sql needs a Dolt sql-server workspace and the embedded backend refuses + it too; gc's ReleaseIfCurrent fallback must take the conditional update + path on a remote workspace (DESIGN B3 #10). + refusal: "'bd sql' is not yet supported" +- id: mc-meta-set + reason: >- + `bd meta set` is not an OSS bd command (pack scripts written against an + older pinned CLI); it fails on the embedded backend too. The packs should + use `bd update --set-metadata`, which the corpus covers. + refusal: 'unknown command "meta"' diff --git a/backend/http/corpus/testdata/corpus.yaml b/backend/http/corpus/testdata/corpus.yaml new file mode 100644 index 000000000..ded7e9e30 --- /dev/null +++ b/backend/http/corpus/testdata/corpus.yaml @@ -0,0 +1,735 @@ +# Served command corpus (slice S23): every distinct bd command shape the +# orchestrator (gc) and its packs/formulas issue against a work store. +# +# Sources: +# gc: gascity origin/main internal/beads/bdstore*.go, internal/config/workquery.go, +# cmd/gc (BdStore subprocess shapes, work_query/scale_check/on_boot/on_death +# shell, hook claim, agent-script actions, doctor/preflight probes). +# mc: pack, formula, prompt and script invocations counted across the +# maintainer packs (flag sets, not values; values here are representative). +# +# Each shape runs on a served workspace (bd connect to an in-process bd serve, +# non-loopback bind mode, bearer auth, masked to testdata/reference_server.json) +# and on an embedded workspace, with BEADS_ACTOR=worker and CLAUDE_SESSION_ID set +# on both. See README.md for the fields. Reads come first; writes after them +# each own their target ids (e2e-t-*), so write order never changes a read. +# +# pending: {slice, reason} marks a known failure. The shape still runs; when it +# passes the subtest FAILS until the mark is removed. That is how a slice proves +# itself. Permanent refusals live in allowlist.yaml instead. + +files: + - name: seed-core.json + content: | + {"commit_message": "corpus seed", "nodes": [ + {"key": "r1", "id": "e2e-r1", "title": "Routed open task", "type": "task", "priority": 1, "labels": ["lane:a"], "metadata": {"gc.routed_to": "pool-a"}}, + {"key": "r2", "id": "e2e-r2", "title": "Held routed task", "type": "task", "labels": ["hold:mayor"], "metadata": {"gc.routed_to": "pool-a"}}, + {"key": "r3", "id": "e2e-r3", "title": "Routed epic", "type": "epic", "metadata": {"gc.routed_to": "pool-a"}}, + {"key": "wf1", "id": "e2e-wf1", "title": "Workflow root", "type": "task", "metadata": {"gc.run_target": "pool-a", "gc.kind": "workflow"}}, + {"key": "a1", "id": "e2e-a1", "title": "Alice bug", "type": "bug", "assignee": "alice"}, + {"key": "ip1", "id": "e2e-ip1", "title": "Worker in progress", "type": "task", "priority": 0, "assignee": "worker", "metadata": {"gc.routed_to": "pool-a"}}, + {"key": "wr", "id": "e2e-wr", "title": "Worker ready task", "type": "task", "assignee": "worker"}, + {"key": "ipna", "id": "e2e-ipna", "title": "Orphaned in progress", "type": "task", "metadata": {"gc.routed_to": "pool-a"}}, + {"key": "ipnb", "id": "e2e-ipnb", "title": "Orphaned workflow", "type": "task", "metadata": {"gc.run_target": "pool-a", "gc.kind": "workflow"}}, + {"key": "blk", "id": "e2e-blk", "title": "Blocked by r1", "type": "task", "deps": [{"target": "r1"}]}, + {"key": "done", "id": "e2e-done", "title": "Closed task", "type": "task"}, + {"key": "ext", "id": "e2e-ext", "title": "GitHub source", "type": "task", "external_ref": "gh-123", "metadata": {"gc.kind": "github_source", "gc.github.kind": "issue", "gc.github.repo": "o/r"}}, + {"key": "ep", "id": "e2e-ep", "title": "Epic", "type": "epic"}, + {"key": "ep.1", "id": "e2e-ep.1", "title": "Epic child one", "type": "task", "parent_key": "ep"}, + {"key": "ep.2", "id": "e2e-ep.2", "title": "Epic child two", "type": "task", "parent_key": "ep"}, + {"key": "mol", "id": "e2e-mol", "title": "Molecule root", "type": "epic"}, + {"key": "mol.1", "id": "e2e-mol.1", "title": "step one", "type": "task", "assignee": "worker", "parent_key": "mol"}, + {"key": "mol.2", "id": "e2e-mol.2", "title": "step two", "type": "task", "parent_key": "mol", "deps": [{"target": "mol.1"}], "metadata": {"gc.root_bead_id": "e2e-mol", "gc.routed_to": "pool-a"}}, + {"key": "mol.3", "id": "e2e-mol.3", "title": "step three", "type": "task", "parent_key": "mol"}, + {"key": "mac", "id": "e2e-mac", "title": "Auto-close molecule", "type": "epic"}, + {"key": "mac.1", "id": "e2e-mac.1", "title": "done step", "type": "task", "parent_key": "mac"}, + {"key": "mac.2", "id": "e2e-mac.2", "title": "last step", "type": "task", "assignee": "worker", "parent_key": "mac"}, + {"key": "wisp-1", "id": "e2e-wisp-1", "title": "Wisp wisp-1", "type": "task", "ephemeral": true, "metadata": {"gc.routed_to": "pool-a"}, "labels": ["lane:w"]}, + {"key": "wisp-2", "id": "e2e-wisp-2", "title": "Wisp wisp-2", "type": "task", "ephemeral": true, "assignee": "worker", "labels": ["lane:w"]}, + {"key": "wisp-wr", "id": "e2e-wisp-wr", "title": "Wisp worker ready", "type": "task", "ephemeral": true, "priority": 1, "assignee": "worker"}, + {"key": "wisp-c", "id": "e2e-wisp-c", "title": "Wisp wisp-c", "type": "task", "ephemeral": true}, + {"key": "wisp-p", "id": "e2e-wisp-p", "title": "Wisp wisp-p", "type": "task", "ephemeral": true}, + {"key": "wisp-m", "id": "e2e-wisp-m", "title": "Wisp molecule", "type": "epic", "ephemeral": true}, + {"key": "wisp-m.1", "id": "e2e-wisp-m.1", "title": "wisp step", "type": "task", "ephemeral": true, "parent_key": "wisp-m"}, + {"key": "t-upd", "id": "e2e-t-upd", "title": "target upd", "type": "task", "labels": ["lane:z"]}, + {"key": "t-upd2", "id": "e2e-t-upd2", "title": "target upd2", "type": "task"}, + {"key": "t-m1", "id": "e2e-t-m1", "title": "target m1", "type": "task"}, + {"key": "t-m2", "id": "e2e-t-m2", "title": "target m2", "type": "task"}, + {"key": "t-claim", "id": "e2e-t-claim", "title": "target claim", "type": "task"}, + {"key": "t-claim2", "id": "e2e-t-claim2", "title": "target claim2", "type": "task"}, + {"key": "t-rev", "id": "e2e-t-rev", "title": "target rev", "type": "task"}, + {"key": "t-rev2", "id": "e2e-t-rev2", "title": "target rev2", "type": "task"}, + {"key": "t-sling", "id": "e2e-t-sling", "title": "target sling", "type": "task"}, + {"key": "t-as", "id": "e2e-t-as", "title": "target as", "type": "task"}, + {"key": "t-ms", "id": "e2e-t-ms", "title": "target ms", "type": "task"}, + {"key": "t-msc", "id": "e2e-t-msc", "title": "target msc", "type": "task"}, + {"key": "t-um", "id": "e2e-t-um", "title": "target um", "type": "task", "metadata": {"k": "v"}}, + {"key": "t-notes", "id": "e2e-t-notes", "title": "target notes", "type": "task"}, + {"key": "t-er", "id": "e2e-t-er", "title": "target er", "type": "task"}, + {"key": "t-md", "id": "e2e-t-md", "title": "target md", "type": "task"}, + {"key": "t-asg", "id": "e2e-t-asg", "title": "target asg", "type": "task"}, + {"key": "t-dac", "id": "e2e-t-dac", "title": "target dac", "type": "task"}, + {"key": "t-c1", "id": "e2e-t-c1", "title": "target c1", "type": "task"}, + {"key": "t-c2", "id": "e2e-t-c2", "title": "target c2", "type": "task"}, + {"key": "t-c3", "id": "e2e-t-c3", "title": "target c3", "type": "task"}, + {"key": "t-c4", "id": "e2e-t-c4", "title": "target c4", "type": "task"}, + {"key": "t-c5", "id": "e2e-t-c5", "title": "target c5", "type": "task"}, + {"key": "t-d1", "id": "e2e-t-d1", "title": "target d1", "type": "task"}, + {"key": "t-d2", "id": "e2e-t-d2", "title": "target d2", "type": "task"}, + {"key": "t-d3", "id": "e2e-t-d3", "title": "target d3", "type": "task"}, + {"key": "t-d4", "id": "e2e-t-d4", "title": "target d4", "type": "task"}, + {"key": "t-df1", "id": "e2e-t-df1", "title": "target df1", "type": "task"}, + {"key": "t-df2", "id": "e2e-t-df2", "title": "target df2", "type": "task"}, + {"key": "t-da", "id": "e2e-t-da", "title": "target da", "type": "task"}, + {"key": "t-db", "id": "e2e-t-db", "title": "target db", "type": "task"}, + {"key": "t-l", "id": "e2e-t-l", "title": "target l", "type": "task"}, + {"key": "t-as2", "id": "e2e-t-as2", "title": "target as2", "type": "task"}, + {"key": "t-cm", "id": "e2e-t-cm", "title": "target cm", "type": "task"}, + {"key": "t-rel", "id": "e2e-t-rel", "title": "target rel", "type": "task", "assignee": "worker"}, + {"key": "t-xfer", "id": "e2e-t-xfer", "title": "target xfer", "type": "task", "assignee": "worker"}, + {"key": "t-ifa", "id": "e2e-t-ifa", "title": "target ifa", "type": "task", "assignee": "worker"}, + {"key": "t-death", "id": "e2e-t-death", "title": "target death", "type": "task", "assignee": "worker"}, + {"key": "t-rc", "id": "e2e-t-rc", "title": "target rc", "type": "task", "assignee": "worker"}, + {"key": "t-boot", "id": "e2e-t-boot", "title": "target boot", "type": "task"}, + {"key": "t-ro", "id": "e2e-t-ro", "title": "target reopen", "type": "task"}, + {"key": "t-dr", "id": "e2e-t-dr", "title": "target dep remove", "type": "task", "deps": [{"target": "t-db"}]}, + {"key": "t-dc", "id": "e2e-t-dc", "title": "cascade root", "type": "epic"}, + {"key": "t-dc.1", "id": "e2e-t-dc.1", "title": "cascade child", "type": "task", "parent_key": "t-dc"} + ]} + - name: bulk-ready-a.json + bulk: {id_prefix: e2e-bra, count: 60, priority: 3, metadata: {gc.routed_to: pool-a}} + - name: bulk-ready-b.json + bulk: {id_prefix: e2e-brb, count: 60, priority: 3, metadata: {gc.routed_to: pool-a}} + - name: bulk-wisp-a.json + bulk: {id_prefix: e2e-wisp-bwa, count: 55, priority: 3, ephemeral: true, metadata: {gc.routed_to: pool-a}} + - name: bulk-wisp-b.json + bulk: {id_prefix: e2e-wisp-bwb, count: 55, priority: 3, ephemeral: true, metadata: {gc.routed_to: pool-a}} + - name: graph-small.json + content: | + {"commit_message": "gc graph apply", "nodes": [ + {"key": "root", "id": "e2e-g-root", "title": "graph root", "type": "epic", "metadata": {"gc.kind": "workflow"}}, + {"key": "s1", "id": "e2e-g-s1", "title": "graph step 1", "type": "task", "parent_key": "root", "assignee": "worker", "labels": ["lane:g"], "metadata": {"gc.routed_to": "pool-a"}}, + {"key": "s2", "id": "e2e-g-s2", "title": "graph step 2", "type": "task", "parent_key": "root"} + ], "edges": [{"from_key": "s2", "to_key": "s1", "type": "blocks"}]} + - name: graph-eph.json + content: | + {"nodes": [ + {"key": "root", "id": "e2e-wisp-g-root", "title": "wisp graph root", "type": "epic"}, + {"key": "s1", "id": "e2e-wisp-g-s1", "title": "wisp graph step", "type": "task", "parent_key": "root"} + ]} + - name: delete-list.txt + content: | + e2e-t-df1 + e2e-t-df2 + +seed: + - [create, --graph, "{file:seed-core.json}", --json] + - [create, --graph, "{file:bulk-ready-a.json}", --json] + - [create, --graph, "{file:bulk-ready-b.json}", --json] + - [create, --graph, "{file:bulk-wisp-a.json}", --json] + - [create, --graph, "{file:bulk-wisp-b.json}", --json] + # Graph plans carry no status: a node created in_progress or closed stamps + # started_at/closed_at, which batchApply's create vocabulary cannot carry + # (gc's own plans never set a status). Lifecycle goes through update/close. + - [update, e2e-ip1, e2e-ipna, e2e-ipnb, e2e-mol.1, e2e-mac.2, e2e-t-rel, e2e-t-xfer, e2e-t-ifa, e2e-t-death, e2e-t-rc, e2e-t-boot, --status, in_progress] + - [update, e2e-wisp-2, --status, in_progress] + - [close, --force, e2e-done, e2e-ep.2, e2e-mol.3, e2e-mac.1, e2e-t-ro] + - [comments, add, e2e-r1, "seed comment"] + +shapes: + # ---------------------------------------------------------------- show + - id: gc-show-json + source: gc bdstore.go BdStore.Get (show --json ); counts must not read 0 + args: [show, --json, e2e-blk] + - id: gc-show-json-batch + source: gc bdstore_exact_batch.go GetExactBatch + args: [show, --json, e2e-r1, e2e-a1, e2e-blk, e2e-mol] + - id: gc-show-json-wisp + source: gc workquery.go blocked_by enrichment; wisps live in the ready tier + args: [show, e2e-wisp-1, --json] + - id: gc-show-id-flag + source: gc cmd_bd_show_watch.go bdWatchRunner (show --json --id=) + args: [show, --json, --id=e2e-r1] + # The worker holds eight in-progress issues; e2e-ip1 is the only P0 among + # them, so "first in the store's default order" (priority first) names it on + # both sides. Among equal priorities the order rests on created_at, and a + # seed that straddles a second boundary on one side only picked another issue. + - id: gc-show-current + source: gc cmd_bd_show_watch.go bdWatchRunner (show --current --json) + args: [show, --current, --json] + pending: {slice: "S6e", reason: "NEW: resolveCurrentIssueID calls raw store.SearchIssues, the http refusal is swallowed, and it silently falls back to the last-touched issue (wrong issue, exit 0)"} + - id: gc-show-missing + source: gc BdStore.Get not-found classification (routing fallbacks) + args: [show, --json, e2e-nonexistent] + expect_exit: 1 + expect_stderr: "(?i)not found" + compare: exit + - id: mc-show-text + source: mc prompts (bd show , ~80) + args: [show, e2e-r1] + compare: text + + # ---------------------------------------------------------------- ready + # e2e-wr and e2e-wisp-wr are the only open rows assigned to the worker when + # these shapes run, so the assignee shapes compare real rows: a client that + # sends the wrong assignee or drops --include-ephemeral returns a different + # list. The wisp is P1 and the task P2, so --limit=1 picks the wisp on both + # sides by priority alone. + - id: gc-ready-unlimited + source: gc bdstore.go bdReadyArgs (BdStore.Ready, ready --json --limit 0) + args: [ready, --json, --limit, "0"] + compare: ids + pending: {slice: "S15", reason: "ready --limit 0 omits limit and the server default page silently caps the answer (B3 #2)"} + - id: gc-ready-ephemeral-assignee-unlimited + source: gc bdstore.go bdReadyArgs with --include-ephemeral --assignee + args: [ready, --json, --include-ephemeral, --assignee, worker, --limit, "0"] + compare: ids + - id: gc-ready-pool-demand + source: gc workquery.go bdReadyPoolDemandShell (scale_check, --limit=20) + args: [ready, --metadata-field, gc.routed_to=pool-a, --unassigned, --exclude-type=epic, --exclude-label, "hold:mayor", --exclude-label, "hold:external", --json, --limit=20] + compare: ids + - id: gc-ready-pool-demand-unlimited + source: gc workquery.go bdReadyPoolDemandShell (work_query, --include-ephemeral --limit 0) + args: [ready, --include-ephemeral, --metadata-field, gc.routed_to=pool-a, --unassigned, --exclude-type=epic, --exclude-label, "hold:mayor", --exclude-label, "hold:external", --json, --limit, "0"] + compare: ids + pending: {slice: "S15", reason: "ready --limit 0 silently capped by the server's default page (B3 #2)"} + - id: gc-ready-workflow-migration + source: gc workquery.go bdReadyPoolDemandMigrationShell (--sort oldest --limit 0) + args: [ready, --include-ephemeral, --metadata-field, gc.run_target=pool-a, --metadata-field, gc.kind=workflow, --unassigned, --exclude-type=epic, --exclude-label, "hold:mayor", --exclude-label, "hold:external", --json, --sort, oldest, --limit, "0"] + compare: ids + - id: gc-ready-graph-anchor + source: gc workquery.go assignedGraphWorkflowAnchorReadyFunctionScript + args: [ready, --include-ephemeral, --metadata-field, gc.root_bead_id=e2e-mol, --metadata-field, gc.routed_to=pool-a, --exclude-type=epic, --exclude-label, "hold:mayor", --exclude-label, "hold:external", --json, --sort, oldest, --limit=20] + compare: ids + - id: gc-ready-assigned-tier + source: gc workquery.go assignedReadyTierCommand (--assignee= --limit=1) + args: [ready, --include-ephemeral, --assignee=worker, --json, --limit=1] + compare: ids + - id: gc-ready-probe + source: gc examples/bd gc-beads-bd.sh (bare bd ready probe) + args: [ready] + compare: exit + - id: mc-ready-json + source: mc scripts/prompts (bd ready --json, ~36) + args: [ready, --json] + compare: ids + - id: mc-ready-mol + source: mc skills/pr-review (bd ready --mol ) + args: [ready, --mol, e2e-mol, --json] + pending: {slice: "S16", reason: "ready --mol loads molecule steps off a raw path: step rows lose assignee and timestamps (molecule reads over roles)"} + - id: mc-ready-parent + source: mc skills/pr-review (bd ready --parent ) + args: [ready, --parent, e2e-ep, --json] + compare: ids + + # ---------------------------------------------------------------- list + - id: gc-list-full-filters + source: gc bdstore.go listViaBDList (List/ListByLabel/ListByMetadata) + args: [list, --json, --label=lane:a, --status=open, --type=task, --include-infra, --include-gates, --limit, "0", --metadata-field, gc.routed_to=pool-a] + - id: gc-list-ping + source: gc bdstore.go BdStore.Ping (list --json --limit 0) + args: [list, --json, --limit, "0"] + compare: ids + - id: gc-list-limit1 + source: gc doctor_gate_sandbox.go / doctor_store_preflight.go + args: [list, --json, --limit, "1"] + compare: count + - id: gc-list-readonly + source: gc doctor_v2_session_migration.go readOnlyBdRunner + args: [--readonly, list, --json, --limit, "1"] + compare: count + - id: gc-list-assignee-status + source: gc bdstore.go listViaBDList (ListByAssignee) + args: [list, --json, --assignee=worker, --status=in_progress, --include-infra, --include-gates, --limit, "0"] + - id: gc-list-all-created-before + source: gc bdstore.go listViaBDList (--all --created-before --include-templates --skip-labels) + args: [list, --json, --all, --created-before, "2100-01-01T00:00:00Z", --include-infra, --include-gates, --include-templates, --limit, "0", --skip-labels] + compare: ids + - id: gc-list-children + source: gc bdstore.go listViaBDList (Children, --parent) + args: [list, --json, --include-infra, --include-gates, --limit, "0", --parent, e2e-ep, --all] + - id: gc-list-in-progress-tier + source: gc workquery.go assignedInProgressTierCommandWithLimit + args: [list, --status, in_progress, --assignee=worker, --json, --limit=20] + - id: gc-list-on-death + source: gc workquery.go EffectiveOnDeath + args: [list, --assignee=worker, --status=in_progress, --json] + - id: gc-list-on-boot + source: gc workquery.go on_boot recovery (--no-assignee) + args: [list, --metadata-field, gc.routed_to=pool-a, --status=in_progress, --no-assignee, --json] + pending: {slice: "S6c+S15b", reason: "list stub (B3 #1); --no-assignee refused by the list encoder (B3 #11)"} + - id: gc-list-on-boot-workflow + source: gc workquery.go on_boot recovery, workflow lane + args: [list, --metadata-field, gc.run_target=pool-a, --metadata-field, gc.kind=workflow, --status=in_progress, --no-assignee, --json] + pending: {slice: "S6c+S15b", reason: "list stub (B3 #1); --no-assignee refused by the list encoder (B3 #11)"} + - id: mc-list-meta-status + source: mc assets snapshot.md (list --json --limit --metadata-field --status) + args: [list, --json, --limit, "50", --metadata-field, gc.routed_to=pool-a, --status, open] + compare: ids + # More than 50 rows match, most at one priority: --limit 50 cuts a tie set + # whose created_at order differs between the two seeds, so each side's 50 + # rows are held to its own unlimited listing instead. + prefix_of: [list, --json, --limit, "0", --metadata-field, gc.routed_to=pool-a, --status, open] + - id: mc-list-github-source + source: mc github-intake (list --all --json --limit --metadata-field x3) + args: [list, --metadata-field, gc.kind=github_source, --metadata-field, gc.github.kind=issue, --metadata-field, gc.github.repo=o/r, --all, --json, --limit, "0"] + - id: mc-list-all-type + source: mc normalized_candidate_graph.py (list --all --type) + args: [list, --all, --type, epic, --json, --limit, "50"] + - id: mc-list-include-gates-infra-label + source: mc mol-normalized-issue-pr-v1 (list --all --include-gates --include-infra --json --label --limit) + args: [list, --all, --include-gates, --include-infra, --json, --label, "lane:a", --limit, "0"] + - id: mc-list-title + source: mc mol-normalized-issue-pr-v1 (list --all --include-infra --json --limit --title) + args: [list, --all, --include-infra, --json, --limit, "10", --title, "Routed open task"] + pending: {slice: "S6c+S15b", reason: "list stub (B3 #1); --title is a text filter the list encoder refuses (B3 #11)"} + - id: mc-list-id + source: mc skills/pr-review (list --all --id --json --limit) + args: [list, --all, --id, "e2e-r1,e2e-a1", --json, --limit, "0"] + pending: {slice: "S6c+S15+S15b", reason: "list stub (B3 #1); --limit 0 (B3 #3); --id refused by the list encoder (B3 #11)"} + - id: mc-list-assignee-status + source: mc review summaries (list --assignee --json --limit --status) + args: [list, --assignee, alice, --json, --limit, "10", --status, open] + - id: mc-list-label-text + source: mc normalized_child_launch.py (list --label) + args: [list, --label, "lane:a"] + compare: text + - id: mc-list-plain + source: mc scripts (bare bd list) + args: [list] + compare: exit + - id: mc-children + source: mc prompts (bd children ) + args: [children, e2e-ep, --json] + + # ---------------------------------------------------------------- query / count / search + - id: gc-query-wisps + source: gc bdstore.go listEphemeral (query --json "ephemeral=true AND ..." --limit N) + args: [query, --json, "ephemeral=true AND label=lane:w", --limit, "50"] + - id: gc-query-wisps-all + source: gc bdstore.go listEphemeral with IncludeClosed (--all) + args: [query, --json, "ephemeral=true AND status=in_progress", --all, --limit, "50"] + - id: gc-query-wisp-by-id + source: gc bdstore.go getEphemeralByID + args: [query, --json, "ephemeral=true AND id=e2e-wisp-1", --all, --limit, "1"] + - id: gc-query-wisps-unlimited + source: gc workquery.go bdQueryEphemeralStatusShell (--limit=0, wisp tier) + args: [query, --json, "ephemeral=true AND status=open", --limit=0] + compare: ids + pending: {slice: "S15", reason: "query --limit=0 sends limit=0 and a non-loopback server refuses unlimited reads (B3 #3)"} + - id: mc-query + source: mc pr_merge.py (bd query) + args: [query, --json, "status=open AND priority<=1", --limit, "50"] + - id: gc-count + source: gc native CountIssues fallback (count without scope) + args: [count, --json] + - id: gc-count-status + source: gc count with a status filter + args: [count, --json, --status, open] + - id: mc-search + source: mc mol-bug-report-flow-v2 (bd search) + args: [search, "Routed", --json] + compare: ids + pending: {slice: "S24", reason: "NEW: bd search is refused over http ('partial-id search is not available over HTTP'); not in B3"} + - id: mc-status + source: mc normalized_child_launch.py (bd status) + args: [status, --json] + + # ---------------------------------------------------------------- deps / graph reads + - id: gc-dep-list + source: gc bdstore.go DepList + args: [dep, list, e2e-blk, --json] + - id: gc-dep-list-up + source: gc bdstore.go DepList (--direction=up) + args: [dep, list, e2e-r1, --json, --direction=up] + - id: gc-dep-list-batch + source: gc bdstore.go DepListBatch + args: [dep, list, e2e-blk, e2e-mol.2, --json] + - id: mc-dep-tree + source: mc pr_review.py city_dep_subtree (dep tree --json) + args: [dep, tree, e2e-mol, --json] + - id: gc-blocked + source: gc bdstore_ready_projection.go fetchReadyProjectionViaBlocked + args: [blocked, --json] + pending: {slice: "S18", reason: "bd blocked hits the GetBlockedIssues stub (B3 #10)"} + + # ---------------------------------------------------------------- molecules + - id: mc-mol-current + source: mc worker prompts (bd mol current, ~38; inferred from in_progress work) + args: [mol, current, --json] + pending: {slice: "S16", reason: "mol current without an id is silently empty over http (B3 #4)"} + - id: mc-mol-current-text + source: mc gc-role-worker template (bd mol current, text) + args: [mol, current] + compare: text + pending: {slice: "S16", reason: "mol current without an id is silently empty over http (B3 #4)"} + - id: mc-mol-current-id + source: mc prompts (bd mol current ) + args: [mol, current, e2e-mol, --json] + pending: {slice: "S16", reason: "mol current refuses (B3 #4)"} + - id: mc-mol-progress + source: mc prompts (bd mol progress ) + args: [mol, progress, e2e-mol, --json] + pending: {slice: "S16", reason: "mol progress refuses (B3 #4)"} + + # ---------------------------------------------------------------- config / probes / local + - id: gc-config-get-types + source: gc beads_provider_lifecycle.go / checks_custom_types.go (config get --json types.custom) + args: [config, get, --json, types.custom] + - id: gc-config-get-prefix + source: gc gc-beads-bd.sh (config get issue_prefix) + args: [config, get, issue_prefix] + compare: text + - id: gc-types + source: gc checks_custom_types.go (types --json) + args: [types, --json] + - id: gc-context + source: gc beads_preflight_checker.go (context --json); never touches the store + args: [context, --json] + compare: exit + - id: gc-version + source: gc bdstore_blocked_repair.go BDVersion + args: [version] + compare: exit + - id: gc-ping + source: gc cmd_beads_city_migrate_proxied.go / dashboard (ping --json) + args: [ping, --json] + compare: exit + - id: mc-prime + source: mc formulas (bd prime) + args: [prime] + compare: exit + - id: gc-sql-ready-projection + source: gc bdstore_ready_projection.go fetchReadyProjection + args: [sql, "select id,is_blocked from issues where status <> 'closed' union all select id,is_blocked from wisps where status <> 'closed'", --json] + expect_exit: 1 + expect_stderr: "'bd sql' is not yet supported" + compare: exit + - id: gc-sql-release-fallback + source: gc bdstore.go ReleaseIfCurrent fallback (sql UPDATE) + args: [sql, --json, "UPDATE issues SET status='open', assignee='' WHERE id='e2e-t-rel' AND status='in_progress' AND assignee='worker'"] + expect_exit: 1 + expect_stderr: "'bd sql' is not yet supported" + compare: exit + - id: gc-purge-dry-run + source: gc bdstore.go Purge (purge --json --dry-run) + args: [purge, --json, --dry-run] + compare: exit + pending: {slice: "SRV-1", reason: "purge needs the sweep tokens the reference server does not advertise (B3 #13); remove the mark when reference_server.json is upgraded"} + - id: gc-recompute-blocked + source: gc bdstore_blocked_repair.go RecomputeBlocked + args: [recompute-blocked, --json] + compare: exit + pending: {slice: "S25", reason: "NEW: recompute-blocked refuses over http ('storage backend does not support is_blocked recompute'); gc BdStore.RecomputeBlocked issues it; not in B3"} + - id: gc-backup-status + source: gc checks_proxied_backup_coverage.go (backup status --json) + args: [backup, status, --json] + compare: exit + - id: mc-gate-list + source: core pack orders (gate list) + args: [gate, list, --json] + pending: {slice: "S20", reason: "gate list issues a raw SearchIssues shape the bridge refuses (B3 #15)"} + + - id: gc-export-all + source: gc core pack reaper (gc bd export --all -o F) + args: [export, --all, -o, "{file:export.jsonl}"] + compare: exit + pending: {slice: "S21", reason: "export --all is not wired over http (B3 #16)"} + + # ================================================================ writes + # ---------------------------------------------------------------- create + - id: gc-create-full + source: gc bdstore.go CreateWithStorage (every optional flag but --parent) + args: [create, --json, "gc created", -t, task, --id, e2e-n1, --priority, "1", --description, "made by gc", --assignee, worker, --deps, e2e-r1, --labels, "lane:n,x", --metadata, '{"gc.routed_to":"pool-a"}'] + compare: exit + verify: [[show, --json, e2e-n1]] + - id: gc-create-parent + source: gc bdstore.go CreateWithStorage (--parent, server-minted child id) + args: [create, --json, "gc child", -t, task, --parent, e2e-ep] + compare: exit + verify: [[show, --json, e2e-ep.3]] + pending: {slice: "S19", reason: "create --parent calls GetNextChildID (B3 #8)"} + - id: gc-create-ephemeral + source: gc bdstore.go CreateWithStorage (--ephemeral) + args: [create, --json, "gc wisp", -t, task, --id, e2e-wisp-n2, --ephemeral, --metadata, '{"gc.routed_to":"pool-a"}'] + compare: exit + verify: [[show, --json, e2e-wisp-n2]] + - id: gc-create-no-history + source: gc bdstore.go CreateWithStorage (--no-history, agent beads) + args: [create, --json, "gc agent bead", -t, task, --id, e2e-n3, --no-history] + compare: exit + verify: [[show, --json, e2e-n3]] + - id: gc-create-defer + source: gc bdstore.go CreateWithStorage (--defer RFC3339) + args: [create, --json, "gc deferred", -t, task, --id, e2e-n4, --defer, "2100-01-01T00:00:00Z"] + compare: exit + verify: [[show, --json, e2e-n4]] + - id: mc-create-external-ref + source: mc snapshot.md (create --external-ref --labels --metadata --type) + args: [create, "triaged issue", --external-ref, gh-999, --labels, "triage", --metadata, '{"gc.kind":"github_source"}', --type, task, --id, e2e-n5, --json] + compare: exit + verify: [[show, --json, e2e-n5]] + - id: mc-create-text + source: mc prompts (bd create, text output) + args: [create, "plain create", --type, task, --id, e2e-n6] + compare: exit + verify: [[show, --json, e2e-n6]] + - id: gc-create-graph + source: gc bdstore_graph_apply.go ApplyGraphPlanWithStorage + args: [create, --graph, "{file:graph-small.json}", --json] + verify: [[show, --json, e2e-g-root, e2e-g-s1, e2e-g-s2]] + - id: gc-create-graph-ephemeral + source: gc bdstore_graph_apply.go (--ephemeral by storage class) + args: [create, --graph, "{file:graph-eph.json}", --json, --ephemeral] + verify: [[show, --json, e2e-wisp-g-root, e2e-wisp-g-s1]] + + # ---------------------------------------------------------------- update + - id: gc-update-fields + source: gc bdstore.go bdUpdateArgs (Update) + args: [update, --json, e2e-t-upd, --title, "updated", --status, in_progress, --type, bug, --priority, "1", --description, "new", --assignee, bob, --set-metadata, a=1, --set-metadata, b=2, --add-label, x, --remove-label, "lane:z"] + compare: exit + verify: [[show, --json, e2e-t-upd]] + - id: gc-update-parent + source: gc bdstore.go bdUpdateArgs (--parent) + args: [update, --json, e2e-t-upd2, --parent, e2e-ep] + compare: exit + verify: [[show, --json, e2e-t-upd2]] + - id: gc-update-many + source: gc bdstore.go UpdateAll / setMetadataBatchAll + args: [update, --json, e2e-t-m1, e2e-t-m2, --set-metadata, gc.batch=1, --priority, "3"] + compare: exit + verify: [[show, --json, e2e-t-m1, e2e-t-m2]] + - id: gc-update-claim + source: gc bdstore.go BdStore.Claim and gc hook --claim (update --claim --json) + args: [update, e2e-t-claim, --claim, --json] + compare: exit + verify: [[show, --json, e2e-t-claim]] + - id: gc-update-claim-actor + source: gc cmd_agent_script.go bd_claim (update --claim --actor X) + args: [update, e2e-t-claim2, --claim, --actor, alice] + compare: exit + verify: [[show, --json, e2e-t-claim2]] + - id: gc-update-claim-wisp + source: gc ready queues include ephemeral work; claiming a wisp + args: [update, e2e-wisp-c, --claim, --json] + compare: exit + verify: [[show, --json, e2e-wisp-c]] + pending: {slice: "SRV-1", reason: "the reference server lacks UpdateIssueRequest.claim, and the claimIssue fallback excludes wisps (B3 #6); remove the mark when reference_server.json is upgraded"} + - id: gc-update-if-revision + source: gc bdstore_conditional.go UpdateIfMatch / CompareAndSetMetadataKey + args: [update, e2e-t-rev, --set-metadata, k=v, --if-revision, "{rev:e2e-t-rev}"] + compare: exit + verify: [[show, --json, e2e-t-rev]] + - id: gc-update-if-revision-stale + source: gc bdstore_conditional.go UpdateIfMatch mismatch (exit 13) + args: [update, e2e-t-rev2, --set-metadata, k=v, --if-revision, "1"] + expect_exit: 13 + compare: exit + verify: [[show, --json, e2e-t-rev2]] + - id: gc-update-release-if-current + source: gc bdstore_conditional_release.go releaseIfCurrentViaBdVerb + args: [update, e2e-t-rel, --if-assignee, worker, --if-status, in_progress, --status, open, --assignee, ""] + compare: exit + verify: [[show, --json, e2e-t-rel]] + - id: gc-update-transfer-if-current + source: gc bdstore_conditional_release.go TransferIfCurrent + args: [update, e2e-t-xfer, --if-assignee, worker, --if-status, in_progress, --assignee, alice] + compare: exit + verify: [[show, --json, e2e-t-xfer]] + - id: gc-update-if-assignment + source: gc bdstore_conditional.go UpdateIfAssignment + args: [update, --json, e2e-t-ifa, --set-metadata, k=v, --if-status, in_progress, --if-assignee, worker] + compare: exit + verify: [[show, --json, e2e-t-ifa]] + - id: gc-update-if-assignment-assignee-stale + source: gc bdstore_conditional.go UpdateIfAssignment mismatch (exit 13) + args: [update, --json, e2e-t-ifa, --set-metadata, probe=a, --if-status, in_progress, --if-assignee, nobody] + expect_exit: 13 + compare: exit + verify: [[show, --json, e2e-t-ifa]] + - id: gc-update-if-assignment-status-stale + source: gc bdstore_conditional.go UpdateIfAssignment mismatch (exit 13) + args: [update, --json, e2e-t-ifa, --set-metadata, probe=s, --if-status, open, --if-assignee, worker] + expect_exit: 13 + compare: exit + verify: [[show, --json, e2e-t-ifa]] + - id: gc-update-on-death + source: gc workquery.go on_death (update --assignee "" --status open --set-metadata) + args: [update, e2e-t-death, --assignee, "", --status, open, --set-metadata, gc.run_target=pool-a] + compare: exit + verify: [[show, --json, e2e-t-death]] + - id: gc-update-on-boot + source: gc workquery.go on_boot (update --status open) + args: [update, e2e-t-boot, --status, open] + compare: exit + verify: [[show, --json, e2e-t-boot]] + - id: gc-update-sling + source: gc workquery.go DefaultSlingQuery (update --set-metadata gc.routed_to=) + args: [update, e2e-t-sling, --set-metadata, gc.routed_to=pool-b] + compare: exit + verify: [[show, --json, e2e-t-sling]] + - id: gc-update-agent-script + source: gc cmd_agent_script.go bd_update (--assignee --status --append-notes --set-metadata) + args: [update, e2e-t-as, --assignee, worker, --status, in_progress, --append-notes, "agent note", --set-metadata, k=v] + compare: exit + verify: [[show, --json, e2e-t-as]] + - id: gc-update-dolt-auto-commit-off + source: gc bdstore.go bdTransientWriteArgs (--dolt-auto-commit off prefix) + args: [--dolt-auto-commit, "off", update, --json, e2e-t-dac, --set-metadata, k=v] + compare: exit + verify: [[show, --json, e2e-t-dac]] + - id: mc-update-meta-status + source: mc prompts (update --set-metadata --status, ~151) + args: [update, e2e-t-ms, --set-metadata, review.verdict=iterate, --status, in_progress] + compare: exit + verify: [[show, --json, e2e-t-ms]] + - id: mc-update-meta-status-closed + source: mc prompts (update --set-metadata --status closed, under Claude Code) + args: [update, e2e-t-msc, --set-metadata, review.verdict=done, --status, closed] + compare: exit + verify: [[show, --json, e2e-t-msc]] + pending: {slice: "S6d", reason: "a silent data difference, not a refusal: the update succeeds on both sides under CLAUDE_SESSION_ID, but the served update records no closed_by_session (an http update patch cannot carry one, so the CLI drops it without a word; bd close does carry it) where embedded records the session, so the verify show differs (B3 #7)"} + - id: mc-update-unset-metadata + source: mc adopt-pr-operator (update --unset-metadata) + args: [update, e2e-t-um, --unset-metadata, k] + compare: exit + verify: [[show, --json, e2e-t-um]] + - id: mc-update-notes + source: mc github fix-issue formula (update --notes) + args: [update, e2e-t-notes, --notes, "notes text"] + compare: exit + verify: [[show, --json, e2e-t-notes]] + - id: mc-update-external-ref-metadata + source: mc snapshot.md (update --external-ref --metadata) + args: [update, e2e-t-er, --external-ref, gh-1, --metadata, '{"gc.kind":"github_source"}'] + compare: exit + verify: [[show, --json, e2e-t-er]] + - id: mc-update-metadata + source: mc build-base prepare.md (update --metadata) + args: [update, e2e-t-md, --metadata, '{"x":"1"}'] + compare: exit + verify: [[show, --json, e2e-t-md]] + - id: mc-update-assignee + source: mc review reports (update --assignee) + args: [update, e2e-t-asg, --assignee, bob] + compare: exit + verify: [[show, --json, e2e-t-asg]] + + # ---------------------------------------------------------------- close / reopen + - id: gc-close-force-reason-many + source: gc bdstore.go bdCloseArgs (CloseAll/CloseAllWithReason) + args: [close, --force, --json, --reason, done, e2e-t-c1, e2e-t-c2] + compare: exit + verify: [[show, --json, e2e-t-c1, e2e-t-c2]] + - id: gc-close-if-revision + source: gc bdstore_conditional.go CloseIfMatch + args: [close, --force, --json, e2e-t-c3, --if-revision, "{rev:e2e-t-c3}"] + compare: exit + verify: [[show, --json, e2e-t-c3]] + - id: mc-close-text + source: mc formulas (bd close ) + args: [close, e2e-t-c4] + compare: exit + verify: [[show, --json, e2e-t-c4]] + - id: mc-close-reason + source: mc build-basic-review (bd close --reason) + args: [close, e2e-t-c5, --reason, "review applied"] + compare: exit + verify: [[show, --json, e2e-t-c5]] + - id: gc-close-last-step-autocloses-molecule + source: gc/mc molecule lifecycle (closing the last open step closes the root) + args: [close, --force, --json, e2e-mac.2] + compare: exit + verify: [[show, --json, e2e-mac]] + - id: gc-reopen + source: gc bdstore.go Reopen + args: [reopen, --json, e2e-t-ro] + compare: exit + verify: [[show, --json, e2e-t-ro]] + + # ---------------------------------------------------------------- delete / reclaim / promote / burn + - id: gc-delete + source: gc bdstore.go Delete + args: [delete, --force, --json, e2e-t-d1] + compare: exit + verify: [[show, --json, e2e-t-d1]] + - id: gc-delete-if-revision + source: gc bdstore_conditional.go DeleteIfMatch + args: [delete, --force, --json, e2e-t-d2, --if-revision, "{rev:e2e-t-d2}"] + compare: exit + verify: [[show, --json, e2e-t-d2]] + - id: gc-delete-batch + source: gc bdstore.go DeleteBatch (no --json) + args: [delete, e2e-t-d3, e2e-t-d4, --force] + compare: exit + verify: [[show, --json, e2e-t-d3]] + - id: gc-delete-cascade + source: gc cmd_convoy_dispatch.go deleteWorkflowMatches + args: [delete, e2e-t-dc, --cascade, --force] + compare: exit + verify: [[show, --json, e2e-t-dc.1]] + - id: mc-delete-from-file + source: mc session_cleanup.py / core reaper (delete --from-file) + args: [delete, --from-file, "{file:delete-list.txt}", --force] + compare: exit + verify: [[show, --json, e2e-t-df1]] + - id: gc-reclaim + source: gc bdstore.go ReclaimStale (reclaim --id --json) + args: [reclaim, --id, e2e-t-rc, --json] + compare: exit + verify: [[show, --json, e2e-t-rc]] + pending: {slice: "SRV-1", reason: "the reference server does not advertise issues.reclaim (B3 #12), so the client refuses at its capability preflight"} + - id: mc-promote + source: mc mol-normalized-issue-pr-v1 (promote --reason) + args: [promote, e2e-wisp-p, --reason, "keep it"] + compare: exit + pending: {slice: "S21", reason: "promote is not wired over http (B3 #16)"} + - id: mc-mol-burn + source: mc review fixtures (mol burn --force) + args: [mol, burn, e2e-wisp-m, --force] + compare: exit + verify: [[show, --json, e2e-wisp-m.1]] + pending: {slice: "S17", reason: "mol burn needs RunInTransaction (B3 #14)"} + + # ---------------------------------------------------------------- deps / comments / labels / config writes + - id: gc-dep-add + source: gc bdstore.go DepAdd + args: [dep, add, e2e-t-da, e2e-t-db, --type, blocks] + compare: exit + verify: [[dep, list, e2e-t-da, --json]] + - id: gc-dep-remove + source: gc bdstore.go DepRemove + args: [dep, remove, e2e-t-dr, e2e-t-db] + compare: exit + verify: [[dep, list, e2e-t-dr, --json]] + - id: mc-comment + source: mc bug_hunt.py (bd comment ) + args: [comment, e2e-t-cm, "a comment"] + compare: exit + verify: [[comments, e2e-t-cm, --json]] + ignore_fields: [id] # comment ids are minted per workspace + - id: mc-comments-list + source: mc prompts (bd comments ) + args: [comments, e2e-r1, --json] + ignore_fields: [id] # comment ids are minted per workspace + - id: mc-label-add + source: mc prompts (bd label add) + args: [label, add, e2e-t-l, "lane:q"] + compare: exit + verify: [[show, --json, e2e-t-l]] + - id: mc-assign + source: mc prompts (bd assign) + args: [assign, e2e-t-as2, alice] + compare: exit + verify: [[show, --json, e2e-t-as2]] + - id: gc-config-set-types + source: gc beads_provider_lifecycle.go (config set types.custom a,b) + args: [config, set, types.custom, "corpus-a,corpus-b"] + compare: exit + verify: [[config, get, --json, types.custom]] + - id: mc-meta-set + source: mc adopt-pr-review checks (bd meta set k=v) + args: [meta, set, e2e-r1, review.verdict=done] + expect_exit: 1 + expect_stderr: 'unknown command "meta"' + compare: exit diff --git a/backend/http/corpus/testdata/reference_server.json b/backend/http/corpus/testdata/reference_server.json new file mode 100644 index 000000000..7bdf6d0d0 --- /dev/null +++ b/backend/http/corpus/testdata/reference_server.json @@ -0,0 +1,595 @@ +{ + "comment": [ + "The reference server profile the corpus suite masks the in-process server to.", + "It is the wire surface of a deployed bd 1.3.0 `bd serve` that predates wire_revision:", + "capabilities is that server's advertised ContextResponse.capabilities (read off a live GET /v0/beads/context),", + "context_members is its ContextResponse property set, and operations are the method, path template,", + "query parameter names and top-level JSON request members of its published OpenAPI document (MIT).", + "Regenerate by hand when the reference deployment is upgraded; see README.md." + ], + "capabilities": [ + "config.get", + "config.list", + "config.set", + "config.unset", + "dependencies.add", + "dependencies.blocking", + "dependencies.count", + "dependencies.cycles", + "dependencies.list", + "dependencies.remove", + "dependencies.tree", + "events.list", + "events.watch", + "issues.addComment", + "issues.batchApply", + "issues.batchClose", + "issues.batchCreate", + "issues.casMetadata", + "issues.claim", + "issues.claimNext", + "issues.close", + "issues.count", + "issues.create", + "issues.delete", + "issues.get", + "issues.list", + "issues.list.sort", + "issues.query", + "issues.related", + "issues.release", + "issues.reopen", + "issues.sweep", + "issues.update", + "memories.forget", + "memories.get", + "memories.list", + "memories.remember", + "policy.external_dependencies", + "project.enforce", + "ready.count", + "ready.list", + "stats.get" + ], + "context_members": [ + "api_version", + "backend", + "bd_version", + "beads_dir", + "capabilities", + "database", + "dolt_mode", + "project_id", + "repo_root", + "schema_version" + ], + "operations": [ + { + "method": "GET", + "path": "/healthz", + "operation_id": "health", + "query": [], + "body": [] + }, + { + "method": "GET", + "path": "/v0/beads/config", + "operation_id": "listSettings", + "query": [], + "body": [] + }, + { + "method": "DELETE", + "path": "/v0/beads/config/{key}", + "operation_id": "unsetSetting", + "query": [], + "body": [] + }, + { + "method": "GET", + "path": "/v0/beads/config/{key}", + "operation_id": "getSetting", + "query": [], + "body": [] + }, + { + "method": "PUT", + "path": "/v0/beads/config/{key}", + "operation_id": "setSetting", + "query": [], + "body": [ + "value" + ] + }, + { + "method": "GET", + "path": "/v0/beads/context", + "operation_id": "getContext", + "query": [], + "body": [] + }, + { + "method": "GET", + "path": "/v0/beads/dependencies", + "operation_id": "listDependencies", + "query": [ + "issue_id", + "type" + ], + "body": [] + }, + { + "method": "GET", + "path": "/v0/beads/dependencies/blocking", + "operation_id": "listBlockingAnnotations", + "query": [ + "issue_id" + ], + "body": [] + }, + { + "method": "GET", + "path": "/v0/beads/dependencies/cycles", + "operation_id": "listDependencyCycles", + "query": [], + "body": [] + }, + { + "method": "GET", + "path": "/v0/beads/dependencies/tree", + "operation_id": "getDependencyTree", + "query": [ + "direction", + "max_depth", + "root_id", + "status" + ], + "body": [] + }, + { + "method": "POST", + "path": "/v0/beads/dependencies:add", + "operation_id": "addDependencies", + "query": [], + "body": [ + "actor", + "edges" + ] + }, + { + "method": "GET", + "path": "/v0/beads/dependencies:count", + "operation_id": "countDependencyEdges", + "query": [ + "direction", + "issue_id", + "status", + "type" + ], + "body": [] + }, + { + "method": "POST", + "path": "/v0/beads/dependencies:remove", + "operation_id": "removeDependency", + "query": [], + "body": [ + "actor", + "depends_on_id", + "issue_id" + ] + }, + { + "method": "GET", + "path": "/v0/beads/events", + "operation_id": "listEvents", + "query": [ + "limit", + "since" + ], + "body": [] + }, + { + "method": "GET", + "path": "/v0/beads/events:watch", + "operation_id": "watchEvents", + "query": [ + "since" + ], + "body": [] + }, + { + "method": "GET", + "path": "/v0/beads/issues", + "operation_id": "listIssues", + "query": [ + "all", + "assignee", + "brief", + "created_after", + "created_before", + "cursor", + "exclude_label", + "has_metadata_key", + "include_ephemeral", + "include_gates", + "include_infra", + "include_templates", + "label", + "label_any", + "limit", + "metadata_field", + "parent", + "reverse", + "sort", + "status", + "type" + ], + "body": [] + }, + { + "method": "POST", + "path": "/v0/beads/issues", + "operation_id": "createIssue", + "query": [], + "body": [ + "acceptance_criteria", + "actor", + "assignee", + "defer_until", + "dependencies", + "description", + "design", + "due_at", + "ephemeral", + "estimated_minutes", + "external_ref", + "force_id_prefix", + "id", + "inherit_labels_from_parent", + "issue_type", + "labels", + "metadata", + "no_history", + "notes", + "owner", + "parent_id", + "priority", + "sender", + "status", + "title", + "waits_for" + ] + }, + { + "method": "GET", + "path": "/v0/beads/issues/{id}", + "operation_id": "getIssue", + "query": [ + "brief_deps", + "include_comments", + "include_dependents" + ], + "body": [] + }, + { + "method": "PATCH", + "path": "/v0/beads/issues/{id}", + "operation_id": "updateIssue", + "query": [], + "body": [ + "actor", + "expected_assignee", + "expected_status", + "expected_version", + "force_assignee_transfer", + "force_close_policy", + "patch" + ] + }, + { + "method": "POST", + "path": "/v0/beads/issues/{id}/comments", + "operation_id": "addComment", + "query": [], + "body": [ + "author", + "text" + ] + }, + { + "method": "GET", + "path": "/v0/beads/issues/{id}/related", + "operation_id": "listRelatedIssues", + "query": [ + "direction", + "type" + ], + "body": [] + }, + { + "method": "POST", + "path": "/v0/beads/issues/{id}:casMetadata", + "operation_id": "compareAndSetMetadata", + "query": [], + "body": [ + "actor", + "expected", + "key", + "value" + ] + }, + { + "method": "POST", + "path": "/v0/beads/issues/{id}:claim", + "operation_id": "claimIssue", + "query": [], + "body": [ + "actor" + ] + }, + { + "method": "POST", + "path": "/v0/beads/issues/{id}:close", + "operation_id": "closeIssue", + "query": [], + "body": [ + "actor", + "expected_version", + "force", + "reason", + "session" + ] + }, + { + "method": "POST", + "path": "/v0/beads/issues/{id}:release", + "operation_id": "releaseIssue", + "query": [], + "body": [ + "actor", + "expected_assignee", + "force" + ] + }, + { + "method": "POST", + "path": "/v0/beads/issues/{id}:reopen", + "operation_id": "reopenIssue", + "query": [], + "body": [ + "actor", + "expected_version", + "reason" + ] + }, + { + "method": "POST", + "path": "/v0/beads/issues:batchApply", + "operation_id": "applyBatch", + "query": [], + "body": [ + "actor", + "force_id_prefix", + "items", + "provenance", + "skip_per_edge_cycle_check" + ] + }, + { + "method": "POST", + "path": "/v0/beads/issues:batchClose", + "operation_id": "batchCloseIssues", + "query": [], + "body": [ + "actor", + "claim_next", + "force", + "items", + "session" + ] + }, + { + "method": "POST", + "path": "/v0/beads/issues:batchCreate", + "operation_id": "batchCreateIssues", + "query": [], + "body": [ + "actor", + "items" + ] + }, + { + "method": "POST", + "path": "/v0/beads/issues:claimNext", + "operation_id": "claimNextIssue", + "query": [ + "assignee", + "exclude_label", + "exclude_type", + "has_metadata_key", + "include_deferred", + "include_ephemeral", + "label", + "label_any", + "label_pattern", + "label_regex", + "metadata_field", + "parent", + "priority", + "sort", + "type", + "unassigned" + ], + "body": [ + "actor" + ] + }, + { + "method": "GET", + "path": "/v0/beads/issues:count", + "operation_id": "countIssues", + "query": [ + "assignee", + "closed_after", + "closed_before", + "created_after", + "created_before", + "desc_contains", + "empty_description", + "group_by", + "id", + "include_infra", + "label", + "label_any", + "metadata_field", + "no_assignee", + "no_labels", + "notes_contains", + "priority", + "priority_max", + "priority_min", + "status", + "title", + "title_contains", + "type", + "updated_after", + "updated_before" + ], + "body": [] + }, + { + "method": "POST", + "path": "/v0/beads/issues:delete", + "operation_id": "deleteIssues", + "query": [], + "body": [ + "actor", + "cascade", + "dry_run", + "expected_version", + "force", + "ids" + ] + }, + { + "method": "GET", + "path": "/v0/beads/issues:query", + "operation_id": "queryIssues", + "query": [ + "all", + "limit", + "q", + "reverse", + "sort" + ], + "body": [] + }, + { + "method": "POST", + "path": "/v0/beads/issues:sweep", + "operation_id": "sweepIssues", + "query": [], + "body": [ + "actor", + "closed_before", + "dry_run", + "pattern", + "protect_referenced", + "tier" + ] + }, + { + "method": "GET", + "path": "/v0/beads/memories", + "operation_id": "listMemories", + "query": [ + "search" + ], + "body": [] + }, + { + "method": "POST", + "path": "/v0/beads/memories", + "operation_id": "rememberMemory", + "query": [], + "body": [ + "content", + "key" + ] + }, + { + "method": "DELETE", + "path": "/v0/beads/memories/{key}", + "operation_id": "forgetMemory", + "query": [], + "body": [] + }, + { + "method": "GET", + "path": "/v0/beads/memories/{key}", + "operation_id": "getMemory", + "query": [], + "body": [] + }, + { + "method": "GET", + "path": "/v0/beads/ready", + "operation_id": "listReadyWork", + "query": [ + "assignee", + "brief", + "exclude_label", + "exclude_type", + "has_metadata_key", + "include_deferred", + "include_ephemeral", + "label", + "label_any", + "label_pattern", + "label_regex", + "limit", + "metadata_field", + "parent", + "priority", + "sort", + "type", + "unassigned" + ], + "body": [] + }, + { + "method": "GET", + "path": "/v0/beads/ready:count", + "operation_id": "countReadyWork", + "query": [ + "assignee", + "exclude_label", + "exclude_type", + "has_metadata_key", + "include_deferred", + "include_ephemeral", + "label", + "label_any", + "label_pattern", + "label_regex", + "metadata_field", + "parent", + "priority", + "type", + "unassigned" + ], + "body": [] + }, + { + "method": "GET", + "path": "/v0/beads/stats", + "operation_id": "getStats", + "query": [ + "assignee", + "skip_blocked" + ], + "body": [] + } + ] +} \ No newline at end of file diff --git a/cmd/bd/BUILD.bazel b/cmd/bd/BUILD.bazel index 67f4cb9ed..24364853d 100644 --- a/cmd/bd/BUILD.bazel +++ b/cmd/bd/BUILD.bazel @@ -516,6 +516,7 @@ go_binary( race = "off", visibility = [ "//backend/http:__pkg__", + "//backend/http/corpus:__pkg__", "//cmd/bd:__subpackages__", "//internal/beads:__pkg__", "//internal/storage/uow:__pkg__", diff --git a/engdocs/TESTING.md b/engdocs/TESTING.md index 3b4178066..595887d92 100644 --- a/engdocs/TESTING.md +++ b/engdocs/TESTING.md @@ -41,7 +41,7 @@ Each lane, as bazel.yml runs it (add your `--config=fork-cache` or | Integration (`bazel-integration`) | `bazel test //... --config=integration` | The `integration`-tagged build. Runs with the read-only cache too. | | Dolt server (`bazel-doltserver`) | `bazel test //... --config=doltserver` | Starts its own `dolt sql-server` from the pinned binary; no docker. | | cmd/bd Dolt server (`bazel-cmd-dolt`) | `bazel test //cmd/bd:bd_dolt_server_test --config=doltserver-cmd` | 32 shards of the integration-tagged cmd/bd suite (the slowest tests pinned to 16 of them). | -| Embedded Dolt (`bazel-embedded`) | `bazel test //... --config=embedded` | Remote execution only in CI; locally it is slow. | +| Embedded Dolt (`bazel-embedded`) | `bazel test //... --config=embedded` | Remote execution only in CI; locally it is slow. Also runs the served http tiers with `BEADS_HTTP_TEST_REQUIRED=1`: `//backend/http:http_served_test`, `//internal/httpclient:httpclient_served_test` and the command corpus `//backend/http/corpus:corpus_served_test`. | | Proxied server (`bazel-proxied`) | `bazel test //... --config=doltserver-proxied` | Remote execution only: 34 shards, each with a Dolt server. | | Server-Dolt storage (`bazel-server-storage`) | `bazel test //... --config=doltserver-integration` | Remote execution only. | | Docs | `make check-docs` | `//test/docsync:docsync_test` and `//scripts/repochecks:doc_freshness_test` in the test lane's configuration, then `scripts/check-doc-flags.sh` against the pinned release. | diff --git a/scripts/pr_risk_bazel_coverage_test.go b/scripts/pr_risk_bazel_coverage_test.go index 41cab234f..31dd9fa3c 100644 --- a/scripts/pr_risk_bazel_coverage_test.go +++ b/scripts/pr_risk_bazel_coverage_test.go @@ -907,6 +907,12 @@ func TestBazelRetiredLanesCannotBeNarrowed(t *testing.T) { []string{"$(rootpath :http_test)", "-test.v", "-test.count=1", "-test.timeout=19m"}, map[string]string{"BEADS_HTTP_TEST_REQUIRED": "1", "BEADS_TEST_BD_BINARY": "$(rlocationpath //cmd/bd:bd_for_tests)", "BEADS_TEST_EMBEDDED_DOLT": "1"}, }, + // The served command corpus (backend/http/corpus/README.md), with the + // same switches as http_served_test. + "//backend/http/corpus:corpus_served_test": { + []string{"$(rootpath :corpus_test)", "-test.v", "-test.count=1", "-test.timeout=19m"}, + map[string]string{"BEADS_HTTP_TEST_REQUIRED": "1", "BEADS_TEST_BD_BINARY": "$(rlocationpath //cmd/bd:bd_for_tests)", "BEADS_TEST_EMBEDDED_DOLT": "1"}, + }, "//cmd/bd:bd_proxied_test": { []string{"$(rootpath //:.github/scripts/proxied-test-shard.sh)", "BEADS_TEST_CMD_BINARY", "$(rootpath :bd_test)"}, map[string]string{