diff --git a/backend/http/BUILD.bazel b/backend/http/BUILD.bazel index 16ea2bf83..b78058b0c 100644 --- a/backend/http/BUILD.bazel +++ b/backend/http/BUILD.bazel @@ -63,7 +63,23 @@ go_test( # in-process server over embedded Dolt and BEADS_HTTP_TEST_REQUIRED=1 turns a # skip of that tier into a failure. http_test alone skips the TestE2E_* cases # in every lane, because no lane sets those variables for it. It repeats -# http_test's `env` (go_test_variant.sh). +# http_test's `env` (go_test_pinned_shard.sh, like go_test_variant.sh, does +# not inherit it). +# +# Shard count: the package's tests are serial, and as one shard the target +# ran 235-394s on the lane's executors (2026-10-10..13), the embedded lane's +# longest action in most runs where it executed; the lane's other shards end +# by ~90-125s. The 19 TestE2E_* cases that take >= 3s (5-74s each, the +# maximum over three lane runs and one local run) are pinned +# (tools/bazel/go_test_pinned_shard.sh, served_pinned_shards.txt from +# tools/bazel/pin_shards.py --shards 6 --min-seconds 3) onto 6 shards of +# <= ~88 measured seconds each, and the binary's round-robin runs the other +# ~28 (well under a second together) in the 7th. +# Regenerate the manifest from runs of the lane: +# bazel test //backend/http:http_served_test --config=embedded +# python3 tools/bazel/pin_shards.py --shards 6 --min-seconds 3 \ +# --write backend/http/served_pinned_shards.txt \ +# bazel-testlogs/backend/http/http_served_test # # Tags: # embedded: the embedded-Dolt tier; excluded from --config=prcore/ci, @@ -74,14 +90,16 @@ sh_test( # -test.timeout=19m fires (with a goroutine dump) before Bazel's 1200s # action limit (eternal under --config=embedded). timeout = "eternal", - srcs = ["//tools/bazel:go_test_variant.sh"], + srcs = ["//tools/bazel:go_test_pinned_shard.sh"], args = [ + "$(rootpath :served_pinned_shards.txt)", "$(rootpath :http_test)", "-test.v", "-test.count=1", "-test.timeout=19m", ], data = [ + "served_pinned_shards.txt", ":http_test", "//cmd/bd:bd_for_tests", ], @@ -90,6 +108,7 @@ sh_test( "BEADS_TEST_BD_BINARY": "$(rlocationpath //cmd/bd:bd_for_tests)", "BEADS_TEST_EMBEDDED_DOLT": "1", }, + shard_count = 7, tags = ["embedded"], ) diff --git a/backend/http/corpus/BUILD.bazel b/backend/http/corpus/BUILD.bazel index 7a6f87f43..29c507164 100644 --- a/backend/http/corpus/BUILD.bazel +++ b/backend/http/corpus/BUILD.bazel @@ -9,6 +9,7 @@ go_test( size = "large", timeout = "long", srcs = [ + "corpus_parts_test.go", "corpus_test.go", "harness_test.go", "reference_proxy_test.go", @@ -28,10 +29,20 @@ go_test( ) # corpus_test again, with the corpus tier on: BEADS_TEST_EMBEDDED_DOLT=1 runs -# TestServedCorpus and BEADS_HTTP_TEST_REQUIRED=1 turns a skip of it into a -# failure. corpus_test alone skips TestServedCorpus in every lane, because no +# TestServedCorpusPart1..N and BEADS_HTTP_TEST_REQUIRED=1 turns a skip of +# them into a failure. corpus_test alone skips them in every lane, because no # lane sets those variables for it. It repeats corpus_test's `env` -# (go_test_variant.sh). +# (go_test_pinned_shard.sh, like go_test_variant.sh, does not inherit it). +# +# Shard count: as one shard (one TestServedCorpus over every shape) the +# target ran 292-597s on the lane's executors (2026-10-11..13), the embedded +# lane's longest action in most runs where it executed. About 100s of that +# is the per-process server and seed, the rest the ~130 shapes (~1.7s each, +# serial). The shapes are now dealt round robin across corpusParts = 4 parts +# (corpus_test.go), each a top-level test with its own server and seed, and +# served_pinned_shards.txt pins part k to shard k; the 5th shard runs the +# package's engine-free tests through the binary's round-robin. Each part +# pays the seed again, so a part's shard is ~100s + a quarter of the shapes. # # Tags: # embedded: the embedded-Dolt tier; excluded from --config=prcore/ci, @@ -42,14 +53,16 @@ sh_test( # -test.timeout=19m fires (with a goroutine dump) before Bazel's 1200s # action limit (eternal under --config=embedded). timeout = "eternal", - srcs = ["//tools/bazel:go_test_variant.sh"], + srcs = ["//tools/bazel:go_test_pinned_shard.sh"], args = [ + "$(rootpath :served_pinned_shards.txt)", "$(rootpath :corpus_test)", "-test.v", "-test.count=1", "-test.timeout=19m", ], data = [ + "served_pinned_shards.txt", ":corpus_test", "//cmd/bd:bd_for_tests", ], @@ -58,6 +71,7 @@ sh_test( "BEADS_TEST_BD_BINARY": "$(rlocationpath //cmd/bd:bd_for_tests)", "BEADS_TEST_EMBEDDED_DOLT": "1", }, + shard_count = 5, tags = ["embedded"], ) diff --git a/backend/http/corpus/README.md b/backend/http/corpus/README.md index 57e17550b..80a138a09 100644 --- a/backend/http/corpus/README.md +++ b/backend/http/corpus/README.md @@ -8,7 +8,10 @@ embedded workspace. ## What runs -`TestServedCorpus` does the following: +The shapes are dealt round robin, in corpus order, across `corpusParts` +parts (`corpus_test.go`). Each part is a top-level test, +`TestServedCorpusPart1` .. `TestServedCorpusPartN`, and each does the +following for its share of the shapes: 1. Builds `bd` from this tree. `BEADS_TEST_BD_BINARY=` reuses a prebuilt binary instead. @@ -29,8 +32,8 @@ embedded workspace. `seed` from `testdata/corpus.yaml`: one graph of named rows, plus bulk graphs that put more than one server page of ready rows and wisps in the store. Both sides use `BEADS_ACTOR=worker` and set `CLAUDE_SESSION_ID`. -5. Runs each shape as a subtest, on both sides. The embedded run must exit - with the shape's `expect_exit` and match its `expect_stderr`. If it does +5. Runs each of its shapes as a subtest, on both sides. The embedded run must + exit with the shape's `expect_exit` and match its `expect_stderr`. If it does not, the shape is wrong or bd changed, and the subtest fails as a corpus bug that no pending mark or allowlist entry hides. The served run must: - exit like the embedded run; @@ -43,17 +46,35 @@ embedded workspace. - for writes, give the same JSON from every `verify` read on both sides afterwards. -The run takes about five minutes. +A part takes its own server, workspaces and seed, so no shape can see +another part's writes. That is safe because no shape depends on another +shape's effects: +- every check compares the served run with the embedded run after the same + commands, so it holds whatever ran before it on both sides; +- each part keeps corpus order, so its reads still run on the bare seed, + before any of its writes; +- each write owns its target ids (`e2e-t-*`, or ids it creates). + +`TestCorpusPartsDealEveryShapeOnce` checks that every shape is in exactly one +part, in corpus order. A new shape must keep that independence: it may rely on +the seed and its own `setup`, never on another shape having run. + +Every part seeds again (the server and seed take about 100s of a part's run +on the CI executors), so running all the parts in one process takes longer +than one run over every shape did. Run one part with +`-run 'TestServedCorpusPart2$'`. ``` CGO_ENABLED=1 BEADS_TEST_EMBEDDED_DOLT=1 BEADS_HTTP_TEST_REQUIRED=1 \ - go test -tags gms_pure_go -run 'TestServedCorpus|TestCorpusFile' ./backend/http/corpus/ + go test -tags gms_pure_go -run 'TestServedCorpus|TestCorpus' ./backend/http/corpus/ ``` In CI, `//backend/http/corpus:corpus_served_test` runs the suite with both variables set, in the Embedded Dolt lane (`--config=embedded`; see -[engdocs/TESTING.md](../../../engdocs/TESTING.md)). `corpus_test` on its own -sets neither, so it skips `TestServedCorpus`. +[engdocs/TESTING.md](../../../engdocs/TESTING.md)). Its +`served_pinned_shards.txt` pins part k to Bazel shard k, so the parts run +concurrently. `corpus_test` on its own sets neither variable, so it skips +the parts. Two environment variables change the run: - `BEADS_CORPUS_REPORT=` writes a JSON summary with per-shape status diff --git a/backend/http/corpus/corpus_parts_test.go b/backend/http/corpus/corpus_parts_test.go new file mode 100644 index 000000000..bc57f6bb3 --- /dev/null +++ b/backend/http/corpus/corpus_parts_test.go @@ -0,0 +1,111 @@ +//go:build cgo + +package corpus_test + +import ( + "fmt" + "reflect" + "runtime" + "strings" + "testing" +) + +// TestCorpusPartsDealEveryShapeOnce runs without the embedded engine. It +// holds the deal TestServedCorpusPart1..N run to what makes splitting the +// corpus safe: every shape lands in exactly one part, each part keeps corpus +// order (so a part's reads still run before its writes, on its own fresh +// seed), no part is empty, and there is one TestServedCorpusPartN per part. +func TestCorpusPartsDealEveryShapeOnce(t *testing.T) { + if len(servedCorpusParts) != corpusParts { + t.Fatalf("servedCorpusParts has %d tests, want corpusParts = %d", len(servedCorpusParts), corpusParts) + } + c, _ := loadCorpus(t) + index := map[string]int{} + for i, s := range c.Shapes { + index[s.ID] = i + } + seen := map[string]int{} + for part := 1; part <= corpusParts; part++ { + shapes := corpusPart(c.Shapes, part) + if len(shapes) == 0 { + t.Errorf("part %d runs no shapes", part) + } + last := -1 + for _, s := range shapes { + if prev, dup := seen[s.ID]; dup { + t.Errorf("shape %s is in part %d and part %d", s.ID, prev, part) + } + seen[s.ID] = part + i, ok := index[s.ID] + if !ok { + t.Errorf("part %d runs %s, which is not a corpus shape", part, s.ID) + continue + } + if i <= last { + t.Errorf("part %d runs %s out of corpus order", part, s.ID) + } + last = i + } + } + for _, s := range c.Shapes { + if _, ok := seen[s.ID]; !ok { + t.Errorf("shape %s is in no part", s.ID) + } + } +} + +// The deal itself, on a corpus of 10 against 4 parts: round robin in corpus +// order. +func TestCorpusPartIsRoundRobinInOrder(t *testing.T) { + var shapes []shape + for i := 0; i < 10; i++ { + shapes = append(shapes, shape{ID: fmt.Sprint(i)}) + } + want := map[int]string{1: "[0 4 8]", 2: "[1 5 9]", 3: "[2 6]", 4: "[3 7]"} + for part, w := range want { + var ids []string + for _, s := range corpusPartOf(shapes, part, 4) { + ids = append(ids, s.ID) + } + if got := fmt.Sprint(ids); got != w { + t.Errorf("part %d of 4 = %s, want %s", part, got, w) + } + } +} + +// A part test wired to another part's number would run that part's shapes +// twice and drop its own without a word; testServedCorpus refuses to run +// unless the test's name is its part's (corpusPartNameErr). +func TestCorpusPartNameMustMatchItsPart(t *testing.T) { + if err := corpusPartNameErr("TestServedCorpusPart3", 3); err != nil { + t.Errorf("matching name: %v", err) + } + for _, c := range []struct { + name string + part int + }{ + {"TestServedCorpusPart3", 2}, + {"TestServedCorpusPart1", 10}, + {"TestServedCorpusPart10", 1}, + {"TestServedCorpusPart3/sub", 3}, + {"TestServedCorpus", 1}, + } { + if err := corpusPartNameErr(c.name, c.part); err == nil { + t.Errorf("corpusPartNameErr(%q, %d) = nil, want a mismatch", c.name, c.part) + } + } + // Every wired part test passes the guard: run each with the engine + // off, so the guard (which comes first) is all that can fail it. + for k, fn := range servedCorpusParts { + want := fmt.Sprintf("TestServedCorpusPart%d", k+1) + if got := testFuncName(fn); got != want { + t.Errorf("servedCorpusParts[%d] is %s, want %s", k, got, want) + } + } +} + +// testFuncName is fn's unqualified function name. +func testFuncName(fn func(*testing.T)) string { + full := runtime.FuncForPC(reflect.ValueOf(fn).Pointer()).Name() + return full[strings.LastIndex(full, ".")+1:] +} diff --git a/backend/http/corpus/corpus_test.go b/backend/http/corpus/corpus_test.go index 96a3b095e..1bde7ed5b 100644 --- a/backend/http/corpus/corpus_test.go +++ b/backend/http/corpus/corpus_test.go @@ -237,14 +237,69 @@ type outcome struct { requests int64 } -// TestServedCorpus is the suite. Each shape is a subtest; a pending shape is -// still RUN and then skipped with what it did, so a slice that fixes it sees -// the subtest fail with "now passes; remove the pending mark". -// BEADS_CORPUS_SKIP_PENDING=1 skips pending shapes without running them. -// BEADS_CORPUS_REPORT= writes a JSON summary. -func TestServedCorpus(t *testing.T) { +// corpusParts is how many parts the shapes are dealt across. Each part is a +// top-level test of its own (TestServedCorpusPart1..N, servedCorpusParts) +// that starts its own server and seeds its own pair of workspaces, so the +// embedded lane runs each part in a Bazel shard of its own +// (served_pinned_shards.txt pins part k to shard k). The parts can be split +// because no shape depends on another's effects: every comparison is the +// served side against the embedded side after the same commands, and each +// part keeps corpus order, so its reads still run on the bare seed before +// any of its writes (see testdata/corpus.yaml's header). +const corpusParts = 4 + +// servedCorpusParts are the parts' tests, one per part, in part order. +var servedCorpusParts = []func(*testing.T){ + TestServedCorpusPart1, TestServedCorpusPart2, TestServedCorpusPart3, TestServedCorpusPart4, +} + +func TestServedCorpusPart1(t *testing.T) { testServedCorpus(t, 1) } +func TestServedCorpusPart2(t *testing.T) { testServedCorpus(t, 2) } +func TestServedCorpusPart3(t *testing.T) { testServedCorpus(t, 3) } +func TestServedCorpusPart4(t *testing.T) { testServedCorpus(t, 4) } + +// corpusPartNameErr fails a part test whose name is not its part's: a +// TestServedCorpusPart3 that ran part 2 would run part 2's shapes twice and +// silently drop its own. +func corpusPartNameErr(name string, part int) error { + if want := fmt.Sprintf("TestServedCorpusPart%d", part); name != want { + return fmt.Errorf("%s runs corpus part %d; only %s may (fix its testServedCorpus call)", name, part, want) + } + return nil +} + +// corpusPart returns the shapes part (1..corpusParts) runs. +func corpusPart(shapes []shape, part int) []shape { + return corpusPartOf(shapes, part, corpusParts) +} + +// corpusPartOf deals shapes round robin, in corpus order, across parts parts +// and returns part's (1-based) share: shapes part-1, part-1+parts, and so on. +// Round robin keeps each part a mix of the corpus's reads and writes in +// their original order. +func corpusPartOf(shapes []shape, part, parts int) []shape { + var out []shape + for i := part - 1; i < len(shapes); i += parts { + out = append(out, shapes[i]) + } + return out +} + +// testServedCorpus is the suite, for one part of the corpus. Each shape is a +// subtest; a pending shape is still RUN and then skipped with what it did, so +// a slice that fixes it sees the subtest fail with "now passes; remove the +// pending mark". BEADS_CORPUS_SKIP_PENDING=1 skips pending shapes without +// running them. BEADS_CORPUS_REPORT= writes a JSON summary of every +// part this process has run. +func testServedCorpus(t *testing.T, part int) { + // First, before the engine gate: a part test wired to the wrong part + // fails in every lane, not only where the corpus runs. + if err := corpusPartNameErr(t.Name(), part); err != nil { + t.Fatal(err) + } skipUnlessEmbeddedDolt(t) c, allow := loadCorpus(t) + shapes := corpusPart(c.Shapes, part) profile := loadReferenceProfile(t) bin := buildBD(t) @@ -283,7 +338,7 @@ func TestServedCorpus(t *testing.T) { skipPending := os.Getenv("BEADS_CORPUS_SKIP_PENDING") == "1" var outcomes []outcome - for _, s := range c.Shapes { + for _, s := range shapes { s := s t.Run(s.ID, func(t *testing.T) { o := outcome{id: s.ID} @@ -328,7 +383,7 @@ func TestServedCorpus(t *testing.T) { }) } - summarize(t, outcomes) + summarize(t, part, len(c.Shapes), outcomes) } // runShape runs one shape on both sides and returns every way the served run @@ -777,7 +832,11 @@ func bulkPlan(b bulkGraph) string { return string(raw) } -func summarize(t *testing.T, outcomes []outcome) { +// reported is every outcome of every part this process has run, in the order +// the parts ran, for BEADS_CORPUS_REPORT. +var reported []outcome + +func summarize(t *testing.T, part, total int, outcomes []outcome) { t.Helper() counts := map[string]int{} var lines []string @@ -789,7 +848,8 @@ func summarize(t *testing.T, outcomes []outcome) { counts[key]++ lines = append(lines, fmt.Sprintf("%-26s %-48s req=%-3d %s", o.status, o.id, o.requests, firstLine(o.detail))) } - t.Logf("corpus summary: %v\n%s", counts, strings.Join(lines, "\n")) + t.Logf("corpus summary, part %d of %d (%d of %d shapes): %v\n%s", part, corpusParts, len(outcomes), total, counts, strings.Join(lines, "\n")) + reported = append(reported, outcomes...) if path := os.Getenv("BEADS_CORPUS_REPORT"); path != "" { type row struct { ID string `json:"id"` @@ -798,7 +858,7 @@ func summarize(t *testing.T, outcomes []outcome) { Detail string `json:"detail,omitempty"` } var out []row - for _, o := range outcomes { + for _, o := range reported { out = append(out, row{o.id, o.status, o.requests, o.detail}) } raw, _ := json.MarshalIndent(out, "", " ") diff --git a/backend/http/corpus/served_pinned_shards.txt b/backend/http/corpus/served_pinned_shards.txt new file mode 100644 index 000000000..6fc0f44d9 --- /dev/null +++ b/backend/http/corpus/served_pinned_shards.txt @@ -0,0 +1,12 @@ +# Pinned shards for tools/bazel/go_test_pinned_shard.sh. Not pin_shards.py's: +# each corpus part (corpus_test.go's corpusParts; TestServedCorpusPart1..N) +# runs alone in the shard of its number, on its own server and seed, and the +# binary's round-robin runs the package's other (engine-free) tests in the +# one shard left over. Change corpusParts, this file and the target's +# shard_count together (scripts TestCorpusPartsArePinnedOnePerShard). +# +# +1 TestServedCorpusPart1 +2 TestServedCorpusPart2 +3 TestServedCorpusPart3 +4 TestServedCorpusPart4 diff --git a/backend/http/corpus/testdata/corpus.yaml b/backend/http/corpus/testdata/corpus.yaml index ee970ef53..10c43ba34 100644 --- a/backend/http/corpus/testdata/corpus.yaml +++ b/backend/http/corpus/testdata/corpus.yaml @@ -13,6 +13,9 @@ # and on an embedded workspace, with BEADS_ACTOR=worker and CLAUDE_SESSION_ID set # on both. See README.md for the fields. Reads come first; writes after them # each own their target ids (e2e-t-*), so write order never changes a read. +# The shapes are dealt round robin, in this order, across the suite's parts, +# and each part seeds its own workspaces: a shape may rely on the seed and its +# own setup, never on another shape having run. # # pending: {slice, reason} marks a known failure. The shape still runs; when it # passes the subtest FAILS until the mark is removed. That is how a slice proves @@ -87,7 +90,7 @@ files: {"key": "t-cm", "id": "e2e-t-cm", "title": "target cm", "type": "task"}, {"key": "t-rel", "id": "e2e-t-rel", "title": "target rel", "type": "task", "assignee": "worker"}, {"key": "t-xfer", "id": "e2e-t-xfer", "title": "target xfer", "type": "task", "assignee": "worker"}, - {"key": "t-ifa", "id": "e2e-t-ifa", "title": "target ifa", "type": "task", "assignee": "worker"}, + {"key": "t-ifa", "id": "e2e-t-ifa", "title": "target ifa", "type": "task", "assignee": "worker", "metadata": {"seeded": "keep"}}, {"key": "t-death", "id": "e2e-t-death", "title": "target death", "type": "task", "assignee": "worker"}, {"key": "t-rc", "id": "e2e-t-rc", "title": "target rc", "type": "task", "assignee": "worker"}, {"key": "t-boot", "id": "e2e-t-boot", "title": "target boot", "type": "task"}, @@ -530,6 +533,10 @@ shapes: args: [update, --json, e2e-t-ifa, --set-metadata, k=v, --if-status, in_progress, --if-assignee, worker] compare: exit verify: [[show, --json, e2e-t-ifa]] + # e2e-t-ifa is seeded with metadata seeded=keep, so each refused (exit 13) + # update's verify still holds that the refusal left existing metadata + # alone, in whichever corpus part runs it (the parts seed separately, so + # the k=v written by gc-update-if-assignment may not be there). - id: gc-update-if-assignment-assignee-stale source: gc bdstore_conditional.go UpdateIfAssignment mismatch (exit 13) args: [update, --json, e2e-t-ifa, --set-metadata, probe=a, --if-status, in_progress, --if-assignee, nobody] diff --git a/backend/http/served_pinned_shards.txt b/backend/http/served_pinned_shards.txt new file mode 100644 index 000000000..6fca673c7 --- /dev/null +++ b/backend/http/served_pinned_shards.txt @@ -0,0 +1,29 @@ +# Pinned shards for tools/bazel/go_test_pinned_shard.sh. Generated by +# tools/bazel/pin_shards.py; regenerate rather than edit (see its usage). +# +# Every top-level test measured at >= 3s, bin-packed +# longest-first onto these shards; every other test runs in the +# binary's own round-robin shards. Measured seconds per pinned shard: +# 86 87 86 88 88 87 +# Unpinned tests: 28, 0s in total. +# +# +1 TestE2E_ConnectClearRestoresThePreviousBackend +1 TestE2E_DefaultDepTreeOverHTTP +2 TestE2E_PingOverHTTP +2 TestE2E_ShowJSONCountsMatchEmbedded +3 TestE2E_AdvertisedExternalPolicyIsPassedThrough +3 TestE2E_ListWithGCFiltersMatchesEmbedded +3 TestE2E_ShowMissingIDIsNotFoundInOneLookup +4 TestE2E_BDOverHTTP +4 TestE2E_CloseAndReopenUnderClaudeSession +4 TestE2E_SearchOverHTTP +4 TestE2E_ShowCurrentOverHTTP +5 TestE2E_ForceConnectWritesSucceedDespiteDifferingLocalProjectID +5 TestE2E_MoleculeCloseAgainstAnOlderServer +5 TestE2E_ReconnectRefusesADifferentProjectUnlessForced +5 TestE2E_SwitchingBackToDoltLeavesOriginalIdentityIntact +6 TestE2E_ClaimOfAbsentIDIsGCNotFound +6 TestE2E_ExternalDependencyBlockingStillEnforced +6 TestE2E_MoleculesOverHTTP +6 TestE2E_RecomputeBlockedOverHTTP diff --git a/engdocs/TESTING.md b/engdocs/TESTING.md index d38f4e0fd..d3a7177b3 100644 --- a/engdocs/TESTING.md +++ b/engdocs/TESTING.md @@ -41,7 +41,7 @@ Each lane, as bazel.yml runs it (add your `--config=fork-cache` or | Integration (`bazel-integration`) | `bazel test //... --config=integration` | The `integration`-tagged build. Runs with the read-only cache too. | | Dolt server (`bazel-doltserver`) | `bazel test //... --config=doltserver` | Starts its own `dolt sql-server` from the pinned binary; no docker. | | cmd/bd Dolt server (`bazel-cmd-dolt`) | `bazel test //cmd/bd:bd_dolt_server_test --config=doltserver-cmd` | 32 shards of the integration-tagged cmd/bd suite (the slowest tests pinned to 16 of them). | -| Embedded Dolt (`bazel-embedded`) | `bazel test //... --config=embedded` | Remote execution only in CI; locally it is slow. Also runs the served http tiers with `BEADS_HTTP_TEST_REQUIRED=1`: `//backend/http:http_served_test`, `//internal/httpclient:httpclient_served_test` and the command corpus `//backend/http/corpus:corpus_served_test`. | +| Embedded Dolt (`bazel-embedded`) | `bazel test //... --config=embedded` | Remote execution only in CI; locally it is slow. Also runs the served http tiers with `BEADS_HTTP_TEST_REQUIRED=1`: `//backend/http:http_served_test`, `//internal/httpclient:httpclient_served_test` and the command corpus `//backend/http/corpus:corpus_served_test`, each sharded through `tools/bazel/go_test_pinned_shard.sh`. | | Proxied server (`bazel-proxied`) | `bazel test //... --config=doltserver-proxied` | Remote execution only: 34 shards, each with a Dolt server. | | Dolt server + proxied (`bazel-dolt-race`) | `bazel test //... --config=dolt-race` | Both rows above in one invocation; CI runs this instead of them in mode `remote`. | | Server-Dolt storage (`bazel-server-storage`) | `bazel test //... --config=doltserver-integration` | Remote execution only. | diff --git a/scripts/BUILD.bazel b/scripts/BUILD.bazel index ecfe35782..c050f2c01 100644 --- a/scripts/BUILD.bazel +++ b/scripts/BUILD.bazel @@ -202,6 +202,8 @@ go_test( "//:.github/scripts/server-storage-test-shards.txt", "//:nested_go_modules", "//:repo_go_test_srcs", + "//backend/http:repo_other_files", + "//backend/http/corpus:repo_other_files", "//cmd/bd:repo_other_files", "//internal/httpclient:repo_other_files", "//internal/storage/dolt:repo_other_files", diff --git a/scripts/pinned_shards_helpers_test.go b/scripts/pinned_shards_helpers_test.go index 52476fbb5..a4e96e9d2 100644 --- a/scripts/pinned_shards_helpers_test.go +++ b/scripts/pinned_shards_helpers_test.go @@ -19,17 +19,21 @@ const pinnedShardWrapper = "tools/bazel/go_test_pinned_shard.sh" // manifest, the directory holding the go_test's sources, and the rule's one // lane tag. // -// httpclient_served_test is the embedded lane's (a retired tier's) served -// corpus, which may not be narrowed: the wrapper runs every top-level test in -// exactly one shard (TestPinnedShardWrapperSplit), the manifest may name only -// real tests (TestPinnedShardManifestsNameRealTests), and the lane's -// check_testcases.py fails a shard that ran none. +// httpclient_served_test, http_served_test and corpus_served_test are the +// embedded lane's (a retired tier's) served tiers, which may not be narrowed: +// the wrapper runs every top-level test in exactly one shard +// (TestPinnedShardWrapperSplit), the manifest may name only real tests +// (TestPinnedShardManifestNamesAreTests), and the lane's check_testcases.py +// fails a shard that ran none. corpus_served_test pins its corpus parts, one +// per shard (TestCorpusPartsArePinnedOnePerShard). var pinnedShardTargets = []struct { build, rule, manifest, pkg, tag string }{ {"cmd/bd/BUILD.bazel", "bd_dolt_server_test", "cmd/bd/dolt_server_pinned_shards.txt", "cmd/bd", "dolt-server-cmd"}, {"tests/regression/BUILD.bazel", "regression_test", "tests/regression/pinned_shards.txt", "tests/regression", "dolt-server-cmd"}, {"internal/httpclient/BUILD.bazel", "httpclient_served_test", "internal/httpclient/served_pinned_shards.txt", "internal/httpclient", "embedded"}, + {"backend/http/BUILD.bazel", "http_served_test", "backend/http/served_pinned_shards.txt", "backend/http", "embedded"}, + {"backend/http/corpus/BUILD.bazel", "corpus_served_test", "backend/http/corpus/served_pinned_shards.txt", "backend/http/corpus", "embedded"}, } var pinnedLineRe = regexp.MustCompile(`^([1-9][0-9]*) (Test[A-Za-z0-9_]*)$`) diff --git a/scripts/pinned_shards_sources_test.go b/scripts/pinned_shards_sources_test.go index fea3b49e0..1c122f8c7 100644 --- a/scripts/pinned_shards_sources_test.go +++ b/scripts/pinned_shards_sources_test.go @@ -1,10 +1,13 @@ package scripts_test import ( + "fmt" "os" "path/filepath" + "reflect" "regexp" "sort" + "strconv" "testing" ) @@ -46,3 +49,43 @@ func TestPinnedShardManifestNamesAreTests(t *testing.T) { }) } } + +var corpusPartsConst = regexp.MustCompile(`(?m)^const corpusParts = (\d+)$`) + +// corpus_served_test's manifest is not pin_shards.py's: it pins +// TestServedCorpusPart1..N, the corpus parts (backend/http/corpus +// corpusParts), to shards 1..N, one each, and pins nothing else, so every +// part runs alone in its own Bazel shard on its own seed. One shard is left +// for the binary's round-robin over the package's other (engine-free) tests. +func TestCorpusPartsArePinnedOnePerShard(t *testing.T) { + root := sourceRepoRoot(t) + const ( + build = "backend/http/corpus/BUILD.bazel" + manifest = "backend/http/corpus/served_pinned_shards.txt" + src = "backend/http/corpus/corpus_test.go" + ) + m := corpusPartsConst.FindStringSubmatch(readPolicyFile(t, root, src)) + if m == nil { + t.Fatalf("%s has no `const corpusParts = N`", src) + } + parts, _ := strconv.Atoi(m[1]) + if parts < 2 { + t.Fatalf("corpusParts = %d; a single part needs no pinned shards", parts) + } + pinned, maxShard := readPinnedShardManifest(t, root, manifest) + want := map[string]int{} + for k := 1; k <= parts; k++ { + want[fmt.Sprintf("TestServedCorpusPart%d", k)] = k + } + if !reflect.DeepEqual(pinned, want) { + t.Errorf("%s pins %v, want each of the %d corpus parts on its own shard: %v", manifest, pinned, parts, want) + } + rule := bazelRuleBlock(readPolicyFile(t, root, build), "corpus_served_test") + sm := shardCountPattern.FindStringSubmatch(rule) + if sm == nil { + t.Fatalf("%s:corpus_served_test has no shard_count", build) + } + if n, _ := strconv.Atoi(sm[1]); n != maxShard+1 { + t.Errorf("%s:corpus_served_test shard_count = %d, want %d (%d parts and one round-robin shard)", build, n, maxShard+1, maxShard) + } +} diff --git a/scripts/pr_risk_bazel_coverage_test.go b/scripts/pr_risk_bazel_coverage_test.go index 304022e15..ce74d5003 100644 --- a/scripts/pr_risk_bazel_coverage_test.go +++ b/scripts/pr_risk_bazel_coverage_test.go @@ -817,7 +817,7 @@ func TestBazelRetiredLanesCannotBeNarrowed(t *testing.T) { // go_test_pinned_shard.sh selects and skips by design (each test in // exactly one shard: TestPinnedShardWrapperSplit), and is reviewed // here for pinnedShardTargets only (the dolt-server-cmd targets and - // the embedded lane's httpclient_served_test, whose args are pinned + // the embedded lane's served targets, whose args are pinned // above); pinnedShardWrapperUsers fails if anything else runs // through it. exempt := filepath.ToSlash(rel) == pinnedShardWrapper @@ -904,13 +904,13 @@ func TestBazelRetiredLanesCannotBeNarrowed(t *testing.T) { map[string]string{"BEADS_HTTP_TEST_REQUIRED": "1", "BEADS_TEST_EMBEDDED_DOLT": "1"}, }, "//backend/http:http_served_test": { - []string{"$(rootpath :http_test)", "-test.v", "-test.count=1", "-test.timeout=19m"}, + []string{"$(rootpath :served_pinned_shards.txt)", "$(rootpath :http_test)", "-test.v", "-test.count=1", "-test.timeout=19m"}, map[string]string{"BEADS_HTTP_TEST_REQUIRED": "1", "BEADS_TEST_BD_BINARY": "$(rlocationpath //cmd/bd:bd_for_tests)", "BEADS_TEST_EMBEDDED_DOLT": "1"}, }, // The served command corpus (backend/http/corpus/README.md), with the // same switches as http_served_test. "//backend/http/corpus:corpus_served_test": { - []string{"$(rootpath :corpus_test)", "-test.v", "-test.count=1", "-test.timeout=19m"}, + []string{"$(rootpath :served_pinned_shards.txt)", "$(rootpath :corpus_test)", "-test.v", "-test.count=1", "-test.timeout=19m"}, map[string]string{"BEADS_HTTP_TEST_REQUIRED": "1", "BEADS_TEST_BD_BINARY": "$(rlocationpath //cmd/bd:bd_for_tests)", "BEADS_TEST_EMBEDDED_DOLT": "1"}, }, "//cmd/bd:bd_proxied_test": {