* test(bazel): shard http_served_test and corpus_served_test (embedded lane critical path) Both served targets ran as one shard each (235-597s) and were the embedded lane's longest action in most executing runs; every other embedded shard ends by ~90-155s. - http_served_test runs through go_test_pinned_shard.sh like httpclient_served_test: the 19 TestE2E_* cases >= 3s are pinned onto 6 shards (pin_shards.py from three lane runs' test.xml plus one local run, <= ~88 measured seconds each), the ~28 fast tests round-robin in a 7th. - corpus_served_test: TestServedCorpus becomes TestServedCorpusPart1..4. Shapes are dealt round robin in corpus order; each part starts its own server and seeds its own pair of workspaces. The shapes are independent: every check is served-vs-embedded parity after the same commands, each write owns its target ids, and order preservation keeps each part's reads ahead of its writes. TestCorpusPartsDealEveryShapeOnce checks the deal (every shape once, in order, no empty part); the manifest pins part k to shard k, with a 5th round-robin shard for the engine-free tests, and scripts TestCorpusPartsArePinnedOnePerShard ties corpusParts, the manifest and shard_count together. BEADS_CORPUS_REPORT now accumulates every part the process ran. - pinnedShardTargets and the retired-lane args pins list both targets; go_test_sources_test gets the two packages' manifests as data. Embedded goes from 172 to 182 test actions; the dolt-race/dolt-int union budgets do not include the embedded tag and are unchanged. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com> * test(corpus): guard part wiring; seed metadata on e2e-t-ifa (review of #7537) - testServedCorpus fails (before the engine gate, so in every lane) unless the test's name is TestServedCorpusPart<part>: a TestServedCorpusPart3 calling testServedCorpus(t, 2) would run part 2 twice and silently drop part 3's shapes. TestCorpusPartNameMustMatchItsPart covers the check and that servedCorpusParts lists Part1..N in order. Mutation (Part3 -> 2) fails //backend/http/corpus:corpus_test with "TestServedCorpusPart3 runs corpus part 2; only TestServedCorpusPart2 may". - e2e-t-ifa is seeded with metadata seeded=keep. The parts seed separately, so the two refused (exit 13) conditional updates no longer always follow gc-update-if-assignment's k=v; their verify show again holds that a refusal keeps existing metadata. Mutation (served verify output's seeded=keep rewritten) fails both stale shapes at $[0].metadata.seeded. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com> --------- Co-authored-by: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
77 lines
3 KiB
Go
77 lines
3 KiB
Go
package scripts_test
|
|
|
|
// Helpers for the pinned-shard tests, shared by scripts_test
|
|
// (pinned_shards_test.go) and go_test_sources_test
|
|
// (pinned_shards_sources_test.go).
|
|
|
|
import (
|
|
"regexp"
|
|
"strconv"
|
|
"strings"
|
|
"testing"
|
|
)
|
|
|
|
// pinnedShardWrapper is the sh_test src that pins slow tests to shards.
|
|
const pinnedShardWrapper = "tools/bazel/go_test_pinned_shard.sh"
|
|
|
|
// pinnedShardTargets are the sh_tests that run a go_test binary through
|
|
// tools/bazel/go_test_pinned_shard.sh: the BUILD file, the rule, its
|
|
// manifest, the directory holding the go_test's sources, and the rule's one
|
|
// lane tag.
|
|
//
|
|
// httpclient_served_test, http_served_test and corpus_served_test are the
|
|
// embedded lane's (a retired tier's) served tiers, which may not be narrowed:
|
|
// the wrapper runs every top-level test in exactly one shard
|
|
// (TestPinnedShardWrapperSplit), the manifest may name only real tests
|
|
// (TestPinnedShardManifestNamesAreTests), and the lane's check_testcases.py
|
|
// fails a shard that ran none. corpus_served_test pins its corpus parts, one
|
|
// per shard (TestCorpusPartsArePinnedOnePerShard).
|
|
var pinnedShardTargets = []struct {
|
|
build, rule, manifest, pkg, tag string
|
|
}{
|
|
{"cmd/bd/BUILD.bazel", "bd_dolt_server_test", "cmd/bd/dolt_server_pinned_shards.txt", "cmd/bd", "dolt-server-cmd"},
|
|
{"tests/regression/BUILD.bazel", "regression_test", "tests/regression/pinned_shards.txt", "tests/regression", "dolt-server-cmd"},
|
|
{"internal/httpclient/BUILD.bazel", "httpclient_served_test", "internal/httpclient/served_pinned_shards.txt", "internal/httpclient", "embedded"},
|
|
{"backend/http/BUILD.bazel", "http_served_test", "backend/http/served_pinned_shards.txt", "backend/http", "embedded"},
|
|
{"backend/http/corpus/BUILD.bazel", "corpus_served_test", "backend/http/corpus/served_pinned_shards.txt", "backend/http/corpus", "embedded"},
|
|
}
|
|
|
|
var pinnedLineRe = regexp.MustCompile(`^([1-9][0-9]*) (Test[A-Za-z0-9_]*)$`)
|
|
|
|
// readPinnedShardManifest parses a pinned-shard manifest the way
|
|
// go_test_pinned_shard.sh does, reporting malformed lines, duplicate names
|
|
// and empty shards, and returns the pinned names and the pinned shard count.
|
|
func readPinnedShardManifest(t *testing.T, root, manifest string) (map[string]int, int) {
|
|
t.Helper()
|
|
pinned := map[string]int{}
|
|
shards := map[int]int{}
|
|
maxShard := 0
|
|
for i, line := range strings.Split(readPolicyFile(t, root, manifest), "\n") {
|
|
line, _, _ = strings.Cut(line, "#")
|
|
line = strings.TrimSpace(line)
|
|
if line == "" {
|
|
continue
|
|
}
|
|
m := pinnedLineRe.FindStringSubmatch(line)
|
|
if m == nil {
|
|
t.Errorf("%s:%d: %q is not \"<shard> <TestName>\"", manifest, i+1, line)
|
|
continue
|
|
}
|
|
shard, _ := strconv.Atoi(m[1])
|
|
if _, dup := pinned[m[2]]; dup {
|
|
t.Errorf("%s: %s pinned twice", manifest, m[2])
|
|
}
|
|
pinned[m[2]] = shard
|
|
shards[shard]++
|
|
maxShard = max(maxShard, shard)
|
|
}
|
|
for s := 1; s <= maxShard; s++ {
|
|
if shards[s] == 0 {
|
|
t.Errorf("%s: pinned shard %d is empty", manifest, s)
|
|
}
|
|
}
|
|
if maxShard == 0 {
|
|
t.Errorf("%s pins nothing", manifest)
|
|
}
|
|
return pinned, maxShard
|
|
}
|