After #7440 //internal/httpclient:httpclient_served_test set the embedded lane (149-167s slowest shard in recent CI runs, 63-167s across PRs since #7416). Not a regression: its top-level tests are all serial and rules_go's round-robin put the long ones wherever its index split dropped them, so shard times swung with load and with which tests a PR added (whole-lane rbe-west run on main: 40-119s over 16 shards). The target now runs through tools/bazel/go_test_pinned_shard.sh, as bd_dolt_server_test and regression_test do: every top-level test measured at >= 3s (max over three whole-lane runs) is pinned onto 17 shards (served_pinned_shards.txt, tools/bazel/pin_shards.py), and the binary's own round-robin deals the other ~620 over the remaining 9. Every test still runs in exactly one shard (TestPinnedShardWrapperSplit), the manifest may name only real tests, and the lane's check_testcases.py fails an empty shard. No test changed; the heaviest single test (~52s, no subtests) stays whole. The pinned-wrapper policy listed only dolt-server-cmd targets, since the wrapper selects and skips; pinnedShardTargets now carries each reviewed target's lane tag, and httpclient_served_test is reviewed in as the embedded lane's (its args stay pinned in TestBazelRetiredLanesCannotBeNarrowed). rbe-west, `bazel test //... --config=embedded --nocache_test_results`: served shards max 119s -> 77s, median 73s -> 54s; lane critical path 138-153s -> 116s. Bead: ga-vnycm2.30 Co-authored-by: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
271 lines
9.7 KiB
Go
271 lines
9.7 KiB
Go
package scripts_test
|
|
|
|
import (
|
|
"fmt"
|
|
"os"
|
|
"os/exec"
|
|
"path/filepath"
|
|
"strconv"
|
|
"strings"
|
|
"testing"
|
|
)
|
|
|
|
// fakeTestBinary writes a stand-in for a go_test binary that prints its
|
|
// arguments and the sharding environment it was started with.
|
|
func fakeTestBinary(t *testing.T, dir string) {
|
|
t.Helper()
|
|
script := `#!/usr/bin/env bash
|
|
echo "args=$*"
|
|
echo "total=${TEST_TOTAL_SHARDS-unset} index=${TEST_SHARD_INDEX-unset} status=${TEST_SHARD_STATUS_FILE-unset} fromBazel=${GO_TEST_RUN_FROM_BAZEL-unset}"
|
|
`
|
|
if err := os.WriteFile(filepath.Join(dir, "bin"), []byte(script), 0o755); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
|
|
func runPinnedShard(t *testing.T, dir, manifest string, env ...string) (string, error) {
|
|
t.Helper()
|
|
wrapper := filepath.Join(sourceRepoRoot(t), "tools", "bazel", "go_test_pinned_shard.sh")
|
|
if err := os.WriteFile(filepath.Join(dir, "manifest.txt"), []byte(manifest), 0o644); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
cmd := exec.Command(requireHostTool(t, "bash"), wrapper, "manifest.txt", "bin", "-test.parallel=4")
|
|
cmd.Dir = dir
|
|
cmd.Env = append([]string{"PATH=" + os.Getenv("PATH")}, env...)
|
|
out, err := cmd.CombinedOutput()
|
|
return string(out), err
|
|
}
|
|
|
|
// The wrapper's split: pinned shards run exactly their tests with the
|
|
// binary's own sharding off; the rest are the binary's round-robin shards
|
|
// minus every pinned test; unsharded runs everything. Every test runs in
|
|
// exactly one shard because the skip set is the union of the run sets.
|
|
func TestPinnedShardWrapperSplit(t *testing.T) {
|
|
const manifest = `# comment
|
|
1 TestSlow # trailing comment
|
|
2 TestB
|
|
2 TestA
|
|
|
|
`
|
|
status := filepath.Join(t.TempDir(), "status")
|
|
cases := []struct {
|
|
name string
|
|
env []string
|
|
want []string
|
|
}{
|
|
{"unsharded", nil, []string{
|
|
"args=-test.parallel=4\n",
|
|
"total=unset index=unset status=unset fromBazel=1\n",
|
|
}},
|
|
{"pinned shard 1", []string{"TEST_TOTAL_SHARDS=5", "TEST_SHARD_INDEX=0", "TEST_SHARD_STATUS_FILE=" + status}, []string{
|
|
"args=-test.run=^(TestSlow)$ -test.parallel=4\n",
|
|
"total=unset index=unset status=unset fromBazel=1\n",
|
|
}},
|
|
{"pinned shard 2", []string{"TEST_TOTAL_SHARDS=5", "TEST_SHARD_INDEX=1", "TEST_SHARD_STATUS_FILE=" + status}, []string{
|
|
"args=-test.run=^(TestB|TestA)$ -test.parallel=4\n",
|
|
"total=unset index=unset",
|
|
}},
|
|
{"first round-robin shard", []string{"TEST_TOTAL_SHARDS=5", "TEST_SHARD_INDEX=2", "TEST_SHARD_STATUS_FILE=" + status}, []string{
|
|
"args=-test.skip=^(TestSlow|TestB|TestA)$ -test.parallel=4\n",
|
|
"total=3 index=0 status=" + status + " fromBazel=1\n",
|
|
}},
|
|
{"last round-robin shard", []string{"TEST_TOTAL_SHARDS=5", "TEST_SHARD_INDEX=4", "TEST_SHARD_STATUS_FILE=" + status}, []string{
|
|
"args=-test.skip=^(TestSlow|TestB|TestA)$ -test.parallel=4\n",
|
|
"total=3 index=2 ",
|
|
}},
|
|
}
|
|
for _, tc := range cases {
|
|
t.Run(tc.name, func(t *testing.T) {
|
|
dir := t.TempDir()
|
|
fakeTestBinary(t, dir)
|
|
_ = os.Remove(status)
|
|
out, err := runPinnedShard(t, dir, manifest, tc.env...)
|
|
if err != nil {
|
|
t.Fatalf("wrapper failed: %v\n%s", err, out)
|
|
}
|
|
for _, w := range tc.want {
|
|
if !strings.Contains(out, w) {
|
|
t.Errorf("output lacks %q:\n%s", w, out)
|
|
}
|
|
}
|
|
if len(tc.env) > 0 {
|
|
if _, err := os.Stat(status); err != nil {
|
|
t.Errorf("TEST_SHARD_STATUS_FILE not touched: %v", err)
|
|
}
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestPinnedShardWrapperRejectsBadManifests(t *testing.T) {
|
|
cases := map[string]string{
|
|
"gap in shard numbers": "1 TestA\n3 TestB\n",
|
|
"test pinned twice": "1 TestA\n2 TestA\n",
|
|
"not a test name": "1 BenchmarkA\n",
|
|
"regex in name": "1 TestA.*\n",
|
|
"extra field": "1 TestA TestB\n",
|
|
"shard zero": "0 TestA\n",
|
|
"no pinned shards": "# empty\n",
|
|
"no round-robin shards left": "1 TestA\n2 TestB\n3 TestC\n4 TestD\n5 TestE\n",
|
|
"more pinned than Bazel runs": "1 TestA\n2 TestB\n3 TestC\n4 TestD\n5 TestE\n6 TestF\n",
|
|
}
|
|
for name, manifest := range cases {
|
|
t.Run(name, func(t *testing.T) {
|
|
dir := t.TempDir()
|
|
fakeTestBinary(t, dir)
|
|
out, err := runPinnedShard(t, dir, manifest, "TEST_TOTAL_SHARDS=5", "TEST_SHARD_INDEX=0")
|
|
if err == nil {
|
|
t.Fatalf("wrapper accepted manifest %q:\n%s", manifest, out)
|
|
}
|
|
if strings.Contains(out, "args=") {
|
|
t.Errorf("wrapper ran the binary despite a bad manifest:\n%s", out)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
// The committed manifests are well formed and each target is wired to its
|
|
// manifest with round-robin shards left over. (That every pinned name is a
|
|
// test of the package: TestPinnedShardManifestNamesAreTests.)
|
|
func TestPinnedShardManifests(t *testing.T) {
|
|
root := sourceRepoRoot(t)
|
|
for _, tgt := range pinnedShardTargets {
|
|
t.Run(tgt.rule, func(t *testing.T) {
|
|
_, maxShard := readPinnedShardManifest(t, root, tgt.manifest)
|
|
rule := bazelRuleBlock(readPolicyFile(t, root, tgt.build), tgt.rule)
|
|
if rule == "" {
|
|
t.Fatalf("%s has no rule %s", tgt.build, tgt.rule)
|
|
}
|
|
base := filepath.Base(tgt.manifest)
|
|
for _, want := range []string{
|
|
`srcs = ["//tools/bazel:go_test_pinned_shard.sh"]`,
|
|
`"$(rootpath :` + base + `)",`,
|
|
} {
|
|
if !strings.Contains(rule, want) {
|
|
t.Errorf("%s lacks %s", tgt.rule, want)
|
|
}
|
|
}
|
|
m := shardCountPattern.FindStringSubmatch(rule)
|
|
if m == nil {
|
|
t.Fatalf("%s has no shard_count", tgt.rule)
|
|
}
|
|
if n, _ := strconv.Atoi(m[1]); n <= maxShard {
|
|
t.Errorf("%s shard_count = %d, but %s pins %d shards; round-robin shards must remain for the other tests", tgt.rule, n, tgt.manifest, maxShard)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
// pin_shards.py bin-packs every test at or above the threshold longest
|
|
// first, keeps the round-robin tests out, and takes the slowest of several
|
|
// runs.
|
|
func TestPinShardsGenerator(t *testing.T) {
|
|
root := sourceRepoRoot(t)
|
|
python := requireHostTool(t, "python3")
|
|
logs := t.TempDir()
|
|
write := func(run, shard, body string) {
|
|
dir := filepath.Join(logs, run, shard)
|
|
if err := os.MkdirAll(dir, 0o755); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
xml := `<testsuites><testsuite name="x">` + body + `</testsuite></testsuites>`
|
|
if err := os.WriteFile(filepath.Join(dir, "test.xml"), []byte(xml), 0o644); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
write("a", "shard_1_of_2", `<testcase name="TestA" time="50"/><testcase name="TestA/sub" time="49"/><testcase name="TestB" time="30"/>`)
|
|
write("a", "shard_2_of_2", `<testcase name="TestC" time="25"/><testcase name="TestD" time="20"/><testcase name="TestFast" time="1"/>`)
|
|
write("b", "shard_1_of_1", `<testcase name="TestD" time="28"/>`)
|
|
|
|
out := filepath.Join(t.TempDir(), "pinned.txt")
|
|
cmd := exec.Command(python, filepath.Join(root, "tools", "bazel", "pin_shards.py"),
|
|
"--shards", "2", "--min-seconds", "5", "--write", out, filepath.Join(logs, "a"), filepath.Join(logs, "b"))
|
|
if b, err := cmd.CombinedOutput(); err != nil {
|
|
t.Fatalf("pin_shards.py: %v\n%s", err, b)
|
|
}
|
|
b, err := os.ReadFile(out)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
var got []string
|
|
for _, line := range strings.Split(string(b), "\n") {
|
|
if line != "" && !strings.HasPrefix(line, "#") {
|
|
got = append(got, line)
|
|
}
|
|
}
|
|
// LPT: A(50)->1, B(30)->2, D(28, the slower run)->2, C(25)->1.
|
|
want := []string{"1 TestA", "1 TestC", "2 TestB", "2 TestD"}
|
|
if strings.Join(got, ";") != strings.Join(want, ";") {
|
|
t.Errorf("manifest = %q, want %q", got, want)
|
|
}
|
|
if !strings.Contains(string(b), "# 75 58\n") {
|
|
t.Errorf("manifest header lacks per-shard totals 75 58:\n%s", b)
|
|
}
|
|
|
|
cmd = exec.Command(python, filepath.Join(root, "tools", "bazel", "pin_shards.py"),
|
|
"--shards", "5", "--min-seconds", "5", filepath.Join(logs, "a"))
|
|
if b, err := cmd.CombinedOutput(); err == nil {
|
|
t.Errorf("pin_shards.py accepted 5 shards for 4 slow tests:\n%s", b)
|
|
}
|
|
}
|
|
|
|
// pinnedShardWrapperUsers returns an error for every BUILD rule that runs
|
|
// through go_test_pinned_shard.sh other than pinnedShardTargets, and for any
|
|
// of those not tagged exactly its pinnedShardTargets lane: the wrapper
|
|
// selects and skips tests, which the retired tiers' lanes forbid
|
|
// (TestBazelRetiredLanesCannotBeNarrowed) except for the reviewed targets
|
|
// listed there.
|
|
func pinnedShardWrapperUsers(t *testing.T, root string) []string {
|
|
t.Helper()
|
|
allowed := map[string]string{}
|
|
for _, tgt := range pinnedShardTargets {
|
|
allowed[tgt.build+":"+tgt.rule] = tgt.tag
|
|
}
|
|
const label = "//tools/bazel:go_test_pinned_shard.sh"
|
|
var errs []string
|
|
seen := 0
|
|
for _, f := range repoFiles(t, root, 300) {
|
|
if filepath.Base(f) != "BUILD.bazel" {
|
|
continue
|
|
}
|
|
build := readPolicyFile(t, root, f)
|
|
if !strings.Contains(build, "go_test_pinned_shard.sh") {
|
|
continue
|
|
}
|
|
for _, rule := range bazelTopRules(stripStarlarkComments(build)) {
|
|
if !strings.Contains(rule, "go_test_pinned_shard.sh") {
|
|
continue
|
|
}
|
|
name := ""
|
|
if m := bazelRuleNameRe.FindStringSubmatch(rule); m != nil {
|
|
name = m[1]
|
|
}
|
|
if !strings.Contains(rule, `srcs = ["`+label+`"]`) {
|
|
if f == "tools/bazel/BUILD.bazel" && strings.HasPrefix(rule, "exports_files(") {
|
|
continue
|
|
}
|
|
errs = append(errs, f+": rule "+name+" references go_test_pinned_shard.sh other than as its sh_test src "+label)
|
|
continue
|
|
}
|
|
seen++
|
|
tag, ok := allowed[f+":"+name]
|
|
if !ok {
|
|
errs = append(errs, f+": "+name+" runs through "+pinnedShardWrapper+"; add it to pinnedShardTargets after reviewing that every test still runs exactly once")
|
|
continue
|
|
}
|
|
if !strings.Contains(rule, `tags = ["`+tag+`"]`) {
|
|
errs = append(errs, f+": "+name+" runs through "+pinnedShardWrapper+" but is not tagged exactly "+tag)
|
|
}
|
|
}
|
|
}
|
|
if seen != len(pinnedShardTargets) {
|
|
errs = append(errs, fmt.Sprintf("found %d rules running through %s, want the %d pinnedShardTargets", seen, pinnedShardWrapper, len(pinnedShardTargets)))
|
|
}
|
|
return errs
|
|
}
|
|
|
|
func TestPinnedShardWrapperUsers(t *testing.T) {
|
|
for _, e := range pinnedShardWrapperUsers(t, sourceRepoRoot(t)) {
|
|
t.Error(e)
|
|
}
|
|
}
|