diff --git a/.github/workflows/bazel.yml b/.github/workflows/bazel.yml index 66322c9d5..5ca224450 100644 --- a/.github/workflows/bazel.yml +++ b/.github/workflows/bazel.yml @@ -2065,12 +2065,12 @@ jobs: run: echo "result=$JOB_STATUS" >> "$GITHUB_OUTPUT" # The cmd/bd Dolt-server tier, as --config=doltserver-cmd: the whole - # integration-tagged cmd/bd suite (//cmd/bd:bd_dolt_server_test, 16 - # shards) against a hermetic dolt sql-server, i.e. the Dolt-gated cmd/bd + # integration-tagged cmd/bd suite (//cmd/bd:bd_dolt_server_test, 32 + # shards, the slowest tests pinned to 16 of them) against a hermetic dolt sql-server, i.e. the Dolt-gated cmd/bd # tests (TestCLI_*, TestInit*, the store-backed suites) that every other # lane and legacy job skips with BEADS_TEST_SKIP=dolt or leaves out of its # manifest; plus the differential regression suite - # (//tests/regression:regression_test, 8 shards: the candidate + # (//tests/regression:regression_test, 14 shards: the candidate # bd_for_tests against the catalog-pinned v0.49.6 release, formerly # regression.yml). --config=doltserver-cmd has exactly # --config=integration's build flags, so it shares bazel-integration's @@ -2120,9 +2120,10 @@ jobs: RBE_FORK_PR: ${{ github.event.pull_request.number }} # Remote: a compile shared with bazel-integration plus the longest - # of 16 shards (~3 minutes measured; each shard pays ~30-40 s of - # TestMain and its own dolt sql-server; Go's -test.timeout=19m, - # Bazel's 1200s cap). Mode cache on a 4-CPU runner: the compile hits + # of 32 shards (~1 minute: tools/bazel/go_test_pinned_shard.sh pins + # the slowest tests to shards of their own; each shard pays a few + # seconds of TestMain and its own dolt sql-server; Go's + # -test.timeout=19m, Bazel's 1200s cap). Mode cache on a 4-CPU runner: the compile hits # the read-only cache, and a PR that changes cmd/bd or what it imports # runs every shard locally, four at a time (~2,200 test-seconds # measured unloaded, slower four-up: ~20-30 minutes); with the cache diff --git a/cmd/bd/BUILD.bazel b/cmd/bd/BUILD.bazel index f220e60a9..a38bc8bd7 100644 --- a/cmd/bd/BUILD.bazel +++ b/cmd/bd/BUILD.bazel @@ -1563,10 +1563,16 @@ sh_test( # instead of skipping every Dolt-gated test into a cached pass. These are # the cmd/bd tests no other lane runs: every other configuration of bd_test # passes BEADS_TEST_SKIP=dolt (prcore/ci, integration) or runs only its -# manifest (bd_embedded_test, bd_proxied_test). No -test.run/-test.skip: the Go binary shards -# itself over every top-level test (rules_go's TEST_SHARD_INDEX), so each -# runs in exactly one shard, and check_testcases.py fails a shard that ran -# none. It repeats bd_test's `env` (the wrapper cannot inherit it). +# manifest (bd_embedded_test, bd_proxied_test). Shards 1-16 each run the +# slowest tests dolt_server_pinned_shards.txt pins to them (duration-balanced +# by tools/bazel/pin_shards.py); shards 17-32 are the Go binary's own +# round-robin shards (rules_go's TEST_SHARD_INDEX) over every other top-level +# test, a test added since the manifest was written included +# (tools/bazel/go_test_pinned_shard.sh). So each test runs in exactly one +# shard, and check_testcases.py fails a shard that ran none. Round-robin +# alone left the slowest of 16 shards at 2-3x the median (TestSharedServerConcurrent, +# the TestInit* family). It repeats bd_test's `env` (the wrapper cannot +# inherit it). # # Tags: # dolt-server-cmd: the cmd/bd Dolt-server tier; starts hermetic dolt @@ -1580,11 +1586,15 @@ sh_test( # dump) before Bazel's 1200s action limit (eternal under the lane's # --test_timeout), which kills without one. timeout = "eternal", - srcs = ["//tools/bazel:go_test_variant.sh"], - args = ["$(rootpath :bd_test)"], + srcs = ["//tools/bazel:go_test_pinned_shard.sh"], + args = [ + "$(rootpath :dolt_server_pinned_shards.txt)", + "$(rootpath :bd_test)", + ], data = [ ":bd_for_tests", ":bd_test", + ":dolt_server_pinned_shards.txt", ":preflight_go_fixture", "@go_sdk//:bin/gofmt", ], @@ -1607,7 +1617,7 @@ sh_test( # against the one shared server. "GOMAXPROCS": "4", }, - shard_count = 16, + shard_count = 32, tags = ["dolt-server-cmd"], ) diff --git a/cmd/bd/cli_fast_test.go b/cmd/bd/cli_fast_test.go index 47c82ced1..096a64afd 100644 --- a/cmd/bd/cli_fast_test.go +++ b/cmd/bd/cli_fast_test.go @@ -15,6 +15,8 @@ import ( "testing" "time" + "github.com/steveyegge/beads/internal/beads" + "github.com/steveyegge/beads/internal/git" "github.com/steveyegge/beads/internal/testutil/bazeltest" "github.com/steveyegge/beads/internal/testutil/credentialcmd" ) @@ -151,6 +153,17 @@ func createTempDirWithCleanup(t *testing.T) string { // TestMain having installed it). func restoreEnvSnapshot(before []string) { restoreProcessEnv(before) } +// resetInProcessDiscovery drops the workspace discovery an earlier in-process +// execution cached (beads' RepoContext, git's repository context), so each +// one resolves its .beads from its own working directory as a bd process +// would. Without it TestCLI_Import, run as the binary's first test, ran +// `bd init` in a fresh directory against the .beads it had just exported +// from ("Found existing Dolt database"). +func resetInProcessDiscovery() { + beads.ResetCaches() + git.ResetCaches() +} + // runBDInProcess runs bd commands in-process by calling rootCmd.Execute // This is ~10-20x faster than exec.Command because it avoids process spawn overhead func runBDInProcess(t *testing.T, dir string, args ...string) string { @@ -171,6 +184,7 @@ func runBDInProcess(t *testing.T, dir string, args ...string) string { if err := os.Chdir(dir); err != nil { t.Fatalf("Failed to chdir to %s: %v", dir, err) } + resetInProcessDiscovery() // Capture stdout/stderr rOut, wOut, _ := os.Pipe() @@ -1235,6 +1249,7 @@ func runBDInProcessAllowError(t *testing.T, dir string, args ...string) (string, if err := os.Chdir(dir); err != nil { t.Fatalf("Failed to chdir to %s: %v", dir, err) } + resetInProcessDiscovery() rOut, wOut, _ := os.Pipe() rErr, wErr, _ := os.Pipe() diff --git a/cmd/bd/dolt_server_pinned_shards.txt b/cmd/bd/dolt_server_pinned_shards.txt new file mode 100644 index 000000000..d7ab46c5b --- /dev/null +++ b/cmd/bd/dolt_server_pinned_shards.txt @@ -0,0 +1,103 @@ +# Pinned shards for tools/bazel/go_test_pinned_shard.sh. Generated by +# tools/bazel/pin_shards.py; regenerate rather than edit (see its usage). +# +# Every top-level test measured at >= 5s, bin-packed +# longest-first onto these shards; every other test runs in the +# binary's own round-robin shards. Measured seconds per pinned shard: +# 59 60 59 62 59 59 58 57 57 57 62 62 57 57 57 62 +# Unpinned tests: 3322, 570s in total. +# +# +1 TestCLI_NoteAddMisplacedSyntax +1 TestSharedServerConcurrent +2 TestAcquireInitMutationGateWaitsPastGenericBudget +2 TestCLI_ReparentDottedIDExcludesOldParent +2 TestContributorRoutingDaemon +2 TestMultiIDUpdatePartialFailureJSONReportsFailedIDs +2 TestRoutingWithAllOperationalModes +3 TestADORoundTripLossyPriority +3 TestCLI_UpdateEphemeral +3 TestDetermineAutoRoutedRepoPath_DefaultFallback +3 TestDirectTrackerConformance +3 TestGetNotionConfigReadsDBPathWhenStoreUnset +4 TestBareParentWorktreeCoreCommandsWithoutRedirect +4 TestCLI_Blocked +4 TestDoltAutoCommit_Batch_DefersCommit +4 TestE2E_DoctorFixMetadataRoundtrip +4 TestE2E_MigrateDoltMetadata +4 TestInitDatabaseFlag +5 TestADORoundTripLabels +5 TestBackwardCompatContributorConfig +5 TestContributorRoutingSyncBranch +5 TestEmbeddedChangeDirOverridesInheritedBeadsDir +5 TestImportBlockedDuringMigrationFreeze +5 TestResolveCloseTargets +6 TestADORoundTripBlockedStatus +6 TestCLI_CommentsAddReservedWordRejectedThroughCobraDispatch +6 TestCLI_UpdateEphemeralMutualExclusion +6 TestDepRoutedTargetOpensReadOnly +6 TestE2E_AutoStartedRepoLocalServerPersistsAcrossCommands +6 TestInitCommand +7 TestAutoExportSkipsWhenExistingJSONLHasIDsMissingFromStore +7 TestCLI_CreateRejectsFlagLikeTitles +7 TestContributorRoutingTracer +7 TestGlobalDBIdentityCheck +7 TestMultiIDUpdateAllGoodPathUnchanged +7 TestSharedServerDeleteIfRevisionSingleWinner +8 TestCLI_Import_GlobalModeIgnoresProjectConfigYAMLPrefix +8 TestCLI_Update +8 TestCreateDepsAtomicity +8 TestDoctorPathArgMaintenanceSkippedByTargetFreeze +8 TestOpenRoutedReadStore_ContributorRouting +8 TestPrimeBinaryPortfolio +9 TestCLI_CommentAbbreviatedIDRejectedWithTruthfulMessage +9 TestContributorRoutingDirect +9 TestDoltRemoteAddPersistsSyncRemoteToSharedWorktreeConfig +9 TestInitNonInteractiveAlwaysSetsRole +9 TestServeAnswersFromARegisteredBackendStore +9 TestSharedServerVersionBumpDoesNotPromoteCursor +10 TestAutoMigrateSkippedDuringMigrationFreeze +10 TestCLI_Import +10 TestCLI_NoteListMisplacedSyntax +10 TestEmbeddedCreateDryRunCrossRepoDoesNotMigrateTarget +10 TestResolveAndGetIssueForMutationExactSurfacesRoutedAbbreviationRefusal +10 TestSharedServerIfRevisionSingleWinner +11 TestBEADS_DIRPrecedence +11 TestDoctorCheckHealthReportsVersionMismatchOnRepoLocalPort +11 TestDoctorPathArgBlockedByTargetFreeze +11 TestInitBackendFlag +11 TestInitContributorSetsBeadsRoleContributor +11 TestInit_WithBEADS_DIR_DoltBackend +11 TestTemplateSuite +12 TestCLI_Labels +12 TestCLI_NoteCommand +12 TestCLI_Show_SupersedesIsNotABlocker +12 TestCheckBeadGateCrossRigPrefixRoute +12 TestInitIfMissing +12 TestInitMetricsEmittedPerDoltMode +12 TestReadonlyShowPreservesLastTouched +13 TestCLI_DepRemove +13 TestCLI_UpdateAppendNotes +13 TestDepListCrossRigRouting +13 TestExplicitRepoOverride +13 TestInitIfMissingMatchingPrefixSkips +13 TestMultiIDUpdatePartialFailureExitsNonzero +14 TestCLI_CommentsAddShortID +14 TestCLI_DepTree +14 TestContributorPlanningGitIgnoresInheritedRouting +14 TestContributorRoutingLocalOnly +14 TestMaintainerRoutingUnaffected +14 TestWhereCommand_ReadsPrefixFromEmbeddedStore +15 TestADORoundTripCoreFields +15 TestBdMolPour_NeedsAndWaitsForSameTarget_SpawnerOpenBlocksWaiter +15 TestCLI_CreateDryRun +15 TestCLI_PriorityFormats +15 TestInitBEADS_DIR +15 TestInitPromptNonGitRepo +16 TestBdMolBond_ResolvesPlainFormulaName +16 TestCLI_UpdateLabels +16 TestCLI_UpdatePersistent +16 TestContributorRoutingNoDb +16 TestInitDoltMetadata +16 TestInitMetricsEnvConfigPrecedence +16 TestInitPromptExistingRole diff --git a/engdocs/CI_REQUIRED_CHECK_TOPOLOGY.md b/engdocs/CI_REQUIRED_CHECK_TOPOLOGY.md index 09ebaa67f..a9f34e510 100644 --- a/engdocs/CI_REQUIRED_CHECK_TOPOLOGY.md +++ b/engdocs/CI_REQUIRED_CHECK_TOPOLOGY.md @@ -1274,7 +1274,7 @@ lane runs `//internal/storage/dolt:dolt_server_full_test` (16 shards) and ### Regression Tests `regression.yml` is retired. The suite runs as -`//tests/regression:regression_test` (8 shards) in `bazel.yml`'s +`//tests/regression:regression_test` (14 shards) in `bazel.yml`'s `bazel-cmd-dolt` lane, which `pr.yml`'s `CI Gate / Required` requires (`BAZEL_CMD_DOLT_REQUIRED`), so regression is now part of the aggregate gate on PRs and merge groups instead of a separate, path-detected advisory diff --git a/engdocs/CI_TEST_SURFACE_AUDIT.md b/engdocs/CI_TEST_SURFACE_AUDIT.md index 202b0dd33..1f32213fd 100644 --- a/engdocs/CI_TEST_SURFACE_AUDIT.md +++ b/engdocs/CI_TEST_SURFACE_AUDIT.md @@ -199,7 +199,7 @@ Key jobs preserved by display name: | Workflow | Triggers | Main validation | |---|---|---| -| `bazel.yml` `bazel-cmd-dolt` (formerly `regression.yml`) | PR, merge group, push to `main` | `//tests/regression:regression_test` (8 shards, cached until its inputs change) against the catalog-pinned v0.49.6 baseline; local entrypoint stays `make test-regression`. | +| `bazel.yml` `bazel-cmd-dolt` (formerly `regression.yml`) | PR, merge group, push to `main` | `//tests/regression:regression_test` (14 shards, cached until its inputs change) against the catalog-pinned v0.49.6 baseline; local entrypoint stays `make test-regression`. | | `bazel.yml` `bazel-test` (formerly `cross-version-smoke.yml`) | PR, merge group, push to `main` | `//tests/upgrade_smoke:upgrade_smoke__test` for the newest 30 catalog-pinned releases, via `scripts/upgrade-smoke-test.sh` with `PREV_BIN`; cached until their inputs change. | | `bazel.yml` `bazel-test` (formerly `migration-test.yml`) | PR, merge group, push to `main` | `//tests/migration:historical_upgrade__test`, one target per reviewed release (14, v0.9.1 source-built offline) plus `legacy_bridge_test`, against catalog-pinned release binaries; cached until their inputs change. Local entrypoint stays `scripts/migration-test/run.sh`. | | `nightly.yml` | Daily schedule, manual | `go test -v -race -tags=integration,gms_pure_go -coverprofile=coverage.out -timeout=30m ./...` with `BEADS_TEST_SKIP=dolt`; checks coverage >= 30%. | diff --git a/engdocs/TESTING.md b/engdocs/TESTING.md index 9adaed89f..41efd4955 100644 --- a/engdocs/TESTING.md +++ b/engdocs/TESTING.md @@ -40,7 +40,7 @@ Each lane, as bazel.yml runs it (add your `--config=fork-cache` or | Release cross-compile (`bazel-release-cross`) | `./scripts/ci/bazel-release-cross-compile.sh` | Every `go_library`/`go_binary` for each row of `scripts/ci/release-targets.txt`, cgo off, with nogo. | | Integration (`bazel-integration`) | `bazel test //... --config=integration` | The `integration`-tagged build. Runs with the read-only cache too. | | Dolt server (`bazel-doltserver`) | `bazel test //... --config=doltserver` | Starts its own `dolt sql-server` from the pinned binary; no docker. | -| cmd/bd Dolt server (`bazel-cmd-dolt`) | `bazel test //cmd/bd:bd_dolt_server_test --config=doltserver-cmd` | 16 shards of the integration-tagged cmd/bd suite. | +| cmd/bd Dolt server (`bazel-cmd-dolt`) | `bazel test //cmd/bd:bd_dolt_server_test --config=doltserver-cmd` | 32 shards of the integration-tagged cmd/bd suite (the slowest tests pinned to 16 of them). | | Embedded Dolt (`bazel-embedded`) | `bazel test //... --config=embedded` | Remote execution only in CI; locally it is slow. | | Proxied server (`bazel-proxied`) | `bazel test //... --config=doltserver-proxied` | Remote execution only: 30 shards, each with a Dolt server. | | Server-Dolt storage (`bazel-server-storage`) | `bazel test //... --config=doltserver-integration` | Remote execution only. | @@ -193,8 +193,13 @@ advisory `bazel-cmd-dolt` job) runs the whole integration-tagged cmd/bd suite on the `local` backend: the Dolt-gated cmd/bd tests (`TestCLI_*`, the init and store-backed suites) that every other lane skips with `BEADS_TEST_SKIP=dolt` or leaves out of its manifest. It shares -`--config=integration`'s build, passes the binary no test selection (the Go -binary shards itself over every top-level test, 16 shards), and runs where +`--config=integration`'s build and runs it in 32 shards: 16 run the slowest +tests `cmd/bd/dolt_server_pinned_shards.txt` pins to them, and 16 are the Go +binary's own round-robin shards over every other top-level test +(`tools/bazel/go_test_pinned_shard.sh`; `//tests/regression:regression_test` +splits the same way). After a change that moves test durations, regenerate the +manifest from a run of the lane with `tools/bazel/pin_shards.py` (see its +usage); a stale manifest only costs balance, never coverage. It runs where the integration lane runs (remote, or with the read-only cache). pr.yml's gate requires it once `BAZEL_CMD_DOLT_REQUIRED` is `"true"`; pr.yml then also passes bazel.yml `cmd-dolt-required: true`, and the PR's diff --git a/scripts/BUILD.bazel b/scripts/BUILD.bazel index 92a065548..ef89f2709 100644 --- a/scripts/BUILD.bazel +++ b/scripts/BUILD.bazel @@ -53,6 +53,8 @@ go_test( "makefile_bazel_primary_test.go", "migration_corpus_bazel_test.go", "nogo_lint_policy_test.go", + "pinned_shards_helpers_test.go", + "pinned_shards_test.go", "pr_lanes_bazel_coverage_test.go", "pr_risk_bazel_coverage_test.go", "rbe_worker_env_test.go", @@ -131,6 +133,8 @@ go_test( "embedded_shard_count_test.go", "host_tools_test.go", "managed_local_lane_test.go", + "pinned_shards_helpers_test.go", + "pinned_shards_sources_test.go", "repo_helpers_test.go", "scripts_targets_test.go", "shard_manifest_test.go", @@ -157,6 +161,7 @@ go_test( "//cmd/bd:repo_other_files", "//internal/storage/dolt:repo_other_files", "//internal/storage/embeddeddolt:repo_other_files", + "//tests/regression:repo_other_files", "//tools/bazel:check_shard_coverage.py", "//tools/bazel:equivalence_allowlist.txt", ], diff --git a/scripts/ci_workflow_test.go b/scripts/ci_workflow_test.go index eb75ab320..174e8255e 100644 --- a/scripts/ci_workflow_test.go +++ b/scripts/ci_workflow_test.go @@ -5801,9 +5801,11 @@ func btoi(b bool) int { // whole integration-tagged cmd/bd suite against a hermetic dolt sql-server, // because every other lane, Bazel or go test, runs cmd/bd with // BEADS_TEST_SKIP=dolt or only a manifest's tests, so its Dolt-gated tests -// ran nowhere. Coverage is exact by construction: the target passes the -// binary no test selection, the Go binary shards itself over every -// top-level test, and check_testcases.py fails a shard that ran none. +// ran nowhere. Coverage is exact by construction: the target's only test +// selection is go_test_pinned_shard.sh's, whose round-robin shards skip +// exactly the tests its pinned shards run (TestPinnedShardWrapperSplit, +// TestPinnedShardManifests), and check_testcases.py fails a shard that ran +// none. func TestBazelCmdDoltJob(t *testing.T) { const config, target = "doltserver-cmd", "bd_dolt_server_test" workflow := readCIWorkflow(t, bazelWorkflowName) @@ -5881,8 +5883,8 @@ func TestBazelCmdDoltJob(t *testing.T) { t.Error(err) } for _, want := range []string{ - `srcs = ["//tools/bazel:go_test_variant.sh"]`, - `args = ["$(rootpath :bd_test)"]`, + `srcs = ["//tools/bazel:go_test_pinned_shard.sh"]`, + "args = [\n \"$(rootpath :dolt_server_pinned_shards.txt)\",\n \"$(rootpath :bd_test)\",\n ]", `tags = ["dolt-server-cmd"]`, `"GOMAXPROCS": "4"`, `"BEADS_TEST_GIT_IDENTITY": "1"`, @@ -5891,12 +5893,14 @@ func TestBazelCmdDoltJob(t *testing.T) { t.Errorf("%s lacks %s", target, want) } } - // 16 shards: each pays ~30-40 s of fixed cost (TestMain, its own dolt - // sql-server), so more shards mostly add farm actions; with the suite's - // long pole gone the slowest of 16 stays within bazel-integration's - // critical path. Change it together with bazel.yml's comment. - if m := regexp.MustCompile(`shard_count = (\d+)`).FindStringSubmatch(rule); m == nil || m[1] != "16" { - t.Errorf("%s shard_count = %v, want 16", target, m) + // 32 shards: 16 that run the slowest tests the manifest pins to them + // and 16 round-robin shards over the rest. Measured, a shard's fixed + // cost (TestMain, its own dolt sql-server) is a few seconds; the wall + // time is the serial sum of its tests, which round-robin alone left at + // 2-3x the median on the shard that drew the long tests. Change it + // together with bazel.yml's comment. + if m := regexp.MustCompile(`shard_count = (\d+)`).FindStringSubmatch(rule); m == nil || m[1] != "32" { + t.Errorf("%s shard_count = %v, want 32", target, m) } envKey := regexp.MustCompile(`"(BEADS_TEST_[A-Z_]+)": "\$\(rlocationpath [^)]+\)"`) for _, m := range envKey.FindAllStringSubmatch(bdTest, -1) { diff --git a/scripts/pinned_shards_helpers_test.go b/scripts/pinned_shards_helpers_test.go new file mode 100644 index 000000000..862fcd0b4 --- /dev/null +++ b/scripts/pinned_shards_helpers_test.go @@ -0,0 +1,65 @@ +package scripts_test + +// Helpers for the pinned-shard tests, shared by scripts_test +// (pinned_shards_test.go) and go_test_sources_test +// (pinned_shards_sources_test.go). + +import ( + "regexp" + "strconv" + "strings" + "testing" +) + +// pinnedShardWrapper is the sh_test src that pins slow tests to shards. +const pinnedShardWrapper = "tools/bazel/go_test_pinned_shard.sh" + +// pinnedShardTargets are the sh_tests that run a go_test binary through +// tools/bazel/go_test_pinned_shard.sh: the BUILD file, the rule, its +// manifest, and the directory holding the go_test's sources. +var pinnedShardTargets = []struct { + build, rule, manifest, pkg string +}{ + {"cmd/bd/BUILD.bazel", "bd_dolt_server_test", "cmd/bd/dolt_server_pinned_shards.txt", "cmd/bd"}, + {"tests/regression/BUILD.bazel", "regression_test", "tests/regression/pinned_shards.txt", "tests/regression"}, +} + +var pinnedLineRe = regexp.MustCompile(`^([1-9][0-9]*) (Test[A-Za-z0-9_]*)$`) + +// readPinnedShardManifest parses a pinned-shard manifest the way +// go_test_pinned_shard.sh does, reporting malformed lines, duplicate names +// and empty shards, and returns the pinned names and the pinned shard count. +func readPinnedShardManifest(t *testing.T, root, manifest string) (map[string]int, int) { + t.Helper() + pinned := map[string]int{} + shards := map[int]int{} + maxShard := 0 + for i, line := range strings.Split(readPolicyFile(t, root, manifest), "\n") { + line, _, _ = strings.Cut(line, "#") + line = strings.TrimSpace(line) + if line == "" { + continue + } + m := pinnedLineRe.FindStringSubmatch(line) + if m == nil { + t.Errorf("%s:%d: %q is not \" \"", manifest, i+1, line) + continue + } + shard, _ := strconv.Atoi(m[1]) + if _, dup := pinned[m[2]]; dup { + t.Errorf("%s: %s pinned twice", manifest, m[2]) + } + pinned[m[2]] = shard + shards[shard]++ + maxShard = max(maxShard, shard) + } + for s := 1; s <= maxShard; s++ { + if shards[s] == 0 { + t.Errorf("%s: pinned shard %d is empty", manifest, s) + } + } + if maxShard == 0 { + t.Errorf("%s pins nothing", manifest) + } + return pinned, maxShard +} diff --git a/scripts/pinned_shards_sources_test.go b/scripts/pinned_shards_sources_test.go new file mode 100644 index 000000000..fea3b49e0 --- /dev/null +++ b/scripts/pinned_shards_sources_test.go @@ -0,0 +1,48 @@ +package scripts_test + +import ( + "os" + "path/filepath" + "regexp" + "sort" + "testing" +) + +var goTopLevelTestRe = regexp.MustCompile(`(?m)^func (Test[A-Za-z0-9_]*)\(t \*testing\.T\) \{`) + +// Every name a pinned-shard manifest pins is a top-level test of the +// target's package: a stale name runs nowhere, and a renamed test silently +// falls back to the round-robin shards (regenerate with +// tools/bazel/pin_shards.py). +func TestPinnedShardManifestNamesAreTests(t *testing.T) { + root := sourceRepoRoot(t) + for _, tgt := range pinnedShardTargets { + t.Run(tgt.rule, func(t *testing.T) { + tests := map[string]bool{} + srcs, err := filepath.Glob(filepath.Join(root, tgt.pkg, "*_test.go")) + if err != nil || len(srcs) == 0 { + t.Fatalf("no *_test.go in %s: %v", tgt.pkg, err) + } + for _, src := range srcs { + b, err := os.ReadFile(src) + if err != nil { + t.Fatal(err) + } + for _, m := range goTopLevelTestRe.FindAllStringSubmatch(string(b), -1) { + tests[m[1]] = true + } + } + pinned, _ := readPinnedShardManifest(t, root, tgt.manifest) + var stale []string + for name := range pinned { + if !tests[name] { + stale = append(stale, name) + } + } + sort.Strings(stale) + for _, name := range stale { + t.Errorf("%s pins %s, which is not a top-level test in %s/*_test.go; regenerate it (tools/bazel/pin_shards.py)", tgt.manifest, name, tgt.pkg) + } + }) + } +} diff --git a/scripts/pinned_shards_test.go b/scripts/pinned_shards_test.go new file mode 100644 index 000000000..ff95e9f3f --- /dev/null +++ b/scripts/pinned_shards_test.go @@ -0,0 +1,268 @@ +package scripts_test + +import ( + "fmt" + "os" + "os/exec" + "path/filepath" + "strconv" + "strings" + "testing" +) + +// fakeTestBinary writes a stand-in for a go_test binary that prints its +// arguments and the sharding environment it was started with. +func fakeTestBinary(t *testing.T, dir string) { + t.Helper() + script := `#!/usr/bin/env bash +echo "args=$*" +echo "total=${TEST_TOTAL_SHARDS-unset} index=${TEST_SHARD_INDEX-unset} status=${TEST_SHARD_STATUS_FILE-unset} fromBazel=${GO_TEST_RUN_FROM_BAZEL-unset}" +` + if err := os.WriteFile(filepath.Join(dir, "bin"), []byte(script), 0o755); err != nil { + t.Fatal(err) + } +} + +func runPinnedShard(t *testing.T, dir, manifest string, env ...string) (string, error) { + t.Helper() + wrapper := filepath.Join(sourceRepoRoot(t), "tools", "bazel", "go_test_pinned_shard.sh") + if err := os.WriteFile(filepath.Join(dir, "manifest.txt"), []byte(manifest), 0o644); err != nil { + t.Fatal(err) + } + cmd := exec.Command(requireHostTool(t, "bash"), wrapper, "manifest.txt", "bin", "-test.parallel=4") + cmd.Dir = dir + cmd.Env = append([]string{"PATH=" + os.Getenv("PATH")}, env...) + out, err := cmd.CombinedOutput() + return string(out), err +} + +// The wrapper's split: pinned shards run exactly their tests with the +// binary's own sharding off; the rest are the binary's round-robin shards +// minus every pinned test; unsharded runs everything. Every test runs in +// exactly one shard because the skip set is the union of the run sets. +func TestPinnedShardWrapperSplit(t *testing.T) { + const manifest = `# comment +1 TestSlow # trailing comment +2 TestB +2 TestA + +` + status := filepath.Join(t.TempDir(), "status") + cases := []struct { + name string + env []string + want []string + }{ + {"unsharded", nil, []string{ + "args=-test.parallel=4\n", + "total=unset index=unset status=unset fromBazel=1\n", + }}, + {"pinned shard 1", []string{"TEST_TOTAL_SHARDS=5", "TEST_SHARD_INDEX=0", "TEST_SHARD_STATUS_FILE=" + status}, []string{ + "args=-test.run=^(TestSlow)$ -test.parallel=4\n", + "total=unset index=unset status=unset fromBazel=1\n", + }}, + {"pinned shard 2", []string{"TEST_TOTAL_SHARDS=5", "TEST_SHARD_INDEX=1", "TEST_SHARD_STATUS_FILE=" + status}, []string{ + "args=-test.run=^(TestB|TestA)$ -test.parallel=4\n", + "total=unset index=unset", + }}, + {"first round-robin shard", []string{"TEST_TOTAL_SHARDS=5", "TEST_SHARD_INDEX=2", "TEST_SHARD_STATUS_FILE=" + status}, []string{ + "args=-test.skip=^(TestSlow|TestB|TestA)$ -test.parallel=4\n", + "total=3 index=0 status=" + status + " fromBazel=1\n", + }}, + {"last round-robin shard", []string{"TEST_TOTAL_SHARDS=5", "TEST_SHARD_INDEX=4", "TEST_SHARD_STATUS_FILE=" + status}, []string{ + "args=-test.skip=^(TestSlow|TestB|TestA)$ -test.parallel=4\n", + "total=3 index=2 ", + }}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + dir := t.TempDir() + fakeTestBinary(t, dir) + _ = os.Remove(status) + out, err := runPinnedShard(t, dir, manifest, tc.env...) + if err != nil { + t.Fatalf("wrapper failed: %v\n%s", err, out) + } + for _, w := range tc.want { + if !strings.Contains(out, w) { + t.Errorf("output lacks %q:\n%s", w, out) + } + } + if len(tc.env) > 0 { + if _, err := os.Stat(status); err != nil { + t.Errorf("TEST_SHARD_STATUS_FILE not touched: %v", err) + } + } + }) + } +} + +func TestPinnedShardWrapperRejectsBadManifests(t *testing.T) { + cases := map[string]string{ + "gap in shard numbers": "1 TestA\n3 TestB\n", + "test pinned twice": "1 TestA\n2 TestA\n", + "not a test name": "1 BenchmarkA\n", + "regex in name": "1 TestA.*\n", + "extra field": "1 TestA TestB\n", + "shard zero": "0 TestA\n", + "no pinned shards": "# empty\n", + "no round-robin shards left": "1 TestA\n2 TestB\n3 TestC\n4 TestD\n5 TestE\n", + "more pinned than Bazel runs": "1 TestA\n2 TestB\n3 TestC\n4 TestD\n5 TestE\n6 TestF\n", + } + for name, manifest := range cases { + t.Run(name, func(t *testing.T) { + dir := t.TempDir() + fakeTestBinary(t, dir) + out, err := runPinnedShard(t, dir, manifest, "TEST_TOTAL_SHARDS=5", "TEST_SHARD_INDEX=0") + if err == nil { + t.Fatalf("wrapper accepted manifest %q:\n%s", manifest, out) + } + if strings.Contains(out, "args=") { + t.Errorf("wrapper ran the binary despite a bad manifest:\n%s", out) + } + }) + } +} + +// The committed manifests are well formed and each target is wired to its +// manifest with round-robin shards left over. (That every pinned name is a +// test of the package: TestPinnedShardManifestNamesAreTests.) +func TestPinnedShardManifests(t *testing.T) { + root := sourceRepoRoot(t) + for _, tgt := range pinnedShardTargets { + t.Run(tgt.rule, func(t *testing.T) { + _, maxShard := readPinnedShardManifest(t, root, tgt.manifest) + rule := bazelRuleBlock(readPolicyFile(t, root, tgt.build), tgt.rule) + if rule == "" { + t.Fatalf("%s has no rule %s", tgt.build, tgt.rule) + } + base := filepath.Base(tgt.manifest) + for _, want := range []string{ + `srcs = ["//tools/bazel:go_test_pinned_shard.sh"]`, + `"$(rootpath :` + base + `)",`, + } { + if !strings.Contains(rule, want) { + t.Errorf("%s lacks %s", tgt.rule, want) + } + } + m := shardCountPattern.FindStringSubmatch(rule) + if m == nil { + t.Fatalf("%s has no shard_count", tgt.rule) + } + if n, _ := strconv.Atoi(m[1]); n <= maxShard { + t.Errorf("%s shard_count = %d, but %s pins %d shards; round-robin shards must remain for the other tests", tgt.rule, n, tgt.manifest, maxShard) + } + }) + } +} + +// pin_shards.py bin-packs every test at or above the threshold longest +// first, keeps the round-robin tests out, and takes the slowest of several +// runs. +func TestPinShardsGenerator(t *testing.T) { + root := sourceRepoRoot(t) + python := requireHostTool(t, "python3") + logs := t.TempDir() + write := func(run, shard, body string) { + dir := filepath.Join(logs, run, shard) + if err := os.MkdirAll(dir, 0o755); err != nil { + t.Fatal(err) + } + xml := `` + body + `` + if err := os.WriteFile(filepath.Join(dir, "test.xml"), []byte(xml), 0o644); err != nil { + t.Fatal(err) + } + } + write("a", "shard_1_of_2", ``) + write("a", "shard_2_of_2", ``) + write("b", "shard_1_of_1", ``) + + out := filepath.Join(t.TempDir(), "pinned.txt") + cmd := exec.Command(python, filepath.Join(root, "tools", "bazel", "pin_shards.py"), + "--shards", "2", "--min-seconds", "5", "--write", out, filepath.Join(logs, "a"), filepath.Join(logs, "b")) + if b, err := cmd.CombinedOutput(); err != nil { + t.Fatalf("pin_shards.py: %v\n%s", err, b) + } + b, err := os.ReadFile(out) + if err != nil { + t.Fatal(err) + } + var got []string + for _, line := range strings.Split(string(b), "\n") { + if line != "" && !strings.HasPrefix(line, "#") { + got = append(got, line) + } + } + // LPT: A(50)->1, B(30)->2, D(28, the slower run)->2, C(25)->1. + want := []string{"1 TestA", "1 TestC", "2 TestB", "2 TestD"} + if strings.Join(got, ";") != strings.Join(want, ";") { + t.Errorf("manifest = %q, want %q", got, want) + } + if !strings.Contains(string(b), "# 75 58\n") { + t.Errorf("manifest header lacks per-shard totals 75 58:\n%s", b) + } + + cmd = exec.Command(python, filepath.Join(root, "tools", "bazel", "pin_shards.py"), + "--shards", "5", "--min-seconds", "5", filepath.Join(logs, "a")) + if b, err := cmd.CombinedOutput(); err == nil { + t.Errorf("pin_shards.py accepted 5 shards for 4 slow tests:\n%s", b) + } +} + +// pinnedShardWrapperUsers returns an error for every BUILD rule that runs +// through go_test_pinned_shard.sh other than pinnedShardTargets, and for any +// of those tagged for another lane than the cmd/bd Dolt-server tier: the +// wrapper selects and skips tests, which the retired tiers' lanes forbid +// (TestBazelRetiredLanesCannotBeNarrowed). +func pinnedShardWrapperUsers(t *testing.T, root string) []string { + t.Helper() + allowed := map[string]bool{} + for _, tgt := range pinnedShardTargets { + allowed[tgt.build+":"+tgt.rule] = true + } + const label = "//tools/bazel:go_test_pinned_shard.sh" + var errs []string + seen := 0 + for _, f := range repoFiles(t, root, 300) { + if filepath.Base(f) != "BUILD.bazel" { + continue + } + build := readPolicyFile(t, root, f) + if !strings.Contains(build, "go_test_pinned_shard.sh") { + continue + } + for _, rule := range bazelTopRules(stripStarlarkComments(build)) { + if !strings.Contains(rule, "go_test_pinned_shard.sh") { + continue + } + name := "" + if m := bazelRuleNameRe.FindStringSubmatch(rule); m != nil { + name = m[1] + } + if !strings.Contains(rule, `srcs = ["`+label+`"]`) { + if f == "tools/bazel/BUILD.bazel" && strings.HasPrefix(rule, "exports_files(") { + continue + } + errs = append(errs, f+": rule "+name+" references go_test_pinned_shard.sh other than as its sh_test src "+label) + continue + } + seen++ + if !allowed[f+":"+name] { + errs = append(errs, f+": "+name+" runs through "+pinnedShardWrapper+"; add it to pinnedShardTargets after reviewing that it is not a retired tier's lane") + } + if !strings.Contains(rule, `tags = ["dolt-server-cmd"]`) { + errs = append(errs, f+": "+name+" runs through "+pinnedShardWrapper+" but is not tagged exactly dolt-server-cmd") + } + } + } + if seen != len(pinnedShardTargets) { + errs = append(errs, fmt.Sprintf("found %d rules running through %s, want the %d pinnedShardTargets", seen, pinnedShardWrapper, len(pinnedShardTargets))) + } + return errs +} + +func TestPinnedShardWrapperUsers(t *testing.T) { + for _, e := range pinnedShardWrapperUsers(t, sourceRepoRoot(t)) { + t.Error(e) + } +} diff --git a/scripts/pr_risk_bazel_coverage_test.go b/scripts/pr_risk_bazel_coverage_test.go index b058e806d..a5853126f 100644 --- a/scripts/pr_risk_bazel_coverage_test.go +++ b/scripts/pr_risk_bazel_coverage_test.go @@ -759,12 +759,23 @@ func TestBazelRetiredLanesCannotBeNarrowed(t *testing.T) { if err != nil { t.Fatal(err) } + // go_test_pinned_shard.sh selects and skips by design (each test in + // exactly one shard: TestPinnedShardWrapperSplit), and is reviewed + // here for the dolt-server-cmd targets only, which are none of the + // retired tiers' lanes; pinnedShardWrapperUsers fails if anything + // else runs through it. + exempt := filepath.ToSlash(rel) == pinnedShardWrapper + if exempt { + for _, e := range pinnedShardWrapperUsers(t, root) { + t.Error(e) + } + } for i, line := range strings.Split(string(data), "\n") { code := strings.TrimSpace(line) if strings.HasPrefix(code, "#") { continue } - if scriptNarrow.MatchString(code) { + if scriptNarrow.MatchString(code) && !exempt { t.Errorf("%s:%d %q can select, skip or re-run the retired tiers' lanes' tests", rel, i+1, code) } for _, m := range regexp.MustCompile(`--config=([A-Za-z0-9_-]+)`).FindAllStringSubmatch(code, -1) { diff --git a/tests/regression/BUILD.bazel b/tests/regression/BUILD.bazel index b0c7f12ca..2c1f527f2 100644 --- a/tests/regression/BUILD.bazel +++ b/tests/regression/BUILD.bazel @@ -1,4 +1,5 @@ load("@rules_go//go:def.bzl", "go_test") +load("@rules_shell//shell:sh_test.bzl", "sh_test") load("//tools/bazel:bd_releases.bzl", "LINUX_AMD64") # The differential regression suite (formerly regression.yml, which ran @@ -10,26 +11,50 @@ load("//tools/bazel:bd_releases.bzl", "LINUX_AMD64") # `go build -tags gms_pure_go ./cmd/bd`. Both bd's talk to a hermetic dolt # sql-server TestMain starts from the pinned dolt (BEADS_TEST_DOLT_SERVER=local) # where the job pulled the dolt-sql-server image; a server that cannot start -# fails the target (BEADS_TEST_REQUIRE_DOLT_CONTAINER=1). The Go binary shards -# itself over the top-level tests, so each runs in exactly one shard. +# fails the target (BEADS_TEST_REQUIRE_DOLT_CONTAINER=1). # +# The go_test is only the binary (manual: never run on its own); +# :regression_test runs it in 14 shards. Shards 1-10 each run the scenarios +# pinned_shards.txt pins to them (duration-balanced by +# tools/bazel/pin_shards.py); shards 11-14 are the Go binary's own round-robin +# shards over every other top-level test, a test added since the manifest was +# written included (tools/bazel/go_test_pinned_shard.sh), so each runs in +# exactly one shard. Round-robin alone over 8 shards left the slowest at twice +# the median (TestListAllOverridesLimitWithLabel, TestDepTypeExportFidelity). +go_test( + name = "regression_suite", + srcs = [ + "regression_test.go", + "scenarios_test.go", + ], + tags = ["manual"], + target_compatible_with = LINUX_AMD64, + deps = [ + "//internal/testutil", + "//internal/testutil/bazeltest", + ], +) + # Tags: # dolt-server-cmd: starts a hermetic dolt sql-server and needs the # integration build tag; selected by --config=doltserver-cmd (bazel.yml's # bazel-cmd-dolt lane) and excluded from every other lane, whose # configurations either drop both test files or set BEADS_TEST_SKIP=dolt. -go_test( +sh_test( name = "regression_test", size = "large", # --config=doltserver-cmd's -test.timeout=19m fires (with a goroutine # dump) before Bazel's 1200s action limit. timeout = "eternal", - srcs = [ - "regression_test.go", - "scenarios_test.go", + srcs = ["//tools/bazel:go_test_pinned_shard.sh"], + args = [ + "$(rootpath :pinned_shards.txt)", + "$(rootpath :regression_suite)", ], data = [ "BASELINE_VERSION", + "pinned_shards.txt", + ":regression_suite", "//cmd/bd:bd_for_tests", "@bd_releases//:v0.49.6", ], @@ -39,13 +64,9 @@ go_test( "BEADS_TEST_DOLT_SERVER": "local", "BEADS_TEST_REQUIRE_DOLT_CONTAINER": "1", }, - shard_count = 8, + shard_count = 14, tags = ["dolt-server-cmd"], target_compatible_with = LINUX_AMD64, - deps = [ - "//internal/testutil", - "//internal/testutil/bazeltest", - ], ) # --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- diff --git a/tests/regression/pinned_shards.txt b/tests/regression/pinned_shards.txt new file mode 100644 index 000000000..0734b701a --- /dev/null +++ b/tests/regression/pinned_shards.txt @@ -0,0 +1,55 @@ +# Pinned shards for tools/bazel/go_test_pinned_shard.sh. Generated by +# tools/bazel/pin_shards.py; regenerate rather than edit (see its usage). +# +# Every top-level test measured at >= 5s, bin-packed +# longest-first onto these shards; every other test runs in the +# binary's own round-robin shards. Measured seconds per pinned shard: +# 50 43 43 42 42 40 40 40 39 44 +# Unpinned tests: 40, 128s in total. +# +# +1 TestListAllOverridesLimitWithLabel +2 TestDepRemovalUnblocks +2 TestDepTypeExportFidelity +2 TestReparenting +3 TestCloseGuardNonBlocksDepTypes +3 TestDiamondDependency +3 TestListResolvedBlockerAnnotation +3 TestListStatusFilterParity +3 TestSpecialCharactersInFields +4 TestCustomTypeRoundTrip +4 TestDeleteWithDependencies +4 TestEpicWithChildrenStatus +4 TestMultipleDepTypesOnSameIssue +4 TestReopenPreservesDeps +5 TestBlocksDepCloseUnblocks +5 TestExportImportEpicTree +5 TestNonBlocksDepCloseSemantics +5 TestRemoveParent +5 TestUpdateDoesNotClobberRelationalData +6 TestMultipleCommentsRoundTrip +6 TestMultipleLabelsStressRoundTrip +6 TestParentChildDepExportBothDirections +6 TestParentChildDependency +6 TestReadySemantics +7 TestBlockedCommandParity +7 TestCountCommandParity +7 TestDeleteCascade +7 TestDepTreeOutput +7 TestTransitiveBlockingChain +8 TestEpicAllChildrenClosedNotBlocked +8 TestLargeExportImportRoundTrip +8 TestListByLabelFilter +8 TestRapidCommentAddStability +8 TestRapidLabelAddRemoveStability +9 TestBlockedEpicChildrenNotReady +9 TestChildrenCommand +9 TestCommentOnClosedIssue +9 TestDependenciesRoundTrip +9 TestQueryDSLParity +10 TestBasicLifecycle +10 TestFilteredExport +10 TestListCountParity +10 TestRapidDepAddRemoveStability +10 TestSearchByTitleSubstring +10 TestShowJSONFieldCompleteness diff --git a/tools/bazel/BUILD.bazel b/tools/bazel/BUILD.bazel index 6b17dde5c..fcddec75c 100644 --- a/tools/bazel/BUILD.bazel +++ b/tools/bazel/BUILD.bazel @@ -38,6 +38,14 @@ exports_files( visibility = ["//visibility:public"], ) +# sh_test src that runs one Bazel shard of an existing go_test binary with +# its slowest tests pinned to shards of their own by a manifest (see the +# script header and pin_shards.py), e.g. //cmd/bd:bd_dolt_server_test. +exports_files( + ["go_test_pinned_shard.sh"], + visibility = ["//visibility:public"], +) + # sh_test src that runs an existing go_test binary with no network but # loopback (see the script header), e.g. //cmd/bd:bd_managed_local_test. exports_files( diff --git a/tools/bazel/go_test_pinned_shard.sh b/tools/bazel/go_test_pinned_shard.sh new file mode 100755 index 000000000..d1c2b5d7d --- /dev/null +++ b/tools/bazel/go_test_pinned_shard.sh @@ -0,0 +1,90 @@ +#!/usr/bin/env bash +# Runs one Bazel shard of an existing go_test binary with the suite's slowest +# top-level tests pinned to shards of their own, so no shard's wall time is +# set by where the binary's round-robin split happens to drop its long tests. +# Like go_test_variant.sh it reuses the go_test's binary instead of compiling +# the package again. +# +# Usage: go_test_pinned_shard.sh [binary args...] +# , : runfiles paths ($(rootpath ...)), relative to the +# workspace runfiles directory, which is the test's working directory. +# The remaining arguments (and any --test_arg) go to the binary unchanged, +# after this wrapper's own -test.run/-test.skip. +# +# The manifest (tools/bazel/pin_shards.py writes it from measured test.xml +# durations) has one " " line per pinned test, pinned +# shards numbered 1..P with none empty; '#' starts a comment. With Bazel's +# shard_count N > P: +# Bazel shards 0..P-1 run exactly their pinned tests (-test.run), with the +# binary's own sharding switched off; +# Bazel shards P..N-1 are the binary's own round-robin shards 0..N-P-1 over +# every top-level test, minus every pinned one +# (-test.skip). +# So every test runs in exactly one shard: a pinned test only in its pinned +# shard, any other test (one added since the manifest was written included) +# where the binary's round-robin puts it. A manifest name with no test runs +# nothing (scripts/pinned_shards_test.go rejects one). Unsharded (no +# TEST_TOTAL_SHARDS, e.g. --test_sharding_strategy=disabled) the binary runs +# every test. +# +# rules_go sets GO_TEST_RUN_FROM_BAZEL=1 only for the go_test itself; this +# exports it, as go_test_variant.sh does, so the binary still changes into its +# package directory. The go_test's `env` is not inherited; the target repeats +# it. +set -euo pipefail +manifest="$1" +bin="$2" +shift 2 +export GO_TEST_RUN_FROM_BAZEL=1 + +total="${TEST_TOTAL_SHARDS:-1}" +if [[ "$total" -le 1 ]]; then + exec "./$bin" "$@" +fi +index="${TEST_SHARD_INDEX:?TEST_TOTAL_SHARDS is set but TEST_SHARD_INDEX is not}" + +declare -A shard_of=() +declare -a pinned_by_shard=() +pinned_shards=0 +all="" +lineno=0 +while IFS= read -r line || [[ -n "$line" ]]; do + lineno=$((lineno + 1)) + line="${line%%#*}" + read -r shard name extra <<<"$line" || true + [[ -z "${shard:-}" ]] && continue + if ! [[ "$shard" =~ ^[1-9][0-9]*$ && "${name:-}" =~ ^Test[A-Za-z0-9_]*$ && -z "${extra:-}" ]]; then + echo "$manifest:$lineno: want \" \", got: $line" >&2 + exit 1 + fi + if [[ -n "${shard_of[$name]:-}" ]]; then + echo "$manifest:$lineno: $name is pinned twice" >&2 + exit 1 + fi + shard_of[$name]="$shard" + pinned_by_shard[shard]="${pinned_by_shard[shard]:+${pinned_by_shard[shard]}|}$name" + all="${all:+$all|}$name" + ((shard > pinned_shards)) && pinned_shards="$shard" +done <"$manifest" + +for ((s = 1; s <= pinned_shards; s++)); do + if [[ -z "${pinned_by_shard[s]:-}" ]]; then + echo "$manifest: pinned shard $s has no tests (shards must be numbered 1..P with none empty)" >&2 + exit 1 + fi +done +if ((pinned_shards == 0 || total <= pinned_shards)); then + echo "$manifest pins $pinned_shards shards; the target's shard_count ($total) must be larger" >&2 + exit 1 +fi + +if [[ -n "${TEST_SHARD_STATUS_FILE:-}" ]]; then + touch "$TEST_SHARD_STATUS_FILE" +fi +if ((index < pinned_shards)); then + unset TEST_TOTAL_SHARDS TEST_SHARD_INDEX TEST_SHARD_STATUS_FILE + exec "./$bin" "-test.run=^(${pinned_by_shard[index + 1]})\$" "$@" +fi +export TEST_TOTAL_SHARDS=$((total - pinned_shards)) +export TEST_SHARD_INDEX=$((index - pinned_shards)) +exec "./$bin" "-test.skip=^(${all})\$" "$@" diff --git a/tools/bazel/pin_shards.py b/tools/bazel/pin_shards.py new file mode 100755 index 000000000..59e58b73d --- /dev/null +++ b/tools/bazel/pin_shards.py @@ -0,0 +1,117 @@ +#!/usr/bin/env python3 +"""Write a pinned-shard manifest for tools/bazel/go_test_pinned_shard.sh. + +Usage: pin_shards.py --shards P --min-seconds S [--write MANIFEST] TESTLOGS... + +TESTLOGS are a sharded test's bazel-testlogs directories (each holding +shard_*_of_*/test.xml, or one test.xml), from one or more runs. Every +top-level (a name without "/") at or above S seconds in any run is +pinned: its duration is the maximum seen, and the tests are bin-packed +longest-processing-time first onto P shards. Every other test stays in the +binary's own round-robin shards. Without --write the manifest is printed. + +The durations come from the test.xml -test.v writes (GO_TEST_WRAP_TESTV=1, as +--config=doltserver-cmd sets), so regenerate from a run of the lane itself: + + bazel test //cmd/bd:bd_dolt_server_test --config=doltserver-cmd + python3 tools/bazel/pin_shards.py --shards 16 --min-seconds 5 \\ + --write cmd/bd/dolt_server_pinned_shards.txt \\ + bazel-testlogs/cmd/bd/bd_dolt_server_test + +Exit status: 0 on success, 1 on bad input (no test.xml, fewer pinned tests +than shards). +""" + +import argparse +import glob +import os +import sys +import xml.etree.ElementTree as ET + + +def durations(testlogs_dirs): + """Return {top-level test name: max seconds} over every test.xml.""" + out = {} + for d in testlogs_dirs: + paths = sorted(glob.glob(os.path.join(d, "shard_*_of_*", "test.xml"))) + if not paths and os.path.exists(os.path.join(d, "test.xml")): + paths = [os.path.join(d, "test.xml")] + if not paths: + raise ValueError(f"{d}: no test.xml") + for p in paths: + for tc in ET.parse(p).getroot().iter("testcase"): + name = tc.get("name") or "" + if not name.startswith("Test") or "/" in name: + continue + secs = float(tc.get("time") or 0) + out[name] = max(out.get(name, 0.0), secs) + return out + + +def pin(durs, shards, min_seconds): + """Bin-pack the tests >= min_seconds onto shards (LPT). Returns a list of + (shard number 1..shards, [(name, secs)...]) and the per-shard totals.""" + heavy = sorted( + ((n, s) for n, s in durs.items() if s >= min_seconds), + key=lambda ns: (-ns[1], ns[0]), + ) + if len(heavy) < shards: + raise ValueError(f"{len(heavy)} tests take >= {min_seconds}s, fewer than {shards} shards") + bins = [[] for _ in range(shards)] + totals = [0.0] * shards + for name, secs in heavy: + k = min(range(shards), key=lambda i: (totals[i], i)) + bins[k].append((name, secs)) + totals[k] += secs + return bins, totals + + +def render(bins, totals, min_seconds, rest): + lines = [ + "# Pinned shards for tools/bazel/go_test_pinned_shard.sh. Generated by", + "# tools/bazel/pin_shards.py; regenerate rather than edit (see its usage).", + "#", + f"# Every top-level test measured at >= {min_seconds:g}s, bin-packed", + "# longest-first onto these shards; every other test runs in the", + "# binary's own round-robin shards. Measured seconds per pinned shard:", + "# " + " ".join(f"{t:.0f}" for t in totals), + f"# Unpinned tests: {rest[0]}, {rest[1]:.0f}s in total.", + "#", + "# ", + ] + for k, tests in enumerate(bins, start=1): + for name, _ in sorted(tests): + lines.append(f"{k} {name}") + return "\n".join(lines) + "\n" + + +def main(argv=None): + ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + ap.add_argument("--shards", type=int, required=True) + ap.add_argument("--min-seconds", type=float, required=True) + ap.add_argument("--write") + ap.add_argument("testlogs", nargs="+") + args = ap.parse_args(argv) + if args.shards < 1: + print("--shards must be at least 1", file=sys.stderr) + return 1 + try: + durs = durations(args.testlogs) + bins, totals = pin(durs, args.shards, args.min_seconds) + except (ValueError, OSError, ET.ParseError) as e: + print(f"pin_shards.py: {e}", file=sys.stderr) + return 1 + rest = [s for s in durs.values() if s < args.min_seconds] + text = render(bins, totals, args.min_seconds, (len(rest), sum(rest))) + if args.write: + tmp = args.write + ".tmp" + with open(tmp, "w", encoding="utf-8") as f: + f.write(text) + os.replace(tmp, args.write) + else: + sys.stdout.write(text) + return 0 + + +if __name__ == "__main__": + sys.exit(main())