ci(bazel): pin httpclient_served_test's slow tests to shards (16 -> 26 shards) (#7443)

After #7440 //internal/httpclient:httpclient_served_test set the embedded
lane (149-167s slowest shard in recent CI runs, 63-167s across PRs since
#7416). Not a regression: its top-level tests are all serial and rules_go's
round-robin put the long ones wherever its index split dropped them, so
shard times swung with load and with which tests a PR added (whole-lane
rbe-west run on main: 40-119s over 16 shards).

The target now runs through tools/bazel/go_test_pinned_shard.sh, as
bd_dolt_server_test and regression_test do: every top-level test measured
at >= 3s (max over three whole-lane runs) is pinned onto 17 shards
(served_pinned_shards.txt, tools/bazel/pin_shards.py), and the binary's own
round-robin deals the other ~620 over the remaining 9. Every test still runs
in exactly one shard (TestPinnedShardWrapperSplit), the manifest may name
only real tests, and the lane's check_testcases.py fails an empty shard.
No test changed; the heaviest single test (~52s, no subtests) stays whole.

The pinned-wrapper policy listed only dolt-server-cmd targets, since the
wrapper selects and skips; pinnedShardTargets now carries each reviewed
target's lane tag, and httpclient_served_test is reviewed in as the
embedded lane's (its args stay pinned in
TestBazelRetiredLanesCannotBeNarrowed).

rbe-west, `bazel test //... --config=embedded --nocache_test_results`:
served shards max 119s -> 77s, median 73s -> 54s; lane critical path
138-153s -> 116s.

Bead: ga-vnycm2.30

Co-authored-by: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Julian Knutsen 2026-10-09 22:11:43 +00:00 • committed by GitHub
commit 7c47c3c0a9
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
7 changed files with 199 additions and 28 deletions

View file

@ -207,14 +207,26 @@ go_test(
# fails each served case rather than skipping it. It is the script's run on
# the lane's race build (raceEnabled shrinks its one large apply).
#
# Shard count: on the lane's executors (rbe-west, 2026-10-09) the package is
# 725 tests and ~1,300 test-seconds, the heaviest about 54s. Each served case
# copies one per-process template workspace instead of initializing its own
# embedded Dolt schema (served_harness_test.go's servedTemplateWorkspace),
# and TestServedBatchGetterContract seeds its 1000 issues in one write: before
# Shard count: on the lane's executors (rbe-west, 2026-10-09, the whole
# embedded lane running) the package is ~750 top-level tests and ~1,100
# test-seconds, all serial, the heaviest about 52s. Each served case copies
# one per-process template workspace instead of initializing its own embedded
# Dolt schema (served_harness_test.go's servedTemplateWorkspace), and
# TestServedBatchGetterContract seeds its 1000 issues in one write: before
# both, the package was ~3,700 test-seconds over 24 shards, the heaviest shard
# 390s. rules_go deals the tests out round-robin; at 16 shards the measured
# loads come to 62-112s per shard.
# 390s. rules_go's round-robin alone (16 shards) left shards at 40-119s,
# wherever its split dropped the long tests, so the tests measured at >= 3s
# (the maximum over three whole-lane runs) are pinned
# (tools/bazel/go_test_pinned_shard.sh, served_pinned_shards.txt from
# tools/bazel/pin_shards.py --shards 17 --min-seconds 3) onto 17 shards, and
# the binary's round-robin deals the other ~650 (~340s of measured test
# time; those shards ran ~1.4x their tests' summed times) over the remaining
# 9.
# Regenerate the manifest from runs of the lane:
# bazel test //... --config=embedded
# python3 tools/bazel/pin_shards.py --shards 17 --min-seconds 3 \
# --write internal/httpclient/served_pinned_shards.txt \
# bazel-testlogs/internal/httpclient/httpclient_served_test
#
# Tags:
# embedded: the embedded-Dolt tier; excluded from --config=prcore/ci,
@ -227,19 +239,23 @@ sh_test(
# that SIGTERM, so a hang would end as a bare TIMEOUT. 19m lets Go's own
# timeout fire first and dump every goroutine to test.log.
timeout = "eternal",
srcs = ["//tools/bazel:go_test_variant.sh"],
srcs = ["//tools/bazel:go_test_pinned_shard.sh"],
args = [
"$(rootpath :served_pinned_shards.txt)",
"$(rootpath :httpclient_test)",
"-test.v",
"-test.count=1",
"-test.timeout=19m",
],
data = [":httpclient_test"],
data = [
"served_pinned_shards.txt",
":httpclient_test",
],
env = {
"BEADS_HTTP_TEST_REQUIRED": "1",
"BEADS_TEST_EMBEDDED_DOLT": "1",
},
shard_count = 16,
shard_count = 26,
tags = ["embedded"],
)

View file

@ -0,0 +1,142 @@
# Pinned shards for tools/bazel/go_test_pinned_shard.sh. Generated by
# tools/bazel/pin_shards.py; regenerate rather than edit (see its usage).
#
# Every top-level test measured at >= 3s, bin-packed
# longest-first onto these shards; every other test runs in the
# binary's own round-robin shards. Measured seconds per pinned shard:
# 86 88 85 89 86 86 86 86 86 86 86 86 86 89 89 86 86
# Unpinned tests: 617, 335s in total.
#
# <pinned shard> <TestName>
1 TestServedBatchApplyEndGateCycleSurvivesSkipPerEdgeCycleCheck
1 TestServedBatchApplyUpdateAfterCloseInOneRequest
1 TestServedDualWriteContract
1 TestServedIssueOperationsCreateUnderAParentMintsTheNextChildID
1 TestServedReaderReadyPageWiderThanTheHydrationBatchIsStillThatPrefix
2 TestServedBatchApplyShape356CommitsAtomically
2 TestServedBatchGetterContract
2 TestServedDetailReadsAgreeWithTheReferenceStore
2 TestServedLifecycleCloseAdmitsATransitivelyBlockedTarget
2 TestServedReaderListAppliesTheClientSideDisplayOrder
2 TestServedReaderListStatusORSetUnderALabelScope
2 TestServedReaderReadyEphemeralPageKeepsBothPlanesCountsAtItsBoundary
3 TestServedBatchApplyAllowsAClosedParentToGainAnOpenChild
3 TestServedBatchApplyBoundsTheItemCount
3 TestServedBatchApplyShape356PlusAStaleGuardLeavesZeroRows
3 TestServedCycleDetectorContract
3 TestServedLeaseReclaimerScopesToAssigneesAndLabels
3 TestServedLifecycleCloseIsIdempotentAndKeepsTheFirstClose
3 TestServedReaderListMaxRowsUnderPushdown
4 TestServedBatchApplyANoOpBatchRecordsNoHistory
4 TestServedBatchApplyAppliesEveryItemInDeclarationOrder
4 TestServedBatchApplyRefusesExpectedVersionOnARowAnEarlierItemTouched
4 TestServedDeleteGuardsADurableNamedWithAWispDependent
4 TestServedDeleterCountsCrossPlaneEdgesItRemoves
4 TestServedDeleterSettlesTheSurvivorsOfADeletedBlocker
4 TestServedLifecycleReopenRecordsItsReason
4 TestServedReaderListOrderMatchesTheReferenceStore
5 TestServedBatchApplyAnEphemeralBatchKeepsItsWispsAndRecordsNoDurableHistory
5 TestServedBatchApplyEndGateRefusesAHierarchyTheRequestBuilt
5 TestServedBatchApplyRefusesAKeyDeclaredLater
5 TestServedLifecycleUpdateClosePolicy
5 TestServedLifecycleUpdateRefusesATemplate
5 TestServedReaderListMaxRowsBoundsWireRowsFetched
5 TestServedReadsAndWritesRefuseAfterAPostHandshakeServerSwap
6 TestServedBatchApplyRefusesARefNamingNeitherOrBoth
6 TestServedBatchApplyRefusesExpectedVersionOnARowAnEarlierItemCreated
6 TestServedDeleteGuardsAWispNamedWithADurableDependent
6 TestServedDeleterSettlesTheSurvivorsOfADeletedWispBlocker
6 TestServedLifecycleCloseRefusalsCarryTheirTypesAndWriteNothing
6 TestServedLifecycleCloseSettlesTheClosedRowItselfAndItsChild
6 TestServedReaderListSortPushdownIsOneDial
6 TestServedReaderReadyPageIsThePrefixOfTheUnboundedAnswerCountsIncluded
7 TestServedBatchApplyARefusedRequestRecordsNoHistory
7 TestServedBatchApplyAnswersEveryItemWithoutASnapshot
7 TestServedBatchApplyClosePolicyEvaluatesAtTheCloseItem
7 TestServedBatchApplySplicesAForwardMetadataRef
7 TestServedBatchCloserSettlesTheDependersOfWhatItClosed
7 TestServedLifecycleCloseCountsOpenChildrenInBothPlanes
7 TestServedQuerierDisjunctionAnswersEveryMatch
7 TestServedWorkspaceConfigContract
8 TestDualRunCloseHistoryAgreesWithTheReferenceStore
8 TestServedBatchApplyBindsEachNamedKeyToItsMintedID
8 TestServedBatchApplyEvaluatesExpectedAssigneeAsModified
8 TestServedDependencyEditorClosedChildAddSatisfiesAnAnyChildrenGate
8 TestServedGraphCounterContract
8 TestServedIssueOperationsCreateParentChildRecomputesWaitsForClosure
8 TestServedLifecycleCloseSettlesItsTransitiveAndCrossPlaneDependers
8 TestServedQuerierSortTieBreaksByIDInBothDirections
9 TestDualRunReaderAgreesWithTheReferenceStore
9 TestServedBatchApplyResolvesABackwardKeyRef
9 TestServedDependencyEditorMaintainsBlockedStateAcrossPlanes
9 TestServedDependencyEditorRecordsOneHistoryEntryPerLandedRequest
9 TestServedLifecycleCloseEnforcesTheCloseGuards
9 TestServedLifecycleCloseOnASpawnersLastChildSatisfiesAWaitsForGate
9 TestServedReaderListWeldedOrderKeepsSendingNoSortParameter
9 TestServedReaderRefusesTheFiltersTheWireCannotCarry
10 TestServedBatchApplyEvaluatesExpectedStatusAsModified
10 TestServedBatchCloserForceBypassesOnlyClosePolicy
10 TestServedBlockingAnnotatorContract
10 TestServedCounterContract
10 TestServedDependencyEditorRemoveUnmarksItsSourceAndDescendants
10 TestServedQuerierSortSeesTheWholeMatchingSet
10 TestServedReaderListIncludeEphemeralMergesThePlanesUnderALabelScope
10 TestServedReadyClaimerFencesTheClaimByEveryLabelSetAndTheParentItWasGiven
11 TestServedBatchApplyExpectedVersionThatMatchesLetsTheItemThrough
11 TestServedBatchApplyNeverReordersItsItems
11 TestServedBatchCloserPerItemRefusalIsAResultAndSurvivorsCommit
11 TestServedRawCloseKeepsParityWithTheOtherBackends
11 TestServedReaderListKeysetPositionSkipsForward
11 TestServedReaderListLimitBoundaryUnderALabelScope
11 TestServedTreeWalkerContract
11 TestTheParentWalkAnswersTheSameRowsOverHTTP
12 TestDualRunIDResolutionProbesAgreeWithTheReferenceStore
12 TestServedBatchApplyRecordsOneEntryForAWriteThatLandedNothing
12 TestServedIssueOperationsCreateWithDependenciesSettlesInTheCreatingTransaction
12 TestServedIssueOperationsUpdateStatusCrossingSettlesAConditionalBlocksDepender
12 TestServedReaderDoesNotMutateTheCallerRequestOverTheWire
12 TestServedReaderReadyParentScopesToItsTransitiveDescendants
12 TestTheCompositionCatchesAWrongParameter
12 TestTheReadyBridgeDualRunsBuildReadyFilterOverBothReadPaths
13 TestServedBatchApplyNormalizesTheWaitsForGate
13 TestServedBatchApplyRefusesACrossPlaneEdgeBetweenRowsItCreated
13 TestServedIssueOperationsUpdateClaimConflictCarriesTheLosingState
13 TestServedIssueOperationsUpdateStatusCrossingSettlesDependers
13 TestServedJournalContract
13 TestServedReaderListUnlimitedStaysAWalk
13 TestServedReaderListWalksMultiplePages
13 TestServedRelationsContract
14 TestServedBatchApplyRefusesAKeyNoItemDeclares
14 TestServedBatchApplyRollsBackEverythingWhenTheLastItemRefuses
14 TestServedLifecycleReopenReblocksItsDependers
14 TestServedLifecycleUpdateParentIDReplacesTheParentEdge
14 TestServedQuerierPageIsAPrefixAndHasMoreIsExact
14 TestServedReaderListMessageWispsNeedTheInfraIntentAndNotJustThePlane
14 TestServedReaderListMultiPageWalkHasNoSnapshotIsolation
14 TestServedSweeperProtectsLiveDependents
14 TestTheParentWalkRefusesAResidualOverHTTP
15 TestDualRunQuerierAndReadyCounterAgreeWithTheReferenceStore
15 TestServedBatchApplyStaleExpectedVersionRefusesTheWholeRequest
15 TestServedDependencyEditorRefusesACycleThroughAParentChildHop
15 TestServedLifecycleUpdateAssigneeTransferFence
15 TestServedLifecycleUpdateParentIDReplacesEveryParent
15 TestServedQuerierSortBoundsThePageInOrder
15 TestServedReaderListEmptyPageIsWellFormedUnderALabelScope
15 TestServedReaderListFallsBackToTheWalkWithoutTheCapability
15 TestServedReadyClaimerHydratesOnlyItsBlocksEdgesIntoTheCardinalities
16 TestServedBatchApplyAcceptsAnExternalEdgeTarget
16 TestServedBatchApplyHistoryNamesTheActorAndReadsTheProvenance
16 TestServedDeleterSettlesTheChildrenOfADeletedParent
16 TestServedEdgeReaderContract
16 TestServedQuerierSortByClosedPutsTheUnclosedRowsAtTheFarEnd
16 TestServedQuerierSortByTitleFoldsCaseBeforeItCutsThePage
16 TestServedReaderListPriorityKeysetPositionResumesThePriorityOrder
16 TestServedSweeperProtectsLiveDependentsAcrossPlanes
17 TestServedBatchApplyRecordsExactlyOneHistoryEntry
17 TestServedBatchApplyReportsChangedPerItem
17 TestServedBatchCloserIdempotentRecloseIsAPerItemSuccess
17 TestServedCommenterContract
17 TestServedLifecycleCreateInheritsParentLabels
17 TestServedQuerierHidesClosedUnlessTheExpressionOrTheFlagSaysOtherwise
17 TestServedReaderReadyDeferredAndEphemeralGates
17 TestServedStatsReporterContract

View file

@ -160,6 +160,7 @@ go_test(
"//:nested_go_modules",
"//:repo_go_test_srcs",
"//cmd/bd:repo_other_files",
"//internal/httpclient:repo_other_files",
"//internal/storage/dolt:repo_other_files",
"//internal/storage/embeddeddolt:repo_other_files",
"//tests/regression:repo_other_files",

View file

@ -123,7 +123,7 @@ func TestConformanceServedTierRunsUnderBazel(t *testing.T) {
if rule == "" {
t.Fatal("internal/httpclient/BUILD.bazel has no httpclient_served_test")
}
want := []string{`"$(rootpath :httpclient_test)",`, `data = [":httpclient_test"],`, `tags = ["embedded"],`}
want := []string{`"$(rootpath :httpclient_test)",`, `":httpclient_test",`, `tags = ["embedded"],`}
for _, sw := range switches {
want = append(want, `"`+sw[1]+`": "`+sw[2]+`",`)
}

View file

@ -16,12 +16,20 @@ const pinnedShardWrapper = "tools/bazel/go_test_pinned_shard.sh"
// pinnedShardTargets are the sh_tests that run a go_test binary through
// tools/bazel/go_test_pinned_shard.sh: the BUILD file, the rule, its
// manifest, and the directory holding the go_test's sources.
// manifest, the directory holding the go_test's sources, and the rule's one
// lane tag.
//
// httpclient_served_test is the embedded lane's (a retired tier's) served
// corpus, which may not be narrowed: the wrapper runs every top-level test in
// exactly one shard (TestPinnedShardWrapperSplit), the manifest may name only
// real tests (TestPinnedShardManifestsNameRealTests), and the lane's
// check_testcases.py fails a shard that ran none.
var pinnedShardTargets = []struct {
build, rule, manifest, pkg string
build, rule, manifest, pkg, tag string
}{
{"cmd/bd/BUILD.bazel", "bd_dolt_server_test", "cmd/bd/dolt_server_pinned_shards.txt", "cmd/bd"},
{"tests/regression/BUILD.bazel", "regression_test", "tests/regression/pinned_shards.txt", "tests/regression"},
{"cmd/bd/BUILD.bazel", "bd_dolt_server_test", "cmd/bd/dolt_server_pinned_shards.txt", "cmd/bd", "dolt-server-cmd"},
{"tests/regression/BUILD.bazel", "regression_test", "tests/regression/pinned_shards.txt", "tests/regression", "dolt-server-cmd"},
{"internal/httpclient/BUILD.bazel", "httpclient_served_test", "internal/httpclient/served_pinned_shards.txt", "internal/httpclient", "embedded"},
}
var pinnedLineRe = regexp.MustCompile(`^([1-9][0-9]*) (Test[A-Za-z0-9_]*)$`)

View file

@ -211,14 +211,15 @@ func TestPinShardsGenerator(t *testing.T) {
// pinnedShardWrapperUsers returns an error for every BUILD rule that runs
// through go_test_pinned_shard.sh other than pinnedShardTargets, and for any
// of those tagged for another lane than the cmd/bd Dolt-server tier: the
// wrapper selects and skips tests, which the retired tiers' lanes forbid
// (TestBazelRetiredLanesCannotBeNarrowed).
// of those not tagged exactly its pinnedShardTargets lane: the wrapper
// selects and skips tests, which the retired tiers' lanes forbid
// (TestBazelRetiredLanesCannotBeNarrowed) except for the reviewed targets
// listed there.
func pinnedShardWrapperUsers(t *testing.T, root string) []string {
t.Helper()
allowed := map[string]bool{}
allowed := map[string]string{}
for _, tgt := range pinnedShardTargets {
allowed[tgt.build+":"+tgt.rule] = true
allowed[tgt.build+":"+tgt.rule] = tgt.tag
}
const label = "//tools/bazel:go_test_pinned_shard.sh"
var errs []string
@ -247,11 +248,13 @@ func pinnedShardWrapperUsers(t *testing.T, root string) []string {
continue
}
seen++
if !allowed[f+":"+name] {
errs = append(errs, f+": "+name+" runs through "+pinnedShardWrapper+"; add it to pinnedShardTargets after reviewing that it is not a retired tier's lane")
tag, ok := allowed[f+":"+name]
if !ok {
errs = append(errs, f+": "+name+" runs through "+pinnedShardWrapper+"; add it to pinnedShardTargets after reviewing that every test still runs exactly once")
continue
}
if !strings.Contains(rule, `tags = ["dolt-server-cmd"]`) {
errs = append(errs, f+": "+name+" runs through "+pinnedShardWrapper+" but is not tagged exactly dolt-server-cmd")
if !strings.Contains(rule, `tags = ["`+tag+`"]`) {
errs = append(errs, f+": "+name+" runs through "+pinnedShardWrapper+" but is not tagged exactly "+tag)
}
}
}

View file

@ -811,9 +811,10 @@ func TestBazelRetiredLanesCannotBeNarrowed(t *testing.T) {
}
// go_test_pinned_shard.sh selects and skips by design (each test in
// exactly one shard: TestPinnedShardWrapperSplit), and is reviewed
// here for the dolt-server-cmd targets only, which are none of the
// retired tiers' lanes; pinnedShardWrapperUsers fails if anything
// else runs through it.
// here for pinnedShardTargets only (the dolt-server-cmd targets and
// the embedded lane's httpclient_served_test, whose args are pinned
// above); pinnedShardWrapperUsers fails if anything else runs
// through it.
exempt := filepath.ToSlash(rel) == pinnedShardWrapper
if exempt {
for _, e := range pinnedShardWrapperUsers(t, root) {
@ -890,7 +891,7 @@ func TestBazelRetiredLanesCannotBeNarrowed(t *testing.T) {
// script's switches: required, so a missing engine fails each served
// case instead of skipping it.
"//internal/httpclient:httpclient_served_test": {
[]string{"$(rootpath :httpclient_test)", "-test.v", "-test.count=1", "-test.timeout=19m"},
[]string{"$(rootpath :served_pinned_shards.txt)", "$(rootpath :httpclient_test)", "-test.v", "-test.count=1", "-test.timeout=19m"},
map[string]string{"BEADS_HTTP_TEST_REQUIRED": "1", "BEADS_TEST_EMBEDDED_DOLT": "1"},
},
"//cmd/bd:bd_proxied_test": {