From b1a0690249dcecd80035e567f87f80c2302187d3 Mon Sep 17 00:00:00 2001 From: Julian Knutsen Date: Sun, 4 Oct 2026 00:15:08 -0700 Subject: [PATCH] ci: pin four hash-placed proxied tests off overloaded legacy shards 5 and 3 (#7172) Main's "Test (proxied-server cmd 5/15)" has hit the 15m -test.timeout in 6 of the 9 Main runs since #7129 where the shard ran to completion. That PR renamed 12 proxied tests into TestProxiedServer* discovery. They are not in the frozen 15-shard legacy block, so they hash-place, and two landed on shard 5: TestProxiedServerStorageClass (nine parallel inits, the heaviest of the 12) and TestProxiedServerServeRefusesAStaleCheckpointWithGone. Shard 5 went from 430-620s of job time to 800-950s. In those runs the timeout fired while TestProxiedServerMolReadyGated's parallel subtests were still queued behind the rest of the shard. That test is the victim, not a pathological one. Shard 3 (also hash-loaded) runs 810-870s, within about a minute of the same limit. Measured locally (4 pinned CPUs, GOMAXPROCS=4), shard 5 took 897s with its current test set and 524s with the pre-#7129 set. Pin those two tests, plus shard 3's two heaviest hash-placed tests (TestProxiedServerListComments and TestProxiedServerServeRefusesAStreamOnADisabledJournal), onto the legacy shards with the lowest measured job time: 14, 7, 4 and 1. Every test already listed in the block stays where it is, so shard 5 is back to its pre-#7129 test set. pr-risk.yml's fork jobs read the same block, so they get the same relief. The script's 15m timeout, the job and step timeouts, and the Bazel 30-shard block are unchanged. The block header and the generator's legacy header now say a reviewed pin is allowed. Co-authored-by: Gas City Adopt-PR Co-authored-by: Claude Opus 5.5 (1M context) --- .github/scripts/proxied-cmd-test-shards.txt | 17 +++++++++++++++++ scripts/ci/gen_proxied_shard_manifest.py | 8 ++++++++ 2 files changed, 25 insertions(+) diff --git a/.github/scripts/proxied-cmd-test-shards.txt b/.github/scripts/proxied-cmd-test-shards.txt index b33ee52e9..4fe8b38c2 100644 --- a/.github/scripts/proxied-cmd-test-shards.txt +++ b/.github/scripts/proxied-cmd-test-shards.txt @@ -15,6 +15,19 @@ # if you do (a fresh run packs every currently-discovered test, not just # the ones already listed here). # +# Deliberate exception (Oct 2026): four tests that had been hash-placed +# onto the two shards closest to the 15m -test.timeout are pinned here +# by hand, onto shards whose measured CI job time was lowest; nothing +# that was already listed moved. TestProxiedServerStorageClass and +# TestProxiedServerServeRefusesAStaleCheckpointWithGone (discovered +# from #7129 on) had hashed onto shard 5, which then hit the timeout on +# most Main runs; pinning them returns shard 5 to its earlier test set. +# TestProxiedServerListComments and +# TestProxiedServerServeRefusesAStreamOnADisabledJournal had hashed +# onto shard 3, which runs within about a minute of the timeout. An +# incremental --write keeps pinned lines where they are; a pin is a +# reviewed choice, not a regeneration of this block. +# # This file is generated: notes added here are erased by the next # regeneration. Add them to the generator instead. Known blind spots # of the cost proxy (it counts inits, not the work done per init): @@ -40,6 +53,7 @@ 15 1 TestProxiedServerMolLastActivity 15 1 TestProxiedServerReopenConcurrent 15 1 TestProxiedServerServeReadParity +15 1 TestProxiedServerServeRefusesAStreamOnADisabledJournal 15 1 TestProxiedServerUpdateConcurrentClaim 15 2 TestProxiedServerCompactHistory @@ -66,6 +80,7 @@ 15 4 TestProxiedServerCreateInfraTypeRoutesToWisps 15 4 TestProxiedServerEdit +15 4 TestProxiedServerListComments 15 4 TestProxiedServerMoleculeFrontDoorWriteParity 15 4 TestProxiedServerPour 15 4 TestProxiedServerReassignFence @@ -102,6 +117,7 @@ 15 7 TestProxiedServerOrphans 15 7 TestProxiedServerReady 15 7 TestProxiedServerServeClaim +15 7 TestProxiedServerServeRefusesAStaleCheckpointWithGone 15 7 TestProxiedServerServeUpdate 15 7 TestProxiedServerState @@ -181,6 +197,7 @@ 15 14 TestProxiedServerPrime 15 14 TestProxiedServerReadyConcurrent 15 14 TestProxiedServerServeLifecycle +15 14 TestProxiedServerStorageClass 15 14 TestProxiedServerUnclaimIfAssigneeParity 15 15 TestProxiedServerCloseConcurrent diff --git a/scripts/ci/gen_proxied_shard_manifest.py b/scripts/ci/gen_proxied_shard_manifest.py index 90b064fcc..168715580 100644 --- a/scripts/ci/gen_proxied_shard_manifest.py +++ b/scripts/ci/gen_proxied_shard_manifest.py @@ -210,6 +210,14 @@ def render(total, shards, weights): out.append('# after adding tests (they hash-distribute until a maintainer chooses') out.append('# to reweight on purpose). --check does not cover this block; only the') out.append('# Bazel-only --weights=duration block is checked in CI.') + out.append('#') + out.append('# Hand-pinned lines are allowed: when a hash-placed test overloads a') + out.append('# shard toward the 15m -test.timeout, a reviewed change may list it') + out.append('# on a lighter shard without moving anything already listed (done in') + out.append('# Oct 2026 for TestProxiedServerStorageClass,') + out.append('# TestProxiedServerServeRefusesAStaleCheckpointWithGone,') + out.append('# TestProxiedServerListComments and') + out.append('# TestProxiedServerServeRefusesAStreamOnADisabledJournal).') else: out.append(f'# {total}-shard split for the Bazel-only proxied-server tier') out.append('# (bazel-proxied in .github/workflows/bazel.yml), bin-packed')