diff --git a/.github/workflows/bazel.yml b/.github/workflows/bazel.yml index 8174010a2..7e0d8397a 100644 --- a/.github/workflows/bazel.yml +++ b/.github/workflows/bazel.yml @@ -1506,6 +1506,25 @@ jobs: if-no-files-found: ignore retention-days: 7 + # The served targets' per-test durations, on every run: their pinned + # shard manifests (backend/http/served_pinned_shards.txt from + # tools/bazel/pin_shards.py, the corpus parts' split) are sized from a + # sharded run of this lane, and the logs above are kept only on failure. + # test.xml only, a few KB each. Telemetry like the CI analytics upload: + # nothing in CI reads it, so an artifact 5xx or stall must not fail or + # hold the lane (scripts TestBazelEmbeddedServedTimingsUploadIsTelemetry). + - name: Upload served test timings + if: ${{ !cancelled() && steps.test.outcome != 'skipped' }} + continue-on-error: true + timeout-minutes: 2 + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 + with: + overwrite: true # a job re-run is the same run: replace, not 409 + name: bazel-embedded-served-testxml + path: bazel-testlogs/backend/http/**/test.xml + if-no-files-found: ignore + retention-days: 7 + # Same redacted summary as bazel-test's; see its comment. - name: CI analytics summary id: ci-analytics diff --git a/backend/http/BUILD.bazel b/backend/http/BUILD.bazel index b78058b0c..d96187690 100644 --- a/backend/http/BUILD.bazel +++ b/backend/http/BUILD.bazel @@ -69,17 +69,22 @@ go_test( # Shard count: the package's tests are serial, and as one shard the target # ran 235-394s on the lane's executors (2026-10-10..13), the embedded lane's # longest action in most runs where it executed; the lane's other shards end -# by ~90-125s. The 19 TestE2E_* cases that take >= 3s (5-74s each, the -# maximum over three lane runs and one local run) are pinned +# by ~90-125s. The 19 TestE2E_* cases that take >= 3s are pinned # (tools/bazel/go_test_pinned_shard.sh, served_pinned_shards.txt from -# tools/bazel/pin_shards.py --shards 6 --min-seconds 3) onto 6 shards of -# <= ~88 measured seconds each, and the binary's round-robin runs the other -# ~28 (well under a second together) in the 7th. -# Regenerate the manifest from runs of the lane: -# bazel test //backend/http:http_served_test --config=embedded +# tools/bazel/pin_shards.py --shards 6 --min-seconds 3) onto 6 shards, and +# the binary's round-robin runs the other ~28 (well under a second +# together) in the 7th. The durations are the maximum over two sharded runs +# of the lane (bazel.yml's bazel-embedded-served-testxml artifact): 11-90s +# each, ~770s together. The same test's time varied 2-5x between those runs +# with the executors' load, and sharded runs took up to 2x what the +# single-shard runs did, so size the pins from sharded runs of the lane, +# never from a single-shard or local one. +# Regenerate the manifest from PR runs' bazel-embedded-served-testxml +# artifacts (each http_served_test directory holds shard_*_of_7/test.xml; +# pass several runs, pin_shards.py takes each test's maximum): # python3 tools/bazel/pin_shards.py --shards 6 --min-seconds 3 \ # --write backend/http/served_pinned_shards.txt \ -# bazel-testlogs/backend/http/http_served_test +# /http_served_test /http_served_test # # Tags: # embedded: the embedded-Dolt tier; excluded from --config=prcore/ci, diff --git a/backend/http/corpus/BUILD.bazel b/backend/http/corpus/BUILD.bazel index 29c507164..537c3bdd1 100644 --- a/backend/http/corpus/BUILD.bazel +++ b/backend/http/corpus/BUILD.bazel @@ -37,12 +37,17 @@ go_test( # Shard count: as one shard (one TestServedCorpus over every shape) the # target ran 292-597s on the lane's executors (2026-10-11..13), the embedded # lane's longest action in most runs where it executed. About 100s of that -# is the per-process server and seed, the rest the ~130 shapes (~1.7s each, -# serial). The shapes are now dealt round robin across corpusParts = 4 parts -# (corpus_test.go), each a top-level test with its own server and seed, and -# served_pinned_shards.txt pins part k to shard k; the 5th shard runs the -# package's engine-free tests through the binary's round-robin. Each part -# pays the seed again, so a part's shard is ~100s + a quarter of the shapes. +# is the per-process server and seed, the rest the ~130 shapes (~1.9s each, +# serial, ~242s together). The shapes are now dealt round robin across +# corpusParts = 4 parts (corpus_test.go), each a top-level test with its own +# server and seed, and served_pinned_shards.txt pins part k to shard k; the +# 5th shard runs the package's engine-free tests through the binary's +# round-robin. Each part pays the setup again, though it seeds the served +# and embedded sides concurrently (~62-76s on the lane's executors), so a +# part's shard is that plus ~60s of shapes: 114-146s measured. Three parts +# ran 170-180s each; they save one setup (~75 test-seconds a run) but +# lengthen the lane's served long pole, which is on the gate's critical +# path, by ~34s. # # Tags: # embedded: the embedded-Dolt tier; excluded from --config=prcore/ci, diff --git a/backend/http/corpus/README.md b/backend/http/corpus/README.md index 80a138a09..cacb6e881 100644 --- a/backend/http/corpus/README.md +++ b/backend/http/corpus/README.md @@ -32,6 +32,10 @@ following for its share of the shapes: `seed` from `testdata/corpus.yaml`: one graph of named rows, plus bulk graphs that put more than one server page of ready rows and wisps in the store. Both sides use `BEADS_ACTOR=worker` and set `CLAUDE_SESSION_ID`. + The embedded side is initialized and seeded on its own goroutine while + steps 2 and 3 run and the served side is seeded (`prepareSide`): the two + sides share no directory, env or process, and each runs the seed in + order, so the seeded data does not depend on the overlap. 5. Runs each of its shapes as a subtest, on both sides. The embedded run must exit with the shape's `expect_exit` and match its `expect_stderr`. If it does not, the shape is wrong or bd changed, and the subtest fails as a corpus @@ -59,11 +63,19 @@ shape's effects: part, in corpus order. A new shape must keep that independence: it may rely on the seed and its own `setup`, never on another shape having run. -Every part seeds again (the server and seed take about 100s of a part's run -on the CI executors), so running all the parts in one process takes longer -than one run over every shape did. Run one part with +Every part seeds again, so running all the parts in one process takes +longer than one run over every shape did; each part logs its setup time +(`part N setup ...`). Run one part with `-run 'TestServedCorpusPart2$'`. +There are four parts. In the embedded lane each part's setup (server start +plus both seeds, run concurrently) takes ~62-76s and the shapes ~242s in +total. Four parts ran 114-146s each and three ran 170-180s. Three would save +one setup (~75 test-seconds a run), but the longest part is the lane's +served long pole and sits on the CI gate's critical path, so four parts win. +To change the count, update `corpusParts`, `servedCorpusParts`, +`served_pinned_shards.txt` and the target's `shard_count` together. + ``` CGO_ENABLED=1 BEADS_TEST_EMBEDDED_DOLT=1 BEADS_HTTP_TEST_REQUIRED=1 \ go test -tags gms_pure_go -run 'TestServedCorpus|TestCorpus' ./backend/http/corpus/ diff --git a/backend/http/corpus/corpus_parts_test.go b/backend/http/corpus/corpus_parts_test.go index bc57f6bb3..e8c3ec84f 100644 --- a/backend/http/corpus/corpus_parts_test.go +++ b/backend/http/corpus/corpus_parts_test.go @@ -3,10 +3,15 @@ package corpus_test import ( + "context" + "errors" "fmt" + "os" + "path/filepath" "reflect" "runtime" "strings" + "sync" "testing" ) @@ -73,6 +78,98 @@ func TestCorpusPartIsRoundRobinInOrder(t *testing.T) { } } +// fakeBD writes a stand-in for bd that appends its arguments, one call per +// line, to calls.log in its working directory, and exits 3 when an argument +// is "boom". +func fakeBD(t *testing.T) string { + t.Helper() + bin := filepath.Join(t.TempDir(), "bd") + script := "#!/bin/sh\necho \"$*\" >> calls.log\nfor a in \"$@\"; do [ \"$a\" = boom ] && { echo boom >&2; exit 3; }; done\nexit 0\n" + if err := os.WriteFile(bin, []byte(script), 0o700); err != nil { + t.Fatal(err) + } + return bin +} + +func calls(t *testing.T, sd *side) []string { + t.Helper() + raw, err := os.ReadFile(filepath.Join(sd.dir, "calls.log")) + if err != nil { + t.Fatal(err) + } + return strings.Split(strings.TrimSpace(string(raw)), "\n") +} + +// prepareSide is what lets testServedCorpus seed the served and embedded +// sides at once: each side attaches, writes its assets and runs the seed in +// its own directories, in seed order, and a failure comes back as an error +// rather than a t.Fatal from a goroutine that is not the test's. +func TestPrepareSideSeedsBothSidesConcurrently(t *testing.T) { + bin := fakeBD(t) + c := &corpusFile{ + Files: []corpusAsset{{Name: "plan.json", Content: "{}"}}, + Seed: [][]string{{"create", "--graph", "{file:plan.json}"}, {"close", "e2e-x"}}, + } + a := &side{name: "served", dir: t.TempDir(), files: t.TempDir()} + b := &side{name: "embedded", dir: t.TempDir(), files: t.TempDir()} + var wg sync.WaitGroup + errs := make([]error, 2) + for i, sd := range []*side{a, b} { + wg.Add(1) + go func() { + defer wg.Done() + errs[i] = prepareSide(context.Background(), t, bin, c, sd, []string{"init", sd.name}) + }() + } + wg.Wait() + for i, sd := range []*side{a, b} { + if errs[i] != nil { + t.Fatalf("%s: %v", sd.name, errs[i]) + } + want := []string{"init " + sd.name, "create --graph " + filepath.Join(sd.files, "plan.json"), "close e2e-x"} + if got := calls(t, sd); !reflect.DeepEqual(got, want) { + t.Errorf("%s ran %q, want %q", sd.name, got, want) + } + if _, err := os.Stat(filepath.Join(sd.files, "plan.json")); err != nil { + t.Errorf("%s: asset not written: %v", sd.name, err) + } + } +} + +func TestPrepareSideReportsAFailedStepAsAnError(t *testing.T) { + bin := fakeBD(t) + c := &corpusFile{Seed: [][]string{{"create", "boom"}, {"close", "e2e-x"}}} + sd := &side{name: "embedded", dir: t.TempDir(), files: t.TempDir()} + err := prepareSide(context.Background(), t, bin, c, sd, []string{"init"}) + if err == nil || !strings.Contains(err.Error(), "seed [create boom] on embedded failed") || !strings.Contains(err.Error(), "exit=3") { + t.Fatalf("err = %v, want the failed seed step with its exit", err) + } + if got := calls(t, sd); !reflect.DeepEqual(got, []string{"init", "create boom"}) { + t.Errorf("ran %q; the seed must stop at the failed step", got) + } + + sd = &side{name: "served", dir: t.TempDir(), files: t.TempDir()} + if err := prepareSide(context.Background(), t, bin, c, sd, []string{"connect", "boom"}); err == nil || !strings.Contains(err.Error(), "bd connect boom on served") { + t.Fatalf("err = %v, want the failed attach", err) + } +} + +// Once the other side has failed, the test cancels: a side stops before its +// next step instead of seeding on for nothing. +func TestPrepareSideStopsWhenCancelled(t *testing.T) { + bin := fakeBD(t) + c := &corpusFile{Seed: [][]string{{"create", "e2e-x"}}} + sd := &side{name: "embedded", dir: t.TempDir(), files: t.TempDir()} + ctx, cancel := context.WithCancel(context.Background()) + cancel() + if err := prepareSide(ctx, t, bin, c, sd, []string{"init"}); !errors.Is(err, context.Canceled) { + t.Fatalf("err = %v, want context.Canceled", err) + } + if _, err := os.Stat(filepath.Join(sd.dir, "calls.log")); !os.IsNotExist(err) { + t.Errorf("bd ran after the cancel (calls.log: %v)", err) + } +} + // A part test wired to another part's number would run that part's shapes // twice and drop its own without a word; testServedCorpus refuses to run // unless the test's name is its part's (corpusPartNameErr). @@ -89,6 +186,9 @@ func TestCorpusPartNameMustMatchItsPart(t *testing.T) { {"TestServedCorpusPart10", 1}, {"TestServedCorpusPart3/sub", 3}, {"TestServedCorpus", 1}, + // A stray part past corpusParts would run no shapes at all. + {fmt.Sprintf("TestServedCorpusPart%d", corpusParts+1), corpusParts + 1}, + {"TestServedCorpusPart0", 0}, } { if err := corpusPartNameErr(c.name, c.part); err == nil { t.Errorf("corpusPartNameErr(%q, %d) = nil, want a mismatch", c.name, c.part) diff --git a/backend/http/corpus/corpus_test.go b/backend/http/corpus/corpus_test.go index 1bde7ed5b..816f99405 100644 --- a/backend/http/corpus/corpus_test.go +++ b/backend/http/corpus/corpus_test.go @@ -8,7 +8,9 @@ package corpus_test import ( + "context" "encoding/json" + "errors" "fmt" "os" "path/filepath" @@ -17,7 +19,9 @@ import ( "sort" "strconv" "strings" + "sync" "testing" + "time" "gopkg.in/yaml.v3" ) @@ -246,6 +250,13 @@ type outcome struct { // served side against the embedded side after the same commands, and each // part keeps corpus order, so its reads still run on the bare seed before // any of its writes (see testdata/corpus.yaml's header). +// +// Four parts, from the embedded lane's measured part times (PR #7538): each +// part's server start and concurrent seed take ~62-76s and the 127 shapes +// ~242s together, so four parts ran 114-146s each and three ran 170-180s. +// Three would save one setup (~75 test-seconds a run), but the longest part +// is the embedded lane's served long pole, on the gate's critical path +// (forks' above all), so the ~34s it saves is worth more. const corpusParts = 4 // servedCorpusParts are the parts' tests, one per part, in part order. @@ -260,8 +271,12 @@ func TestServedCorpusPart4(t *testing.T) { testServedCorpus(t, 4) } // corpusPartNameErr fails a part test whose name is not its part's: a // TestServedCorpusPart3 that ran part 2 would run part 2's shapes twice and -// silently drop its own. +// silently drop its own. A part outside 1..corpusParts (a stray +// TestServedCorpusPart5 left after the parts shrink) would run no shapes. func corpusPartNameErr(name string, part int) error { + if part < 1 || part > corpusParts { + return fmt.Errorf("%s runs corpus part %d; there are parts 1..%d (corpusParts)", name, part, corpusParts) + } if want := fmt.Sprintf("TestServedCorpusPart%d", part); name != want { return fmt.Errorf("%s runs corpus part %d; only %s may (fix its testServedCorpus call)", name, part, want) } @@ -303,37 +318,56 @@ func testServedCorpus(t *testing.T, part int) { profile := loadReferenceProfile(t) bin := buildBD(t) - serverAddr := startCorpusServer(t) - proxy := newReferenceProxy(t, profile, serverAddr) - proxyAddr := startProxy(t, proxy) - common := []string{ "BEADS_ACTOR=worker", "CLAUDE_SESSION_ID=corpus-session-s23", "BD_BACKUP_ENABLED=false", "BD_EXPORT_AUTO=false", } + + // The embedded side needs no server: init and seed it on a goroutine + // while this one starts the server and seeds the served side. Each side + // has its own directories, env and bd processes (prepareSide), and each + // still runs the seed in order, so the seeded data is what it was when + // the two sides were seeded command by command in turn. The deferred + // cancel and Wait run on every exit, a t.Fatal included, so the goroutine + // never outlives the test or its TempDirs. + setupStart := time.Now() + ctx, cancel := context.WithCancel(context.Background()) + var seeding sync.WaitGroup + defer seeding.Wait() + defer cancel() + local := &side{name: "embedded", dir: t.TempDir(), files: t.TempDir(), env: common} + var localErr error + seeding.Add(1) + go func() { + defer seeding.Done() + localErr = prepareSide(ctx, t, bin, c, local, + []string{"init", "-p", corpusPrefix, "--non-interactive", "--skip-hooks", "--skip-agents", "-q"}) + if localErr != nil { + cancel() + } + }() + + serverAddr := startCorpusServer(t) + proxy := newReferenceProxy(t, profile, serverAddr) + proxyAddr := startProxy(t, proxy) served := &side{name: "served", dir: t.TempDir(), files: t.TempDir(), env: append([]string{"BEADS_HTTP_TOKEN=" + proxyAddr + "=" + corpusToken}, common...)} - local := &side{name: "embedded", dir: t.TempDir(), files: t.TempDir(), env: common} - - if r := runBD(t, bin, served.dir, served.env, "", "connect", "http://"+proxyAddr, "--expect-project-id", corpusProjectID, "--json"); r.code != 0 { - t.Fatalf("bd connect: %s", r) + servedErr := prepareSide(ctx, t, bin, c, served, + []string{"connect", "http://" + proxyAddr, "--expect-project-id", corpusProjectID, "--json"}) + if servedErr != nil { + cancel() } - if r := runBD(t, bin, local.dir, local.env, "", "init", "-p", corpusPrefix, "--non-interactive", "--skip-hooks", "--skip-agents", "-q"); r.code != 0 { - t.Fatalf("bd init (embedded): %s", r) - } - for _, sd := range []*side{served, local} { - writeAssets(t, c.Files, sd.files) - } - for _, args := range c.Seed { - for _, sd := range []*side{served, local} { - argv := expand(t, bin, sd, args) - if r := runBD(t, bin, sd.dir, sd.env, "", argv...); r.code != 0 { - t.Fatalf("seed %v on %s failed; the seed must use shapes that work on the reference profile: %s", args, sd.name, r) - } + seeding.Wait() + // The side that failed first is the one to report; the other may only + // have seen the cancel. + for _, err := range []error{servedErr, localErr} { + if err != nil && !errors.Is(err, context.Canceled) { + t.Fatal(err) } } + t.Logf("part %d setup (server start and both sides seeded concurrently): %s", part, time.Since(setupStart).Round(100*time.Millisecond)) proxy.takeRefusals() skipPending := os.Getenv("BEADS_CORPUS_SKIP_PENDING") == "1" @@ -401,16 +435,16 @@ func runShape(t *testing.T, bin string, s shape, served, local *side, allow map[ for _, args := range s.Setup { for _, sd := range []*side{served, local} { - if r := runBD(t, bin, sd.dir, sd.env, "", expand(t, bin, sd, args)...); r.code != 0 { + if r := runBD(t, bin, sd.dir, sd.env, "", expand(context.Background(), t, bin, sd, args)...); r.code != 0 { failf("setup %v failed on %s: %s", args, sd.name, r) return fails } } } proxy.takeRefusals() - sr := runBD(t, bin, served.dir, served.env, s.Stdin, expand(t, bin, served, s.Args)...) + sr := runBD(t, bin, served.dir, served.env, s.Stdin, expand(context.Background(), t, bin, served, s.Args)...) wireRefusals := proxy.takeRefusals() - lr := runBD(t, bin, local.dir, local.env, s.Stdin, expand(t, bin, local, s.Args)...) + lr := runBD(t, bin, local.dir, local.env, s.Stdin, expand(context.Background(), t, bin, local, s.Args)...) t.Logf("bd %s\nserved: %s\nembedded: exit=%d", strings.Join(s.Args, " "), sr, lr.code) if len(wireRefusals) > 0 { t.Logf("reference-profile refusals on the wire: %v", wireRefusals) @@ -457,8 +491,8 @@ func runShape(t *testing.T, bin string, s shape, served, local *side, allow map[ return fails } if len(s.PrefixOf) > 0 { - vs := runBD(t, bin, served.dir, served.env, "", expand(t, bin, served, s.PrefixOf)...) - vl := runBD(t, bin, local.dir, local.env, "", expand(t, bin, local, s.PrefixOf)...) + vs := runBD(t, bin, served.dir, served.env, "", expand(context.Background(), t, bin, served, s.PrefixOf)...) + vl := runBD(t, bin, local.dir, local.env, "", expand(context.Background(), t, bin, local, s.PrefixOf)...) if vs.code != 0 || vl.code != 0 { failf("prefix_of %v: exit served %d, embedded %d: served %s", s.PrefixOf, vs.code, vl.code, vs) } else if d := comparePrefix(sr.stdout, lr.stdout, vs.stdout, vl.stdout); d != "" { @@ -468,8 +502,8 @@ func runShape(t *testing.T, bin string, s shape, served, local *side, allow map[ failf("output parity (%s): %s", modeOf(s), d) } for _, args := range s.Verify { - vs := runBD(t, bin, served.dir, served.env, "", expand(t, bin, served, args)...) - vl := runBD(t, bin, local.dir, local.env, "", expand(t, bin, local, args)...) + vs := runBD(t, bin, served.dir, served.env, "", expand(context.Background(), t, bin, served, args)...) + vl := runBD(t, bin, local.dir, local.env, "", expand(context.Background(), t, bin, local, args)...) if vs.code != vl.code { failf("verify %v: exit served %d, embedded %d: served %s", args, vs.code, vl.code, vs) continue @@ -761,8 +795,11 @@ func brief(v any) string { var placeholder = regexp.MustCompile(`\{(file|rev):([^}]+)\}`) -// expand substitutes {file:NAME} and {rev:ID} in args for side sd. -func expand(t *testing.T, bin string, sd *side, args []string) []string { +// expand substitutes {file:NAME} and {rev:ID} in args for side sd. It never +// fails t (a {rev:} it cannot read becomes "0"), so prepareSide may call it +// off the test's goroutine; its {rev:} lookup runs under ctx, so a cancelled +// prepareSide starts no further bd there either. +func expand(ctx context.Context, t *testing.T, bin string, sd *side, args []string) []string { t.Helper() out := make([]string, len(args)) for i, a := range args { @@ -772,7 +809,11 @@ func expand(t *testing.T, bin string, sd *side, args []string) []string { case "file": return filepath.Join(sd.files, parts[2]) default: - r := runBD(t, bin, sd.dir, sd.env, "", "show", parts[2], "--json") + r, err := execBD(ctx, bin, sd.dir, sd.env, "", "show", parts[2], "--json") + if err != nil { + t.Logf("{rev:%s} on %s: %v", parts[2], sd.name, err) + return "0" + } if r.code != 0 { t.Logf("{rev:%s} on %s: show failed: %s", parts[2], sd.name, r) return "0" @@ -796,17 +837,49 @@ func expand(t *testing.T, bin string, sd *side, args []string) []string { return out } -func writeAssets(t *testing.T, assets []corpusAsset, dir string) { +// prepareSide attaches sd to its store with bd attach (connect or init), +// writes the corpus's asset files into sd.files and runs the seed on sd, in +// order. It touches only sd's own directories and bd processes and returns a +// failure as an error instead of failing t, so testServedCorpus runs it for +// the two sides at once; it stops before its next bd once ctx is done. +func prepareSide(ctx context.Context, t *testing.T, bin string, c *corpusFile, sd *side, attach []string) error { t.Helper() + r, err := execBD(ctx, bin, sd.dir, sd.env, "", attach...) + if err != nil { + return err + } + if r.code != 0 { + return fmt.Errorf("bd %s on %s: %s", strings.Join(attach, " "), sd.name, r) + } + if err := writeAssets(c.Files, sd.files); err != nil { + return fmt.Errorf("%s: %w", sd.name, err) + } + for _, args := range c.Seed { + if err := ctx.Err(); err != nil { + return err + } + r, err := execBD(ctx, bin, sd.dir, sd.env, "", expand(ctx, t, bin, sd, args)...) + if err != nil { + return err + } + if r.code != 0 { + return fmt.Errorf("seed %v on %s failed; the seed must use shapes that work on the reference profile: %s", args, sd.name, r) + } + } + return nil +} + +func writeAssets(assets []corpusAsset, dir string) error { for _, a := range assets { content := a.Content if a.Bulk != nil { content = bulkPlan(*a.Bulk) } if err := os.WriteFile(filepath.Join(dir, a.Name), []byte(content), 0o600); err != nil { - t.Fatalf("write asset %s: %v", a.Name, err) + return fmt.Errorf("write asset %s: %w", a.Name, err) } } + return nil } func bulkPlan(b bulkGraph) string { diff --git a/backend/http/corpus/harness_test.go b/backend/http/corpus/harness_test.go index 3b93d0473..135499b25 100644 --- a/backend/http/corpus/harness_test.go +++ b/backend/http/corpus/harness_test.go @@ -209,7 +209,22 @@ func clip(s string, n int) string { // runBD runs the built bd binary in dir with a hermetic environment. func runBD(t *testing.T, bin, dir string, env []string, stdin string, args ...string) bdResult { t.Helper() - ctx, cancel := context.WithTimeout(context.Background(), bdTimeout) + r, err := execBD(context.Background(), bin, dir, env, stdin, args...) + if err != nil { + t.Fatal(err) + } + return r +} + +// execBD is runBD without the testing.T: it returns an error, rather than +// failing a test, when bd could not be run at all or ctx was cancelled, so it +// is safe on a goroutine that is not the test's. A bd that runs and fails is +// a bdResult with its exit code, as in runBD. +func execBD(ctx context.Context, bin, dir string, env []string, stdin string, args ...string) (bdResult, error) { + if err := ctx.Err(); err != nil { + return bdResult{}, err + } + ctx, cancel := context.WithTimeout(ctx, bdTimeout) defer cancel() cmd := exec.CommandContext(ctx, bin, args...) cmd.Dir = dir @@ -224,16 +239,18 @@ func runBD(t *testing.T, bin, dir string, env []string, stdin string, args ...st if err != nil { var ee *exec.ExitError switch { - case ctx.Err() != nil: + case errors.Is(ctx.Err(), context.DeadlineExceeded): code = -1 e.WriteString(fmt.Sprintf("\n[corpus harness] bd timed out after %s", bdTimeout)) + case ctx.Err() != nil: + return bdResult{}, fmt.Errorf("bd %s: %w", strings.Join(args, " "), ctx.Err()) case errors.As(err, &ee): code = ee.ExitCode() default: - t.Fatalf("running bd %s: %v", strings.Join(args, " "), err) + return bdResult{}, fmt.Errorf("running bd %s: %v", strings.Join(args, " "), err) } } - return bdResult{stdout: o.String(), stderr: e.String(), code: code} + return bdResult{stdout: o.String(), stderr: e.String(), code: code}, nil } // minimalBDEnv is backend/http's hermetic bd environment: PATH and the few diff --git a/backend/http/served_pinned_shards.txt b/backend/http/served_pinned_shards.txt index 6fca673c7..c27d94d56 100644 --- a/backend/http/served_pinned_shards.txt +++ b/backend/http/served_pinned_shards.txt @@ -4,26 +4,26 @@ # Every top-level test measured at >= 3s, bin-packed # longest-first onto these shards; every other test runs in the # binary's own round-robin shards. Measured seconds per pinned shard: -# 86 87 86 88 88 87 +# 135 125 126 134 124 123 # Unpinned tests: 28, 0s in total. # # -1 TestE2E_ConnectClearRestoresThePreviousBackend -1 TestE2E_DefaultDepTreeOverHTTP -2 TestE2E_PingOverHTTP -2 TestE2E_ShowJSONCountsMatchEmbedded -3 TestE2E_AdvertisedExternalPolicyIsPassedThrough -3 TestE2E_ListWithGCFiltersMatchesEmbedded +1 TestE2E_ClaimOfAbsentIDIsGCNotFound +1 TestE2E_ExternalDependencyBlockingStillEnforced +1 TestE2E_SwitchingBackToDoltLeavesOriginalIdentityIntact +2 TestE2E_CloseAndReopenUnderClaudeSession +2 TestE2E_ForceConnectWritesSucceedDespiteDifferingLocalProjectID +2 TestE2E_ListWithGCFiltersMatchesEmbedded +3 TestE2E_BDOverHTTP +3 TestE2E_DefaultDepTreeOverHTTP 3 TestE2E_ShowMissingIDIsNotFoundInOneLookup -4 TestE2E_BDOverHTTP -4 TestE2E_CloseAndReopenUnderClaudeSession +4 TestE2E_ConnectClearRestoresThePreviousBackend +4 TestE2E_MoleculesOverHTTP 4 TestE2E_SearchOverHTTP 4 TestE2E_ShowCurrentOverHTTP -5 TestE2E_ForceConnectWritesSucceedDespiteDifferingLocalProjectID -5 TestE2E_MoleculeCloseAgainstAnOlderServer +5 TestE2E_RecomputeBlockedOverHTTP 5 TestE2E_ReconnectRefusesADifferentProjectUnlessForced -5 TestE2E_SwitchingBackToDoltLeavesOriginalIdentityIntact -6 TestE2E_ClaimOfAbsentIDIsGCNotFound -6 TestE2E_ExternalDependencyBlockingStillEnforced -6 TestE2E_MoleculesOverHTTP -6 TestE2E_RecomputeBlockedOverHTTP +5 TestE2E_ShowJSONCountsMatchEmbedded +6 TestE2E_AdvertisedExternalPolicyIsPassedThrough +6 TestE2E_MoleculeCloseAgainstAnOlderServer +6 TestE2E_PingOverHTTP diff --git a/scripts/ci_workflow_test.go b/scripts/ci_workflow_test.go index d284c8e9d..fc008f811 100644 --- a/scripts/ci_workflow_test.go +++ b/scripts/ci_workflow_test.go @@ -1638,6 +1638,43 @@ func ciAnalyticsContinueOnErrorPaths(t *testing.T) map[string]bool { return paths } +// bazelServedTimingsStepName is bazel-embedded's per-run upload of the served +// targets' test.xml (bazel-embedded-served-testxml), which their pinned +// shard manifests are sized from (backend/http/BUILD.bazel). Telemetry like +// the CI analytics upload: nothing in bazel.yml, pr.yml or the gate reads it, +// so an artifact service 5xx or stall must not turn the required embedded +// lane red or hold it to the job timeout. Its continue-on-error is the one +// TestBazelWorkflowSecretsAndFailureSurface allows here, and only on this +// exact step (bazelServedTimingsContinueOnErrorPath). +const bazelServedTimingsStepName = "Upload served test timings" + +// bazelServedTimingsContinueOnErrorPath returns the walkYAML path of the +// served-timings upload's continue-on-error key, after checking the step is +// still only that upload: upload-artifact, the fixed artifact name, test.xml +// under backend/http only, a 2-minute timeout. +func bazelServedTimingsContinueOnErrorPath(t *testing.T) string { + t.Helper() + job := readCIWorkflow(t, bazelWorkflowName).job(t, bazelEmbedJobName) + for i, step := range job.Steps { + if step.Name != bazelServedTimingsStepName { + continue + } + if !strings.HasPrefix(step.Uses, "actions/upload-artifact@") || + step.With["name"] != "bazel-embedded-served-testxml" || + step.With["path"] != "bazel-testlogs/backend/http/**/test.xml" || + step.TimeoutMinutes != 2 || fmt.Sprint(step.ContinueOnError) != "true" { + t.Errorf("%s step %q = %+v; want a continue-on-error, 2-minute upload-artifact of bazel-testlogs/backend/http/**/test.xml named bazel-embedded-served-testxml", bazelEmbedJobName, step.Name, step) + } + return fmt.Sprintf(".jobs.%s.steps[%d].continue-on-error", bazelEmbedJobName, i) + } + t.Fatalf("%s has no %q step", bazelEmbedJobName, bazelServedTimingsStepName) + return "" +} + +func TestBazelEmbeddedServedTimingsUploadIsTelemetry(t *testing.T) { + bazelServedTimingsContinueOnErrorPath(t) +} + // bazelDoltIntTestContinueOnErrorPath: bazel-dolt-int's `bazel test` step's // continue-on-error key path, one of the step-level exceptions // TestBazelWorkflowSecretsAndFailureSurface allows (must-fix 2). Unlike the @@ -4733,6 +4770,7 @@ func TestBazelWorkflowSecretsAndFailureSurface(t *testing.T) { ciAnalyticsPaths := ciAnalyticsContinueOnErrorPaths(t) doltIntTestPath := bazelDoltIntTestContinueOnErrorPath(t) doltIntCheckPaths := bazelDoltIntCheckContinueOnErrorPaths(t) + servedTimingsPath := bazelServedTimingsContinueOnErrorPath(t) secretRef := regexp.MustCompile(`\bsecrets\s*(\.|\[)`) walkYAML(root, "", func(path string, key bool, value string) { if key && value == "continue-on-error" { @@ -4755,10 +4793,13 @@ func TestBazelWorkflowSecretsAndFailureSurface(t *testing.T) { // always() classify step right after them (no continue-on-error // of its own) folds every one of their outcomes, plus the test // step's own rc, into job.status per tier via --check, so - // nothing is hidden from pr.yml's ci-gate, only moved. + // nothing is hidden from pr.yml's ci-gate, only moved. So is + // bazel-embedded's served-timings upload + // (bazelServedTimingsContinueOnErrorPath): telemetry, read by + // nothing in CI. jobName := strings.TrimSuffix(strings.TrimPrefix(path, ".jobs."), ".continue-on-error") _, flagGated := bazelFlagGatedLanes[jobName] - if path != bazelRBEPrewarmContinueOnErrorPath && path != doltIntTestPath && !doltIntCheckPaths[path] && !flagGated && !ciAnalyticsPaths[path] && !bazelUnionHasFlagGatedMember(jobName) { + if path != bazelRBEPrewarmContinueOnErrorPath && path != doltIntTestPath && path != servedTimingsPath && !doltIntCheckPaths[path] && !flagGated && !ciAnalyticsPaths[path] && !bazelUnionHasFlagGatedMember(jobName) { t.Errorf("%s: %s hides failures from pr.yml's ci-gate", bazelWorkflowName, path) } }