From 671715bb8ba452b3641dbd655f5c9bf39b922441 Mon Sep 17 00:00:00 2001 From: Julian Knutsen Date: Mon, 28 Sep 2026 04:06:49 -0700 Subject: [PATCH] test(testutil): opt-in local dolt sql-server backend for Dolt-backed tests (#6891) * test(testutil): local dolt sql-server backend behind the Dolt test server API Add a second backend to internal/testutil behind the unchanged container API (EnsureDoltContainerForTestMain, RequireDoltContainer, StartIsolatedDoltContainer[Handle], NewContainerProvider, RestartSharedDoltContainer, DoltContainerCrashed, IsolatedDoltContainer Stop/Start/CurrentPort/Exec): the test process spawns `dolt sql-server` from the pinned dolt CLI instead of a dolthub/dolt-sql-server container. - BEADS_TEST_DOLT_SERVER=local|container selects it; unset keeps plain `go test` on the container (no behavior change) and makes `bazel test` use the local backend. Other values fail instead of guessing. - The CLI (BEADS_TEST_DOLT_BINARY, else PATH) must be the image's version. - Provisioned like testcontainers' dolt module: root@% (DOLT_ROOT_HOST=%), database beads_test, user test/test. Clean env, TZ=UTC, GOMAXPROCS=4, own dolt global config with metrics and the version check disabled. - Loopback port picked here (dolt rejects -P 0), never 3307, retried when another process wins the port; readiness requires the server's own "Server ready" line plus a bounded ping, so a foreign listener on a contested port cannot pass for it. - State lives under the TMPDIR captured at package init, outside every PinSuiteTempRoot root, so suite leak sweeps never see it. The server stays in the test's process group (bazel's group kill reaps it) with Pdeathsig on Linux. - BEADS_TEST_REQUIRE_DOLT_CONTAINER=1 now also makes every TestMain that only warned about an unavailable server exit 1 (DoltUnavailableForTestMain), plus the shared proxied harness. - dolt_benchmark_purge_test finds .dolt_dropped_databases relative to Exec's working directory (the data dir in both backends). - TestDoltServerFingerprint checks each available backend against one checked-in fingerprint (users, grants, databases, server variables); local-backend tests cover the bind-race retry, 3307, crash detection, Stop/Start, Exec cwd, state outside the suite root, and teardown. - Bazel: the requires-docker variants pin BEADS_TEST_DOLT_SERVER=container so the docker lane stays the container A/B control; the test wrapper's dolt global config disables metrics and the version check. Co-Authored-By: Claude Opus 5.5 (1M context) * fix(testutil): select the local Dolt test server only when asked for Under Bazel an unset BEADS_TEST_DOLT_SERVER used to mean the local backend, so every non-ci `bazel test` started servers for Dolt suites no lane runs with one (//internal/storage/dolt:dolt_test went from a 16 s skip to a timeout). Unset now means container everywhere: in a Bazel action without docker that is the pre-existing warn+skip, and only a target's env or --test_env=BEADS_TEST_DOLT_SERVER=local starts local servers. Plain `go test` is unchanged. The docker variants keep their container pin: a rule env beats --test_env, so a local-backend invocation cannot flip the docker lane. Co-Authored-By: Claude Opus 5.5 (1M context) * fix(testutil): harden the local Dolt test server's host footprint - DOLT_DISABLE_EVENT_FLUSH=1 for every dolt the backend runs (and in the Bazel test wrapper): the metrics/versioncheck keys stop network calls but not the detached `dolt send-metrics` child each command forks on exit, which re-created the version probe's removed HOME (one leaked bdt-doltver-* per test process) and raced terminate's RemoveAll. - Every bdt-dolt-*/bdt-doltver-* root records its owner PID, and the first use of the backend in a process removes this user's roots whose owner is dead (the SweepDeadSuiteRoots rule; a copy, since doltserver's tests import testutil). A panicking, timed-out or killed test binary, or a package whose TestMain never terminates the singleton, no longer leaks a 40-100 MB state dir per run. - secure_file_priv is the state root's sfp/ directory, set through a per-launch --config file (dolt ignores other flags with --config): a local server runs as the test user on the host with a passwordless root@%, so LOAD_FILE / INTO OUTFILE must not reach the user's files. The fingerprint treats that path as the container's "". - Fingerprint: character_set_server, collation_server, max_allowed_packet, transaction_isolation, lower_case_table_names, lower_case_file_system (compared on Linux only: APFS reports ON), wait_timeout, innodb_lock_wait_timeout. - The local backend's own tests skip off Linux (no Pdeathsig, so a killed test binary would orphan servers on the macOS runners) unless BEADS_TEST_DOLT_SERVER=local asks for them. Co-Authored-By: Claude Opus 5.5 (1M context) * ci: compare the container and local Dolt test servers in the domain+uow job TestDoltServerFingerprint's container half never ran in CI: only the test-domain-uow job has both the pulled image and the pinned dolt CLI, and it never ran ./internal/testutil. Run the fingerprint there (with BEADS_TEST_REQUIRE_DOLT_CONTAINER=1 so the container half cannot skip) so an image bump that drifts the container side fails in the PR. Co-Authored-By: Claude Opus 5.5 (1M context) --------- Co-authored-by: Claude Opus 5.5 (1M context) --- .github/workflows/pr.yml | 9 + beads_test.go | 4 +- cmd/bd/doctor/dolt_e2e_test.go | 4 +- cmd/bd/doctor/fix/BUILD.bazel | 7 +- cmd/bd/doctor/fix/testmain_cgo_test.go | 4 +- cmd/bd/protocol/BUILD.bazel | 3 + cmd/bd/protocol/helpers_test.go | 4 +- cmd/bd/proxied_shared_harness_test.go | 3 + cmd/bd/test_dolt_server_cgo_test.go | 5 +- engdocs/TESTING.md | 18 + internal/beads/testmain_integration_test.go | 5 +- internal/molecules/testmain_test.go | 4 +- .../storage/dolt/dolt_benchmark_purge_test.go | 31 +- internal/storage/dolt/testmain_test.go | 4 +- internal/storage/domain/db/BUILD.bazel | 7 +- internal/storage/uow/BUILD.bazel | 3 + internal/testutil/BUILD.bazel | 4 + internal/testutil/container_provider.go | 14 + internal/testutil/localdoltserver.go | 602 +++++++++++++++++ internal/testutil/localdoltserver_linux.go | 14 + internal/testutil/localdoltserver_other.go | 11 + internal/testutil/localdoltserver_test.go | 608 ++++++++++++++++++ internal/testutil/testdoltcommon.go | 22 + internal/testutil/testdoltserver.go | 93 ++- internal/testutil/testdoltserver_env_test.go | 2 + internal/tracker/testmain_test.go | 4 +- internal/utils/testmain_test.go | 5 +- scripts/ci_workflow_test.go | 8 + tests/regression/regression_test.go | 4 +- tools/bazel/test_env.sh | 11 +- 30 files changed, 1475 insertions(+), 42 deletions(-) create mode 100644 internal/testutil/localdoltserver.go create mode 100644 internal/testutil/localdoltserver_linux.go create mode 100644 internal/testutil/localdoltserver_other.go create mode 100644 internal/testutil/localdoltserver_test.go diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index 0600e4d51..43b573118 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -877,6 +877,15 @@ jobs: sha256sum -c SHA256SUMS chmod +x bd-linux-gms-pure + # The only lane with both the image and the pinned dolt CLI: compares + # the container and the local test server backend against the one + # checked-in fingerprint, so an image bump cannot drift the container + # side unnoticed. REQUIRE makes the container half fail, not skip. + - name: Test Dolt server fingerprint (container + local) + env: + BEADS_TEST_REQUIRE_DOLT_CONTAINER: "1" + run: go test -tags gms_pure_go -count=1 -timeout 5m -v -run '^TestDoltServerFingerprint$' ./internal/testutil/ + - name: Test domain + uow + tracker env: BEADS_TEST_BD_BINARY: ${{ github.workspace }}/ci-build-artifacts/bd-linux-gms-pure diff --git a/beads_test.go b/beads_test.go index 3966bc034..2faaa3ce1 100644 --- a/beads_test.go +++ b/beads_test.go @@ -29,7 +29,9 @@ func testMainInner(m *testing.M) int { // AD-01 (be-c5p): allow root tests to connect to the test container. os.Setenv("BEADS_TEST_SERVER", "1") if err := testutil.EnsureDoltContainerForTestMain(); err != nil { - fmt.Fprintf(os.Stderr, "WARN: %v, skipping Dolt tests\n", err) + if testutil.DoltUnavailableForTestMain(err) { + return 1 + } } else { defer testutil.TerminateDoltContainer() testServerPort = testutil.DoltContainerPortInt() diff --git a/cmd/bd/doctor/dolt_e2e_test.go b/cmd/bd/doctor/dolt_e2e_test.go index 2fc70e8a4..412eae31a 100644 --- a/cmd/bd/doctor/dolt_e2e_test.go +++ b/cmd/bd/doctor/dolt_e2e_test.go @@ -89,7 +89,9 @@ func testMainInner(m *testing.M) int { } if err := testutil.EnsureDoltContainerForTestMain(); err != nil { - fmt.Fprintf(os.Stderr, "WARN: %v, skipping Dolt tests\n", err) + if testutil.DoltUnavailableForTestMain(err) { + return 1 + } } else { defer testutil.TerminateDoltContainer() port := testutil.DoltContainerPortInt() diff --git a/cmd/bd/doctor/fix/BUILD.bazel b/cmd/bd/doctor/fix/BUILD.bazel index 4ca96cb1d..e3b6d5108 100644 --- a/cmd/bd/doctor/fix/BUILD.bazel +++ b/cmd/bd/doctor/fix/BUILD.bazel @@ -114,7 +114,12 @@ sh_test( srcs = ["//tools/bazel:go_test_variant.sh"], args = ["$(rootpath :fix_test)"], data = [":fix_test"], - env = {"BEADS_FIX_REQUIRE_DOLT": "1"}, + # Pinned to the container backend (the docker lane is the A/B control); + # rule env beats --test_env, so =local on the command line cannot flip it. + env = { + "BEADS_FIX_REQUIRE_DOLT": "1", + "BEADS_TEST_DOLT_SERVER": "container", + }, tags = [ "no-remote-exec", "requires-docker", diff --git a/cmd/bd/doctor/fix/testmain_cgo_test.go b/cmd/bd/doctor/fix/testmain_cgo_test.go index b93c2b2f1..19929519e 100644 --- a/cmd/bd/doctor/fix/testmain_cgo_test.go +++ b/cmd/bd/doctor/fix/testmain_cgo_test.go @@ -26,7 +26,9 @@ func testMainInner(m *testing.M) int { // database-name firewall when they connect to the spawned test server. os.Setenv("BEADS_TEST_SERVER", "1") if err := testutil.EnsureDoltContainerForTestMain(); err != nil { - fmt.Fprintf(os.Stderr, "WARN: %v, skipping Dolt tests\n", err) + if testutil.DoltUnavailableForTestMain(err) { + return 1 + } } else { defer testutil.TerminateDoltContainer() } diff --git a/cmd/bd/protocol/BUILD.bazel b/cmd/bd/protocol/BUILD.bazel index c53b22206..7cabc8522 100644 --- a/cmd/bd/protocol/BUILD.bazel +++ b/cmd/bd/protocol/BUILD.bazel @@ -92,9 +92,12 @@ sh_test( ":protocol_race_off", "//cmd/bd:bd_for_tests", ], + # Pinned to the container backend (the docker lane is the A/B control); + # rule env beats --test_env, so =local on the command line cannot flip it. env = { "BEADS_PROTOCOL_REQUIRE_DOLT": "1", "BEADS_TEST_BD_BINARY": "$(rlocationpath //cmd/bd:bd_for_tests)", + "BEADS_TEST_DOLT_SERVER": "container", }, # ~10 min unsharded without -race in pr.yml. shard_count = 8, diff --git a/cmd/bd/protocol/helpers_test.go b/cmd/bd/protocol/helpers_test.go index 1f254ce4d..ba0cc3fa5 100644 --- a/cmd/bd/protocol/helpers_test.go +++ b/cmd/bd/protocol/helpers_test.go @@ -53,7 +53,9 @@ func testMainInner(m *testing.M) int { // AD-01 (be-c5p): allow protocol tests to connect to the spawned test server. os.Setenv("BEADS_TEST_SERVER", "1") if err := testutil.EnsureDoltContainerForTestMain(); err != nil { - fmt.Fprintf(os.Stderr, "WARN: %v, skipping Dolt tests\n", err) + if testutil.DoltUnavailableForTestMain(err) { + return 1 + } } else { defer testutil.TerminateDoltContainer() testDoltPort = testutil.DoltContainerPortInt() diff --git a/cmd/bd/proxied_shared_harness_test.go b/cmd/bd/proxied_shared_harness_test.go index eeb23a0d9..cbaf3c5dd 100644 --- a/cmd/bd/proxied_shared_harness_test.go +++ b/cmd/bd/proxied_shared_harness_test.go @@ -35,6 +35,9 @@ func sharedProxiedServerPort(t *testing.T) int { sharedProxiedPort = port }) if sharedProxiedErr != nil { + if os.Getenv(testutil.EnvRequireDoltContainer) == "1" { + t.Fatalf("shared proxied-server unavailable: %v, but %s=1; this lane must not skip", sharedProxiedErr, testutil.EnvRequireDoltContainer) + } t.Skipf("shared proxied-server unavailable: %v", sharedProxiedErr) } return sharedProxiedPort diff --git a/cmd/bd/test_dolt_server_cgo_test.go b/cmd/bd/test_dolt_server_cgo_test.go index 40c60bb06..b62b3d8d8 100644 --- a/cmd/bd/test_dolt_server_cgo_test.go +++ b/cmd/bd/test_dolt_server_cgo_test.go @@ -38,7 +38,10 @@ func startTestDoltServer() func() { return func() { testutil.TerminateDoltContainer() } } if err := testutil.EnsureDoltContainerForTestMain(); err != nil { - fmt.Fprintf(os.Stderr, "WARN: %v, skipping Dolt tests\n", err) + if testutil.DoltUnavailableForTestMain(err) { + // Nothing has been started yet; there is nothing to clean up. + os.Exit(1) + } return func() {} } diff --git a/engdocs/TESTING.md b/engdocs/TESTING.md index aba1ea3b6..403e3142f 100644 --- a/engdocs/TESTING.md +++ b/engdocs/TESTING.md @@ -81,6 +81,24 @@ To skip an optional service explicitly, use the existing skip mechanism: BEADS_TEST_SKIP=dolt ./scripts/test.sh ./... ``` +Tests that need a Dolt SQL server get one from `internal/testutil` +(`EnsureDoltContainerForTestMain`, `RequireDoltContainer`, +`StartIsolatedDoltContainer[Handle]`, `NewContainerProvider`). Two backends +sit behind that API, selected by `BEADS_TEST_DOLT_SERVER`: + +- `container`: the `dolthub/dolt-sql-server` image through testcontainers + (needs docker and the pulled image). The default under plain `go test`. +- `local`: a `dolt sql-server` started by the test process from the pinned + dolt CLI (`BEADS_TEST_DOLT_BINARY`, else `dolt` on `PATH`; it must be the + image's version). No docker. Used only when explicitly selected, under + `go test` and `bazel test` alike (a Bazel target's `env`, or + `--test_env=BEADS_TEST_DOLT_SERVER=local`); unset means `container`, which + in a Bazel action without docker keeps the usual skip. + +`BEADS_TEST_REQUIRE_DOLT_CONTAINER=1` turns an unavailable backend into a +failure (per test and in every `TestMain`) instead of a skip; lanes that +exist to run the Dolt suites set it. + Tests that need a temporary repository or store should use `t.TempDir()` and `t.Cleanup()`. Temporary repositories must set a repository-local hooks path; do not inherit the developer's global hooks configuration. diff --git a/internal/beads/testmain_integration_test.go b/internal/beads/testmain_integration_test.go index 1fca8dca5..7251802c1 100644 --- a/internal/beads/testmain_integration_test.go +++ b/internal/beads/testmain_integration_test.go @@ -27,7 +27,10 @@ func setupIntegrationTestMain(root string) (func(), error) { os.Setenv("BEADS_TEST_SERVER", "1") if err := testutil.EnsureDoltContainerForTestMain(); err != nil { - fmt.Fprintf(os.Stderr, "WARN: %v, skipping Dolt tests\n", err) + if testutil.DoltUnavailableForTestMain(err) { + cleanup() + return nil, err + } return cleanup, nil } diff --git a/internal/molecules/testmain_test.go b/internal/molecules/testmain_test.go index 4b4bdf56d..c2ea60d24 100644 --- a/internal/molecules/testmain_test.go +++ b/internal/molecules/testmain_test.go @@ -29,7 +29,9 @@ func testMainInner(m *testing.M) int { // AD-01 (be-c5p): allow molecules tests to connect to the test container. os.Setenv("BEADS_TEST_SERVER", "1") if err := testutil.EnsureDoltContainerForTestMain(); err != nil { - fmt.Fprintf(os.Stderr, "WARN: %v, skipping Dolt tests\n", err) + if testutil.DoltUnavailableForTestMain(err) { + return 1 + } } else { defer testutil.TerminateDoltContainer() testServerPort = testutil.DoltContainerPortInt() diff --git a/internal/storage/dolt/dolt_benchmark_purge_test.go b/internal/storage/dolt/dolt_benchmark_purge_test.go index 4c6edda52..55a76489b 100644 --- a/internal/storage/dolt/dolt_benchmark_purge_test.go +++ b/internal/storage/dolt/dolt_benchmark_purge_test.go @@ -12,20 +12,21 @@ import ( "github.com/steveyegge/beads/internal/testutil" ) -// findDroppedDirCmd locates the .dolt_dropped_databases directory anywhere in -// the container. +// findDroppedDirCmd locates the .dolt_dropped_databases directory under the +// server's data directory. // -// It deliberately does NOT use `find -xdev`. The Dolt image declares -// VOLUME /var/lib/dolt (`docker image inspect dolthub/dolt-sql-server:2.2.0` -// -> `{"/var/lib/dolt":{}}`), so the data dir is a *separate mount* from `/`: -// inside the container `stat -c %d / /var/lib/dolt` reports different device -// IDs. `-xdev` stops the walk dead at that boundary, so it never reaches the -// data dir at all โ€” the probe returned "" on every run, the entry count was -// always 0, and the leak assertion below was vacuously true whether or not -// dropBenchDB purged anything (PR #5792 review, finding 1). The pseudo -// filesystems `-xdev` was there to skip are pruned explicitly instead. -const findDroppedDirCmd = `find / -maxdepth 6 \( -path /proc -o -path /sys -o -path /dev \) -prune ` + - `-o -type d -name .dolt_dropped_databases -print 2>/dev/null` +// It is relative on purpose: IsolatedDoltContainer.Exec runs with the data +// directory as its working directory in both test server backends (the +// container's WORKDIR is its data dir, /var/lib/dolt; the local backend runs +// the command on the host in the local server's data dir). A walk from `/` +// would, with the local backend, search the host instead of one server's data. +// +// Walking from `.` also sidesteps the trap an earlier `find / -xdev` fell into: +// the image declares VOLUME /var/lib/dolt, a separate mount from `/`, and +// `-xdev` stopped at that boundary, so the probe returned "" on every run and +// the leak assertion below was vacuously true (PR #5792 review, finding 1). +// requireLeakIsObservable (the positive control) guards against a blind probe. +const findDroppedDirCmd = `find . -maxdepth 4 -type d -name .dolt_dropped_databases -print 2>/dev/null` // TestBenchDBPurgeDoesNotLeak is the regression gate for be-pq5: dropBenchDB // must DROP and then PURGE so the dropped-databases dir does not grow across @@ -35,8 +36,8 @@ const findDroppedDirCmd = `find / -maxdepth 6 \( -path /proc -o -path /sys -o -p // // Dolt 1.86 exposes no SQL view for the dropped-databases list, so the only // way to detect a leak is to count entries in the server's -// .dolt_dropped_databases/ directory, which has no host-visible path โ€” hence -// reading it by exec'ing into the container. +// .dolt_dropped_databases/ directory, which with the container backend has no +// host-visible path โ€” hence reading it through IsolatedDoltContainer.Exec. // // This runs against its own isolated container, not the shared TestMain one. // Six sites in this package refuse to DROP DATABASE against the shared diff --git a/internal/storage/dolt/testmain_test.go b/internal/storage/dolt/testmain_test.go index 02f367f74..dc261aa5e 100644 --- a/internal/storage/dolt/testmain_test.go +++ b/internal/storage/dolt/testmain_test.go @@ -132,7 +132,9 @@ func testMainInner(m *testing.M) int { defer os.Unsetenv(testCircuitBreakerDirEnv) } if err := testutil.EnsureDoltContainerForTestMain(); err != nil { - fmt.Fprintf(os.Stderr, "WARN: %v, skipping Dolt tests\n", err) + if testutil.DoltUnavailableForTestMain(err) { + return 1 + } } else { defer testutil.TerminateDoltContainer() testServerPort = testutil.DoltContainerPortInt() diff --git a/internal/storage/domain/db/BUILD.bazel b/internal/storage/domain/db/BUILD.bazel index fe51e3993..78e0eb5eb 100644 --- a/internal/storage/domain/db/BUILD.bazel +++ b/internal/storage/domain/db/BUILD.bazel @@ -140,7 +140,12 @@ sh_test( srcs = ["//tools/bazel:go_test_variant.sh"], args = ["$(rootpath :db_test)"], data = [":db_test"], - env = {"BEADS_TEST_REQUIRE_DOLT_CONTAINER": "1"}, + # Pinned to the container backend (the docker lane is the A/B control); + # rule env beats --test_env, so =local on the command line cannot flip it. + env = { + "BEADS_TEST_DOLT_SERVER": "container", + "BEADS_TEST_REQUIRE_DOLT_CONTAINER": "1", + }, tags = [ "no-remote-exec", "requires-docker", diff --git a/internal/storage/uow/BUILD.bazel b/internal/storage/uow/BUILD.bazel index 0862a9118..3389c8051 100644 --- a/internal/storage/uow/BUILD.bazel +++ b/internal/storage/uow/BUILD.bazel @@ -204,8 +204,11 @@ sh_test( ":uow_test", "//cmd/bd:bd_for_tests", ], + # Pinned to the container backend (the docker lane is the A/B control); + # rule env beats --test_env, so =local on the command line cannot flip it. env = { "BEADS_TEST_BD_BINARY": "$(rlocationpath //cmd/bd:bd_for_tests)", + "BEADS_TEST_DOLT_SERVER": "container", "BEADS_TEST_REQUIRE_DOLT_CONTAINER": "1", }, # ~12 min unsharded under -race on a loaded host (large allows 15). diff --git a/internal/testutil/BUILD.bazel b/internal/testutil/BUILD.bazel index 1aa4cc278..e71085878 100644 --- a/internal/testutil/BUILD.bazel +++ b/internal/testutil/BUILD.bazel @@ -7,6 +7,9 @@ go_library( "container_provider_windows.go", "dockercontext.go", "git_test_hooks.go", + "localdoltserver.go", + "localdoltserver_linux.go", + "localdoltserver_other.go", "suite_temp.go", "testdoltbranch.go", "testdoltcommon.go", @@ -104,6 +107,7 @@ go_test( name = "testutil_test", srcs = [ "dockercontext_test.go", + "localdoltserver_test.go", "suite_temp_test.go", "testdoltbranch_test.go", "testdoltserver_env_test.go", diff --git a/internal/testutil/container_provider.go b/internal/testutil/container_provider.go index 39f887cff..b6573eb41 100644 --- a/internal/testutil/container_provider.go +++ b/internal/testutil/container_provider.go @@ -18,6 +18,7 @@ import ( // mapped host port, and Stop() to tear it down. type ContainerProvider struct { container *dolt.DoltContainer + local *localDoltServer // local backend instead of container port int } @@ -27,6 +28,14 @@ func NewContainerProvider() (*ContainerProvider, error) { return nil, fmt.Errorf("cannot create container provider: %s", state) } + if useLocalDoltServer() { + s, err := startLocalDoltServer() + if err != nil { + return nil, fmt.Errorf("starting local Dolt server: %w", err) + } + return &ContainerProvider{local: s, port: s.Port()}, nil + } + ctx, cancel := context.WithTimeout(context.Background(), serverStartTimeout) defer cancel() @@ -70,6 +79,11 @@ func (p *ContainerProvider) WritePortFile(serverDir string) error { // Stop terminates the container. func (p *ContainerProvider) Stop() error { + if p.local != nil { + p.local.terminate() + p.local = nil + return nil + } if p.container == nil { return nil } diff --git a/internal/testutil/localdoltserver.go b/internal/testutil/localdoltserver.go new file mode 100644 index 000000000..8b7fd5c7f --- /dev/null +++ b/internal/testutil/localdoltserver.go @@ -0,0 +1,602 @@ +//go:build !windows + +package testutil + +import ( + "bytes" + "context" + "database/sql" + "errors" + "fmt" + "io" + "os" + "os/exec" + "path/filepath" + "regexp" + "strconv" + "strings" + "sync" + "syscall" + "time" +) + +// EnvDoltServerBackend selects where the test Dolt SQL server comes from: +// +// container the dolthub/dolt-sql-server image through testcontainers +// (needs a docker daemon and the pulled image) +// local `dolt sql-server` from the pinned dolt CLI, started by the test +// process itself (no docker; works under remote execution) +// +// Unset means container, under plain `go test` and `bazel test` alike. A +// Bazel action has no docker daemon, so there an unset target keeps the +// pre-existing "Dolt unavailable, skipping" behavior; only targets (or +// invocations, via --test_env) that set local explicitly start servers from +// the pinned CLI that tools/bazel/test_env.sh puts on PATH. Defaulting to +// local under Bazel would light up every Dolt suite that no lane runs with a +// server today in every non-ci `bazel test`. +const EnvDoltServerBackend = "BEADS_TEST_DOLT_SERVER" + +// EnvDoltServerGOMAXPROCS caps the CPUs a local test server uses. The +// container jobs run on 4-vCPU GitHub runners; a remote worker runs many test +// actions side by side, and an uncapped Go server sizes itself for the whole +// machine. +const EnvDoltServerGOMAXPROCS = "BEADS_TEST_DOLT_SERVER_GOMAXPROCS" + +const defaultLocalServerGOMAXPROCS = "4" + +// EnvDoltServerVerbose, set to "1", makes the local backend report each +// server's port and startup time on stderr. +const EnvDoltServerVerbose = "BEADS_TEST_DOLT_SERVER_VERBOSE" + +// useLocalDoltServer reports whether the local backend is selected. An +// unrecognized BEADS_TEST_DOLT_SERVER value is reported by checkDolt (see +// doltBackendErr) rather than guessed at. +func useLocalDoltServer() bool { + return os.Getenv(EnvDoltServerBackend) == "local" +} + +// doltBackendErr rejects a BEADS_TEST_DOLT_SERVER value that names neither +// backend: a typo must not silently fall back to one of them. +func doltBackendErr() error { + switch v := os.Getenv(EnvDoltServerBackend); v { + case "", "local", "container": + return nil + default: + return fmt.Errorf("%s=%q: want \"local\" or \"container\"", EnvDoltServerBackend, v) + } +} + +// localServerBase is where local servers keep their state. It is captured at +// package init, before any TestMain runs PinSuiteTempRoot (which re-points +// TMPDIR): a server under a suite root would be reported, and killed, as a +// leaked fixture by doltserver.SweepSuiteTestServers. The container backend +// kept its data outside the host filesystem entirely; this keeps it outside +// every suite root. +var localServerBase = os.TempDir() + +// pinnedDoltVersion is the version in DoltDockerImage's tag: the local +// backend must run exactly the release the container would. +var pinnedDoltVersion = func() string { + _, tag, _ := strings.Cut(DoltDockerImage, ":") + return tag +}() + +var ( + localDoltBinOnce sync.Once + localDoltBin string + localDoltBinErr error +) + +var doltVersionRe = regexp.MustCompile(`(?m)^dolt version (\S+)`) + +// resolveLocalDoltBinary finds the dolt CLI (BEADS_TEST_DOLT_BINARY, then +// PATH) and checks it is the pinned release. +func resolveLocalDoltBinary() (string, error) { + localDoltBinOnce.Do(func() { + sweepDeadLocalServerRoots(localServerBase, processAlive) + bin := os.Getenv("BEADS_TEST_DOLT_BINARY") + if bin == "" { + p, err := exec.LookPath("dolt") + if err != nil { + localDoltBinErr = fmt.Errorf("dolt CLI not found: %w", err) + return + } + bin = p + } + got, err := probeDoltVersion(bin, localServerBase) + if err != nil { + localDoltBinErr = err + return + } + if got != pinnedDoltVersion { + localDoltBinErr = fmt.Errorf("dolt CLI %s is version %s, want %s (the %s image the container backend runs)", + bin, got, pinnedDoltVersion, DoltDockerImage) + return + } + localDoltBin = bin + }) + return localDoltBin, localDoltBinErr +} + +// doltNoEventFlushEnv stops every dolt command from forking a detached +// `dolt send-metrics` child on exit. The *.disabled config keys below stop +// the network calls but not that child, which outlives its parent, re-creates +// /.dolt after the owner removed it, and races terminate's RemoveAll. +const doltNoEventFlushEnv = "DOLT_DISABLE_EVENT_FLUSH=1" + +// probeDoltVersion runs `bin version` with a throwaway HOME under base (so +// the user's ~/.dolt is neither read nor written) and returns the version. +func probeDoltVersion(bin, base string) (string, error) { + home, err := os.MkdirTemp(base, "bdt-doltver-") + if err != nil { + return "", err + } + defer func() { _ = os.RemoveAll(home) }() + _ = writeLocalRootOwner(home) + if err := writeLocalDoltGlobalConfig(home); err != nil { + return "", err + } + cmd := exec.Command(bin, "version") // #nosec G204 G702 -- test-only; bin is the pinned dolt CLI + cmd.Env = []string{"HOME=" + home, "DOLT_ROOT_PATH=" + home, "PATH=" + os.Getenv("PATH"), doltNoEventFlushEnv} + out, err := cmd.CombinedOutput() + if err != nil { + return "", fmt.Errorf("%s version: %w: %s", bin, err, out) + } + m := doltVersionRe.FindSubmatch(out) + if m == nil { + return "", fmt.Errorf("%s version: unrecognized output %q", bin, out) + } + return string(m[1]), nil +} + +// writeLocalDoltGlobalConfig writes the server's dolt global config. The +// image ships an empty one ({}); the two keys here only turn off usage +// events and the release check, which are network calls, not behavior. +func writeLocalDoltGlobalConfig(root string) error { + dir := filepath.Join(root, ".dolt") + if err := os.MkdirAll(dir, 0o700); err != nil { + return err + } + return os.WriteFile(filepath.Join(dir, "config_global.json"), + []byte(`{"metrics.disabled":"true","versioncheck.disabled":"true"}`+"\n"), 0o600) +} + +// localDoltServer is one `dolt sql-server` owned by this test process. +type localDoltServer struct { + bin string + root string // owned tree: data/, home/, sfp/, server.yaml, server.log, owner.pid + dataDir string + logPath string + + mu sync.Mutex + port int + cmd *exec.Cmd + done chan struct{} // closed when cmd exits + waitErr error +} + +const ( + localServerStartAttempts = 5 + localServerStopTimeout = 10 * time.Second +) + +// startLocalDoltServer creates a fresh data directory and starts a server on +// it, provisioned like the container: root@% with no password, database +// beads_test, user test/test with USAGE and ALL ON beads_test.*. +func startLocalDoltServer() (*localDoltServer, error) { + began := time.Now() + s, err := newLocalDoltServer() + if err != nil { + return nil, err + } + if err := s.start(0); err != nil { + _ = os.RemoveAll(s.root) + return nil, err + } + if err := s.provision(); err != nil { + s.terminate() + return nil, err + } + if os.Getenv(EnvDoltServerVerbose) == "1" { + fmt.Fprintf(os.Stderr, "testutil: local dolt sql-server on 127.0.0.1:%d (state %s) ready+provisioned in %s\n", + s.Port(), s.root, time.Since(began).Round(time.Millisecond)) + } + return s, nil +} + +// newLocalDoltServer creates the state tree of a server that is not started +// yet: data/ (the server's working directory, a multi-database root like the +// image's /var/lib/dolt), home/ (its HOME and DOLT_ROOT_PATH), sfp/ (its +// secure_file_priv), server.log and the owner marker. +func newLocalDoltServer() (*localDoltServer, error) { + bin, err := resolveLocalDoltBinary() + if err != nil { + return nil, err + } + root, err := os.MkdirTemp(localServerBase, "bdt-dolt-") + if err != nil { + return nil, err + } + s := &localDoltServer{ + bin: bin, + root: root, + dataDir: filepath.Join(root, "data"), + logPath: filepath.Join(root, "server.log"), + } + if err := writeLocalRootOwner(root); err != nil { + _ = os.RemoveAll(root) + return nil, err + } + for _, d := range []string{s.dataDir, s.secureFilePrivDir()} { + if err := os.MkdirAll(d, 0o700); err != nil { + _ = os.RemoveAll(root) + return nil, err + } + } + if err := writeLocalDoltGlobalConfig(filepath.Join(root, "home")); err != nil { + _ = os.RemoveAll(root) + return nil, err + } + return s, nil +} + +// start launches the server on preferPort, or on a fresh loopback port when +// preferPort is 0 or taken. dolt sql-server rejects port 0, so the port is +// picked here and the rare bind race (another process takes it between the +// probe and dolt's bind) is retried with a new one. +func (s *localDoltServer) start(preferPort int) error { + var lastErr error + for attempt := 0; attempt < localServerStartAttempts; attempt++ { + port := preferPort + if attempt > 0 || port == 0 { + p, err := FindFreePort() + if err != nil { + return err + } + port = p + } + if port == 3307 { // productionPortReasons Rule 1: never the default port + continue + } + err := s.launch(port) + if err == nil { + return nil + } + lastErr = err + if !errors.Is(err, errLocalServerBind) { + return err + } + } + return fmt.Errorf("dolt sql-server did not start after %d attempts: %w", localServerStartAttempts, lastErr) +} + +var errLocalServerBind = errors.New("port already in use") + +func (s *localDoltServer) launch(port int) error { + logf, err := os.OpenFile(s.logPath, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0o600) + if err != nil { + return err + } + logStart, _ := logf.Seek(0, io.SeekEnd) + gomaxprocs := os.Getenv(EnvDoltServerGOMAXPROCS) + if gomaxprocs == "" { + gomaxprocs = defaultLocalServerGOMAXPROCS + } + home := filepath.Join(s.root, "home") + cfg, err := s.writeServerConfig(port) + if err != nil { + _ = logf.Close() + return err + } + // #nosec G204 -- test-only, pinned binary, loopback listener + cmd := exec.Command(s.bin, "sql-server", "--config", cfg) + cmd.Dir = s.dataDir + // A clean environment, like the container's: nothing from the test + // (BEADS_*, DOLT_* overrides, the wrapper's dolt identity) reaches the + // server. DOLT_ROOT_HOST=% creates root@% on first start, as + // testcontainers asks the image to. + cmd.Env = []string{ + "PATH=" + os.Getenv("PATH"), + "HOME=" + home, + "DOLT_ROOT_PATH=" + home, + "DOLT_ROOT_HOST=%", + "TZ=UTC", + "GOMAXPROCS=" + gomaxprocs, + doltNoEventFlushEnv, + } + cmd.Stdout = logf + cmd.Stderr = logf + // Same process group as the test: `bazel test` (test-setup.sh) kills the + // group when the test exits or times out, which reaps a server that every + // cleanup missed. Pdeathsig covers plain `go test`. + cmd.SysProcAttr = localServerSysProcAttr() + if err := cmd.Start(); err != nil { + _ = logf.Close() + return fmt.Errorf("start dolt sql-server: %w", err) + } + done := make(chan struct{}) + s.mu.Lock() + s.cmd, s.done, s.port, s.waitErr = cmd, done, port, nil + s.mu.Unlock() + go func() { + err := cmd.Wait() + _ = logf.Close() + s.mu.Lock() + s.waitErr = err + s.mu.Unlock() + close(done) + }() + + if err := s.waitReady(port, done, logStart); err != nil { + s.kill() + return err + } + return nil +} + +func (s *localDoltServer) secureFilePrivDir() string { return filepath.Join(s.root, "sfp") } + +// writeServerConfig writes the server's --config file for port. dolt ignores +// every other flag when --config is given, so it carries the listener too. +// Its only other setting is secure_file_priv: the image runs with "" (any +// path), which inside a container reaches only the container's filesystem, +// but a local server runs as the test user on the host, where passwordless +// root@% plus LOAD_FILE / INTO OUTFILE would read and write the user's +// files from any process that can reach the port. Confining it to sfp/ is the +// one deliberate difference from the container (TestDoltServerFingerprint +// normalizes it). +func (s *localDoltServer) writeServerConfig(port int) (string, error) { + path := filepath.Join(s.root, "server.yaml") + cfg := fmt.Sprintf("listener:\n host: 127.0.0.1\n port: %d\nsystem_variables:\n secure_file_priv: %s\n", + port, strconv.Quote(s.secureFilePrivDir())) + return path, os.WriteFile(path, []byte(cfg), 0o600) +} + +// localServerReadyLine is what dolt sql-server logs once its listener is +// bound (the line testcontainers' dolt module waits for, too). +const localServerReadyLine = "Server ready. Accepting connections." + +// waitReady waits until this process has logged that it is listening and +// answers a bounded ping (see waitForDoltReady), and notices an early exit, +// classifying a bind failure so start can retry on another port. The log line +// is what makes the ping trustworthy: if another process owns the port, dolt +// fails to bind and exits, but a ping could still reach the other listener. +func (s *localDoltServer) waitReady(port int, done chan struct{}, logStart int64) error { + dsn := fmt.Sprintf("root@tcp(127.0.0.1:%d)/", port) + deadline := time.Now().Add(serverStartTimeout) + lastErr := errors.New("server has not logged " + strconv.Quote(localServerReadyLine)) + for time.Now().Before(deadline) { + select { + case <-done: + tail := s.logSince(logStart) + // dolt 2.2.0 checks the port itself ("Port N already in use."); + // a bind that loses the race after that check reports the + // kernel's "address already in use". + if strings.Contains(tail, "already in use") { + return fmt.Errorf("%w: port %d: %s", errLocalServerBind, port, tail) + } + return fmt.Errorf("dolt sql-server exited during startup on port %d: %s", port, tail) + default: + } + if bytes.Contains(s.readLogSince(logStart), []byte(localServerReadyLine)) { + if lastErr = pingDoltOnce(dsn); lastErr == nil { + return nil + } + } + time.Sleep(25 * time.Millisecond) + } + return fmt.Errorf("dolt sql-server on port %d not query-ready within %s: %v; log: %s", + port, serverStartTimeout, lastErr, s.logSince(logStart)) +} + +// readLogSince returns the server log from byte offset off on. +func (s *localDoltServer) readLogSince(off int64) []byte { + b, err := os.ReadFile(s.logPath) + if err != nil || int64(len(b)) < off { + return nil + } + return b[off:] +} + +// logSince returns the last 4 KiB of the server log from byte offset off on, +// for error messages. +func (s *localDoltServer) logSince(off int64) string { + b := bytes.TrimSpace(s.readLogSince(off)) + if len(b) > 4096 { + b = b[len(b)-4096:] + } + return string(b) +} + +// provision replays what testcontainers' dolt module and the image +// entrypoint do after the server is up. +func (s *localDoltServer) provision() error { + db, err := sql.Open("mysql", fmt.Sprintf("root@tcp(127.0.0.1:%d)/", s.Port())) + if err != nil { + return err + } + defer db.Close() + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + for _, stmt := range []string{ + "CREATE DATABASE IF NOT EXISTS `beads_test`", + "CREATE USER IF NOT EXISTS 'test'@'%' IDENTIFIED BY 'test'", + "GRANT USAGE ON *.* TO 'test'@'%'", + "GRANT ALL ON `beads_test`.* TO 'test'@'%'", + } { + if _, err := db.ExecContext(ctx, stmt); err != nil { + return fmt.Errorf("provision local dolt server: %s: %w", stmt, err) + } + } + return nil +} + +// Port returns the current listening port. +func (s *localDoltServer) Port() int { + s.mu.Lock() + defer s.mu.Unlock() + return s.port +} + +// exited reports whether the server process has exited. +func (s *localDoltServer) exited() (bool, error) { + s.mu.Lock() + done, werr := s.done, s.waitErr + s.mu.Unlock() + if done == nil { + return true, nil + } + select { + case <-done: + s.mu.Lock() + werr = s.waitErr + s.mu.Unlock() + return true, werr + default: + return false, nil + } +} + +// stop asks the server to exit (SIGTERM, then SIGKILL after +// localServerStopTimeout) and keeps its data directory. +func (s *localDoltServer) stop() error { + s.mu.Lock() + cmd, done := s.cmd, s.done + s.mu.Unlock() + if cmd == nil || cmd.Process == nil { + return nil + } + select { + case <-done: + return nil + default: + } + _ = cmd.Process.Signal(syscall.SIGTERM) + select { + case <-done: + return nil + case <-time.After(localServerStopTimeout): + _ = cmd.Process.Kill() + <-done + return fmt.Errorf("dolt sql-server (pid %d) ignored SIGTERM for %s; killed", cmd.Process.Pid, localServerStopTimeout) + } +} + +func (s *localDoltServer) kill() { + s.mu.Lock() + cmd, done := s.cmd, s.done + s.mu.Unlock() + if cmd == nil || cmd.Process == nil { + return + } + _ = cmd.Process.Kill() + <-done +} + +// restart starts the server again on its existing data directory, on the +// same port when it is still free. +func (s *localDoltServer) restart() error { + if gone, _ := s.exited(); !gone { + return fmt.Errorf("dolt sql-server is still running") + } + return s.start(s.Port()) +} + +// terminate stops the server and removes everything it owned. +func (s *localDoltServer) terminate() { + _ = s.stop() + _ = os.RemoveAll(s.root) +} + +// execInDataDir runs cmd on the host with the server's data directory as its +// working directory: the local counterpart of exec'ing into the container, +// whose working directory is its data directory (/var/lib/dolt). +func (s *localDoltServer) execInDataDir(ctx context.Context, argv []string) (int, string, error) { + if len(argv) == 0 { + return 0, "", fmt.Errorf("empty command") + } + cmd := exec.CommandContext(ctx, argv[0], argv[1:]...) // #nosec G204 -- test-only + cmd.Dir = s.dataDir + out, err := cmd.CombinedOutput() + var ee *exec.ExitError + if errors.As(err, &ee) { + return ee.ExitCode(), string(out), nil + } + if err != nil { + return 0, "", err + } + return 0, string(out), nil +} + +// Local state roots are removed by terminate, but nothing runs it when the +// test binary dies (panic, -timeout, SIGKILL) or when a package's TestMain +// never calls TerminateDoltContainer: the container backend has Ryuk for +// that, the local backend has these. Every root records its owning PID, and +// the first use of the backend in a later process removes the roots whose +// owner is gone (the doltserver.SweepDeadSuiteRoots rule, copied because +// doltserver's own tests import this package). +const ( + localRootOwnerFile = "owner.pid" + localServerRootGlob = "bdt-dolt-*" + localVersionRootGlob = "bdt-doltver-*" +) + +func writeLocalRootOwner(root string) error { + return os.WriteFile(filepath.Join(root, localRootOwnerFile), []byte(strconv.Itoa(os.Getpid())+"\n"), 0o600) +} + +// readLocalRootOwner returns the PID in root's owner marker; ok is false +// when there is none or it does not hold a PID, which means "owner unknown", +// never "owner dead". +func readLocalRootOwner(root string) (pid int, ok bool) { + b, err := os.ReadFile(filepath.Join(root, localRootOwnerFile)) // #nosec G304 -- fixed name under a root this user owns + if err != nil { + return 0, false + } + pid, err = strconv.Atoi(strings.TrimSpace(string(b))) + if err != nil || pid <= 0 { + return 0, false + } + return pid, true +} + +// processAlive reports whether pid names a running process (EPERM: it +// exists but belongs to someone else, so it is alive too). +func processAlive(pid int) bool { + err := syscall.Kill(pid, 0) + return err == nil || errors.Is(err, syscall.EPERM) +} + +// sweepDeadLocalServerRoots removes base's local-backend state roots whose +// owner marker names a dead process and returns them. A root without a +// marker (a pre-marker leftover, or one whose owner is still between +// MkdirTemp and writing it), with a live owner, not a real directory, or not +// owned by this user is left alone. Only directories are reclaimed: on +// Linux Pdeathsig already killed the dead owner's servers; elsewhere a +// surviving server keeps serving its deleted directory until it is killed. +func sweepDeadLocalServerRoots(base string, alive func(int) bool) []string { + var swept []string + for _, glob := range []string{localServerRootGlob, localVersionRootGlob} { + matches, _ := filepath.Glob(filepath.Join(base, glob)) + for _, root := range matches { + info, err := os.Lstat(root) + if err != nil || !info.IsDir() { + continue + } + if st, ok := info.Sys().(*syscall.Stat_t); !ok || int(st.Uid) != os.Getuid() { + continue + } + pid, ok := readLocalRootOwner(root) + if !ok || alive(pid) { + continue + } + if os.RemoveAll(root) == nil { + swept = append(swept, root) + } + } + } + if len(swept) > 0 && os.Getenv(EnvDoltServerVerbose) == "1" { + fmt.Fprintf(os.Stderr, "testutil: removed %d local dolt server state dir(s) of dead test processes: %v\n", len(swept), swept) + } + return swept +} diff --git a/internal/testutil/localdoltserver_linux.go b/internal/testutil/localdoltserver_linux.go new file mode 100644 index 000000000..45b88b86c --- /dev/null +++ b/internal/testutil/localdoltserver_linux.go @@ -0,0 +1,14 @@ +//go:build linux + +package testutil + +import "syscall" + +// localServerSysProcAttr keeps the server in the test's process group and +// asks the kernel to kill it if the thread that started it dies. Pdeathsig +// is tied to that OS thread, not the process, so it is a safety net only; +// the owner's Stop/Terminate and Bazel's process-group kill are the real +// reapers. +func localServerSysProcAttr() *syscall.SysProcAttr { + return &syscall.SysProcAttr{Pdeathsig: syscall.SIGKILL} +} diff --git a/internal/testutil/localdoltserver_other.go b/internal/testutil/localdoltserver_other.go new file mode 100644 index 000000000..eb08c3803 --- /dev/null +++ b/internal/testutil/localdoltserver_other.go @@ -0,0 +1,11 @@ +//go:build !linux && !windows + +package testutil + +import "syscall" + +// localServerSysProcAttr keeps the server in the test's process group (no +// Pdeathsig outside Linux). +func localServerSysProcAttr() *syscall.SysProcAttr { + return &syscall.SysProcAttr{} +} diff --git a/internal/testutil/localdoltserver_test.go b/internal/testutil/localdoltserver_test.go new file mode 100644 index 000000000..b56ade483 --- /dev/null +++ b/internal/testutil/localdoltserver_test.go @@ -0,0 +1,608 @@ +//go:build !windows + +package testutil + +import ( + "context" + "database/sql" + "errors" + "fmt" + "net" + "os" + "path/filepath" + "runtime" + "sort" + "strconv" + "strings" + "syscall" + "testing" + "time" +) + +// serverFingerprint is what a test can observe about "the Dolt test server" +// over SQL. Both backends must produce wantServerFingerprint: the container +// because that is what CI has always run against, the local backend because +// it has to be indistinguishable from it. A Dolt or image bump that changes +// any of these defaults fails here first (design D6 ยง2). +type serverFingerprint struct { + Users []string // user@host, sorted + TestGrants []string // SHOW GRANTS FOR 'test'@'%' + Databases []string // SHOW DATABASES, sorted + CurrentUser string // CURRENT_USER() when connecting as root + Version string + MaxConns string + SQLMode string + SystemTZ string + TimeZone string + Autocommit string + SecureFilePv string // normalized, see fingerprintServer + CharsetSrv string + CollationSrv string + MaxPacket string + TxIsolation string + LowerCaseTbl string + LowerCaseFS string // normalized off Linux, see fingerprintServer + WaitTimeout string + LockWait string // innodb_lock_wait_timeout +} + +var wantServerFingerprint = serverFingerprint{ + Users: []string{ + "__dolt_local_user__@localhost", + "event_scheduler@localhost", + "root@%", + "test@%", + }, + TestGrants: []string{ + "GRANT USAGE ON *.* TO `test`@`%`", + "GRANT SELECT, INSERT, UPDATE, DELETE, CREATE, DROP, REFERENCES, INDEX, ALTER, CREATE TEMPORARY TABLES, LOCK TABLES, EXECUTE, CREATE VIEW, SHOW VIEW, CREATE ROUTINE, ALTER ROUTINE, EVENT, TRIGGER ON `beads_test`.* TO `test`@`%`", + }, + Databases: []string{"beads_test", "information_schema", "mysql"}, + CurrentUser: "root@%", + Version: "8.0.31", + MaxConns: "151", + SQLMode: "NO_ENGINE_SUBSTITUTION,ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES", + SystemTZ: "UTC", + TimeZone: "SYSTEM", + Autocommit: "1", + SecureFilePv: "", + CharsetSrv: "utf8mb4", + CollationSrv: "utf8mb4_0900_bin", + MaxPacket: "1073741824", + TxIsolation: "REPEATABLE-READ", + LowerCaseTbl: "0", + LowerCaseFS: "0", + WaitTimeout: "28800", + LockWait: "1", +} + +func queryStrings(ctx context.Context, db *sql.DB, q string) ([]string, error) { + rows, err := db.QueryContext(ctx, q) + if err != nil { + return nil, fmt.Errorf("%s: %w", q, err) + } + defer rows.Close() + var out []string + for rows.Next() { + var s sql.NullString + if err := rows.Scan(&s); err != nil { + return nil, fmt.Errorf("%s: %w", q, err) + } + out = append(out, s.String) + } + return out, rows.Err() +} + +// fingerprintServer reads the fingerprint of the server on port. secureDir +// is the secure_file_priv the backend sets on purpose: the container's is "" +// (it can only reach the container's own filesystem), the local backend's is +// its state root's sfp/ (it runs on the host; see writeServerConfig). That +// value is reported as "" so both backends compare against one fingerprint. +func fingerprintServer(t *testing.T, port, secureDir string) serverFingerprint { + t.Helper() + db, err := sql.Open("mysql", fmt.Sprintf("root@tcp(127.0.0.1:%s)/", port)) + if err != nil { + t.Fatal(err) + } + defer db.Close() + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + var fp serverFingerprint + must := func(v []string, err error) []string { + t.Helper() + if err != nil { + t.Fatal(err) + } + return v + } + fp.Users = must(queryStrings(ctx, db, "SELECT CONCAT(user, '@', host) FROM mysql.user ORDER BY 1")) + fp.TestGrants = must(queryStrings(ctx, db, "SHOW GRANTS FOR 'test'@'%'")) + fp.Databases = must(queryStrings(ctx, db, "SHOW DATABASES")) + sort.Strings(fp.Databases) + row := db.QueryRowContext(ctx, "SELECT CURRENT_USER(), @@version, @@max_connections, @@sql_mode, "+ + "@@system_time_zone, @@time_zone, @@autocommit, @@secure_file_priv, "+ + "@@character_set_server, @@collation_server, @@max_allowed_packet, @@transaction_isolation, "+ + "@@lower_case_table_names, @@lower_case_file_system, @@wait_timeout, @@innodb_lock_wait_timeout") + var secure sql.NullString + if err := row.Scan(&fp.CurrentUser, &fp.Version, &fp.MaxConns, &fp.SQLMode, + &fp.SystemTZ, &fp.TimeZone, &fp.Autocommit, &secure, + &fp.CharsetSrv, &fp.CollationSrv, &fp.MaxPacket, &fp.TxIsolation, + &fp.LowerCaseTbl, &fp.LowerCaseFS, &fp.WaitTimeout, &fp.LockWait); err != nil { + t.Fatalf("server variables: %v", err) + } + fp.SecureFilePv = secure.String + if secureDir != "" && filepath.Clean(fp.SecureFilePv) == filepath.Clean(secureDir) { + fp.SecureFilePv = "" + } + // lower_case_file_system describes the filesystem under the data + // directory: the container's is Linux, a local server on macOS sits on + // case-insensitive APFS and reports ON. Only Linux hosts compare it. + if runtime.GOOS != "linux" { + fp.LowerCaseFS = wantServerFingerprint.LowerCaseFS + } + return fp +} + +func checkFingerprint(t *testing.T, got serverFingerprint) { + t.Helper() + if g, w := fmt.Sprintf("%+q", got), fmt.Sprintf("%+q", wantServerFingerprint); g != w { + t.Errorf("Dolt test server fingerprint changed\n got: %s\nwant: %s", g, w) + } +} + +// TestDoltServerFingerprint checks every backend available on this host +// against the same checked-in fingerprint, so container and local servers +// are compared with each other through it. The backend BEADS_TEST_DOLT_SERVER +// selects obeys the usual skip/fail rules; the other one is checked only when +// it happens to be available. +func TestDoltServerFingerprint(t *testing.T) { + if hasTestSkip("dolt") { + t.Skip("skipping: Dolt tests skipped (BEADS_TEST_SKIP=dolt)") + } + selectedLocal := useLocalDoltServer() + + t.Run("container", func(t *testing.T) { + if !selectedLocal { + if state := checkDolt(); state != doltReady { + skipOrFailDoltUnavailable(t, state) + } + } else if !isDockerAvailable() || !isDoltImageCached() { + t.Skipf("container backend not available on this host (docker + %s)", DoltDockerImage) + } + c := startIsolatedDoltContainer(t) + checkFingerprint(t, fingerprintServer(t, c.Port, "")) + }) + + t.Run("local", func(t *testing.T) { + skipLocalServerOffLinuxUnlessSelected(t) + if selectedLocal { + if state := checkDolt(); state != doltReady { + skipOrFailDoltUnavailable(t, state) + } + } else if _, err := resolveLocalDoltBinary(); err != nil { + t.Skipf("local backend not available on this host: %v", err) + } + c := startIsolatedLocalDoltServer(t) + checkFingerprint(t, fingerprintServer(t, c.Port, c.local.secureFilePrivDir())) + }) +} + +// skipLocalServerOffLinuxUnlessSelected keeps the local backend's own tests +// off non-Linux hosts unless BEADS_TEST_DOLT_SERVER=local asks for them. The +// macOS CI lanes run `go test -short ./...` with the pinned dolt installed; +// without Pdeathsig (Linux only) a test binary killed mid-test there would +// orphan its servers on the runner, and the backend is validated on Linux. +func skipLocalServerOffLinuxUnlessSelected(t *testing.T) { + t.Helper() + if runtime.GOOS != "linux" && !useLocalDoltServer() { + t.Skipf("skipping local dolt sql-server test on %s (no Pdeathsig); set %s=local to run it", + runtime.GOOS, EnvDoltServerBackend) + } +} + +// requireLocalDoltCLI skips (or, under BEADS_TEST_REQUIRE_DOLT_CONTAINER=1, +// fails) a test of the local backend itself when the pinned dolt CLI is +// missing. These tests run whichever backend is selected. +func requireLocalDoltCLI(t *testing.T) { + t.Helper() + skipLocalServerOffLinuxUnlessSelected(t) + if hasTestSkip("dolt") { + t.Skip("skipping: Dolt tests skipped (BEADS_TEST_SKIP=dolt)") + } + if _, err := resolveLocalDoltBinary(); err != nil { + if os.Getenv(EnvRequireDoltContainer) == "1" && useLocalDoltServer() { + t.Fatalf("local Dolt CLI unavailable (%v) but %s=1", err, EnvRequireDoltContainer) + } + t.Skipf("local Dolt CLI unavailable: %v", err) + } +} + +func TestLocalDoltServer_BindRaceRetriesOnAnotherPort(t *testing.T) { + requireLocalDoltCLI(t) + // Hold a port the way a concurrent action would between FindFreePort and + // dolt's own bind. + l, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + defer l.Close() + taken := l.Addr().(*net.TCPAddr).Port + + s, err := newLocalDoltServer() + if err != nil { + t.Fatal(err) + } + t.Cleanup(s.terminate) + if err := s.start(taken); err != nil { + t.Fatalf("start with port %d taken: %v", taken, err) + } + if s.Port() == taken { + t.Fatalf("server reports the taken port %d", taken) + } + if !strings.Contains(s.logSince(0), "already in use") { + t.Errorf("first attempt did not hit the taken port; log: %s", s.logSince(0)) + } + if err := pingDoltOnce(fmt.Sprintf("root@tcp(127.0.0.1:%d)/", s.Port())); err != nil { + t.Errorf("ping after retry: %v", err) + } +} + +func TestLocalDoltServer_NeverDefaultPort(t *testing.T) { + requireLocalDoltCLI(t) + s, err := newLocalDoltServer() + if err != nil { + t.Fatal(err) + } + t.Cleanup(s.terminate) + if err := s.start(3307); err != nil { + t.Fatal(err) + } + if s.Port() == 3307 { + t.Fatal("local test server listens on 3307, the production default port") + } +} + +func TestLocalDoltServer_Lifecycle(t *testing.T) { + requireLocalDoltCLI(t) + c := startIsolatedLocalDoltServer(t) + s := c.local + ctx := context.Background() + + t.Run("ExecRunsInDataDir", func(t *testing.T) { + code, out, err := c.Exec(ctx, []string{"pwd", "-P"}) + if err != nil || code != 0 { + t.Fatalf("exec pwd: code=%d err=%v out=%q", code, err, out) + } + want, _ := filepath.EvalSymlinks(s.dataDir) + if got := strings.TrimSpace(out); got != want { + t.Errorf("Exec working directory = %q, want the data dir %q", got, want) + } + code, _, err = c.Exec(ctx, []string{"test", "-d", "beads_test"}) + if err != nil || code != 0 { + t.Errorf("provisioned database beads_test not visible from Exec's cwd: code=%d err=%v", code, err) + } + code, _, err = c.Exec(ctx, []string{"false"}) + if err != nil || code != 1 { + t.Errorf("Exec exit status: code=%d err=%v, want 1, nil", code, err) + } + }) + + t.Run("StopStartKeepsData", func(t *testing.T) { + dsn := func(port string) string { return fmt.Sprintf("root@tcp(127.0.0.1:%s)/beads_test", port) } + db, err := sql.Open("mysql", dsn(c.Port)) + if err != nil { + t.Fatal(err) + } + if _, err := db.ExecContext(ctx, "CREATE TABLE kept (id INT PRIMARY KEY)"); err != nil { + t.Fatal(err) + } + if _, err := db.ExecContext(ctx, "INSERT INTO kept VALUES (42)"); err != nil { + t.Fatal(err) + } + _ = db.Close() + + if err := c.Stop(ctx); err != nil { + t.Fatalf("Stop: %v", err) + } + if gone, _ := s.exited(); !gone { + t.Fatal("server still running after Stop") + } + if err := c.Start(ctx); err != nil { + t.Fatalf("Start: %v", err) + } + port, err := c.CurrentPort(ctx) + if err != nil { + t.Fatal(err) + } + db, err = sql.Open("mysql", dsn(port)) + if err != nil { + t.Fatal(err) + } + defer db.Close() + var id int + if err := db.QueryRowContext(ctx, "SELECT id FROM kept").Scan(&id); err != nil || id != 42 { + t.Fatalf("data after restart: id=%d err=%v", id, err) + } + }) + + t.Run("CrashIsDetected", func(t *testing.T) { + s.mu.Lock() + pid := s.cmd.Process.Pid + s.mu.Unlock() + if gone, _ := s.exited(); gone { + t.Fatal("server reported exited before the kill") + } + if err := syscall.Kill(pid, syscall.SIGKILL); err != nil { + t.Fatal(err) + } + deadline := time.Now().Add(10 * time.Second) + for { + if gone, _ := s.exited(); gone { + break + } + if time.Now().After(deadline) { + t.Fatal("killed server not reported as exited") + } + time.Sleep(10 * time.Millisecond) + } + }) +} + +func TestLocalDoltServer_SingletonCrashDetection(t *testing.T) { + requireLocalDoltCLI(t) + s, err := startLocalDoltServer() + if err != nil { + t.Fatal(err) + } + defer s.terminate() + // Exercise the exported crash API on a singleton that is this server, + // without disturbing whatever singleton the package may hold. + doltServerMu.Lock() + saved := doltSingletonSrv + doltSingletonSrv = &doltServer{local: s} + doltServerMu.Unlock() + t.Cleanup(func() { + doltServerMu.Lock() + doltSingletonSrv = saved + doltServerMu.Unlock() + }) + + if DoltContainerCrashed() || DoltContainerCrashError() != nil { + t.Fatal("running server reported as crashed") + } + s.kill() + if !DoltContainerCrashed() { + t.Error("DoltContainerCrashed() = false after the server died") + } + if err := DoltContainerCrashError(); err == nil { + t.Error("DoltContainerCrashError() = nil after the server died") + } + if !ServerUnreachable(errors.New("anything")) { + t.Error("ServerUnreachable ignores a dead local singleton") + } +} + +// The suites' leak sweeps (doltserver.SweepSuiteTestServers) report and kill +// dolt servers whose working directory is under the suite root. Test servers +// must live outside it even when started after PinSuiteTempRoot re-pointed +// TMPDIR. +func TestLocalDoltServer_StateOutsideSuiteRoot(t *testing.T) { + requireLocalDoltCLI(t) + for _, k := range []string{"TMPDIR", "GOTMPDIR", "TMP", "TEMP"} { + t.Setenv(k, os.Getenv(k)) // restored after the test + } + root, err := PinSuiteTempRoot("bdt-suite-root-") + if err != nil { + t.Fatal(err) + } + defer os.RemoveAll(root) + if !PathUnderSuiteRoot(filepath.Join(os.TempDir(), "x"), root) { + t.Fatalf("PinSuiteTempRoot did not re-point os.TempDir() under %s", root) + } + + s, err := startLocalDoltServer() + if err != nil { + t.Fatal(err) + } + defer s.terminate() + for _, p := range []string{s.root, s.dataDir} { + if PathUnderSuiteRoot(p, root) { + t.Errorf("local server state %s is under the suite root %s", p, root) + } + } +} + +func TestLocalDoltServer_TerminateLeavesNothing(t *testing.T) { + requireLocalDoltCLI(t) + s, err := startLocalDoltServer() + if err != nil { + t.Fatal(err) + } + s.mu.Lock() + pid := s.cmd.Process.Pid + s.mu.Unlock() + s.terminate() + if processAlive(pid) { + t.Errorf("dolt sql-server pid %d still alive after terminate", pid) + } + if _, err := os.Stat(s.root); !os.IsNotExist(err) { + t.Errorf("state dir %s not removed: %v", s.root, err) + } + if err := pingDoltOnce(fmt.Sprintf("root@tcp(127.0.0.1:%d)/", s.Port())); err == nil { + t.Errorf("something still answers on port %d", s.Port()) + } + // A detached `dolt send-metrics` child of the exiting server would + // re-create /home/.dolt a few hundred ms later. + time.Sleep(time.Second) + if _, err := os.Stat(s.root); !os.IsNotExist(err) { + t.Errorf("state dir %s re-created after terminate (dolt child outlived the server?): %v", s.root, err) + } +} + +// The version probe's throwaway HOME must be gone for good once it returns. +func TestLocalDoltServer_VersionProbeLeavesNothing(t *testing.T) { + requireLocalDoltCLI(t) + bin, _ := resolveLocalDoltBinary() + base := t.TempDir() + if _, err := probeDoltVersion(bin, base); err != nil { + t.Fatal(err) + } + time.Sleep(time.Second) + if ents, _ := os.ReadDir(base); len(ents) != 0 { + t.Errorf("dolt version left %d entries in %s (dolt child outlived it?): %v", len(ents), base, ents) + } +} + +// Only files under the server's own sfp/ directory are reachable through +// LOAD_FILE and SELECT ... INTO OUTFILE: a local server runs as the test user +// on the host, with a passwordless root@%. +func TestLocalDoltServer_SecureFilePriv(t *testing.T) { + requireLocalDoltCLI(t) + c := startIsolatedLocalDoltServer(t) + db, err := sql.Open("mysql", fmt.Sprintf("root@tcp(127.0.0.1:%s)/", c.Port)) + if err != nil { + t.Fatal(err) + } + defer db.Close() + ctx := context.Background() + + outside := filepath.Join(t.TempDir(), "secret") + if err := os.WriteFile(outside, []byte("secret"), 0o600); err != nil { + t.Fatal(err) + } + var got sql.NullString + if err := db.QueryRowContext(ctx, "SELECT LOAD_FILE(?)", outside).Scan(&got); err != nil { + t.Fatal(err) + } + if got.Valid { + t.Errorf("LOAD_FILE(%s) outside secure_file_priv = %q, want NULL", outside, got.String) + } + written := filepath.Join(filepath.Dir(outside), "written") + if _, err := db.ExecContext(ctx, "SELECT 1 INTO OUTFILE '"+written+"'"); err == nil { + t.Errorf("SELECT ... INTO OUTFILE %s outside secure_file_priv succeeded", written) + } + if _, err := os.Stat(written); !os.IsNotExist(err) { + t.Errorf("INTO OUTFILE created %s: %v", written, err) + } + + inside := filepath.Join(c.local.secureFilePrivDir(), "ok") + if err := os.WriteFile(inside, []byte("ok"), 0o600); err != nil { + t.Fatal(err) + } + if err := db.QueryRowContext(ctx, "SELECT LOAD_FILE(?)", inside).Scan(&got); err != nil || got.String != "ok" { + t.Errorf("LOAD_FILE inside secure_file_priv = %q, %v; want \"ok\"", got.String, err) + } +} + +func TestSweepDeadLocalServerRoots(t *testing.T) { + base := t.TempDir() + const livePID, deadPID = 1001, 1002 + mk := func(name, marker string) string { + t.Helper() + p := filepath.Join(base, name) + if err := os.MkdirAll(filepath.Join(p, "data", "beads_test"), 0o700); err != nil { + t.Fatal(err) + } + if marker != "" { + if err := os.WriteFile(filepath.Join(p, localRootOwnerFile), []byte(marker), 0o600); err != nil { + t.Fatal(err) + } + } + return p + } + deadServer := mk("bdt-dolt-1", strconv.Itoa(deadPID)+"\n") + deadVersion := mk("bdt-doltver-1", strconv.Itoa(deadPID)) + live := mk("bdt-dolt-2", strconv.Itoa(livePID)) + noMarker := mk("bdt-dolt-3", "") + garbage := mk("bdt-dolt-4", "not a pid") + foreign := mk("other-dolt-1", strconv.Itoa(deadPID)) + target := mk("target", strconv.Itoa(deadPID)) + link := filepath.Join(base, "bdt-dolt-5") + if err := os.Symlink(target, link); err != nil { + t.Fatal(err) + } + + alive := func(pid int) bool { return pid == livePID } + swept := sweepDeadLocalServerRoots(base, alive) + sort.Strings(swept) + want := []string{deadServer, deadVersion} + if fmt.Sprint(swept) != fmt.Sprint(want) { + t.Errorf("swept %v, want %v", swept, want) + } + for _, p := range want { + if _, err := os.Lstat(p); !os.IsNotExist(err) { + t.Errorf("%s not removed: %v", p, err) + } + } + for _, p := range []string{live, noMarker, garbage, foreign, target, link} { + if _, err := os.Lstat(p); err != nil { + t.Errorf("%s must be left alone: %v", p, err) + } + } + if got := sweepDeadLocalServerRoots(base, alive); len(got) != 0 { + t.Errorf("second sweep removed %v", got) + } +} + +func TestLocalRootOwnerMarker(t *testing.T) { + root := t.TempDir() + if _, ok := readLocalRootOwner(root); ok { + t.Fatal("marker reported in an empty root") + } + if err := writeLocalRootOwner(root); err != nil { + t.Fatal(err) + } + if pid, ok := readLocalRootOwner(root); !ok || pid != os.Getpid() { + t.Errorf("readLocalRootOwner = %d, %v; want %d, true", pid, ok, os.Getpid()) + } + if !processAlive(os.Getpid()) { + t.Error("processAlive(self) = false") + } +} + +// The server's own dolt global config turns off the release check and usage +// events: a hermetic test action must not depend on (or phone) the network. +func TestLocalDoltServer_NoNetworkCallsConfigured(t *testing.T) { + requireLocalDoltCLI(t) + s, err := newLocalDoltServer() + if err != nil { + t.Fatal(err) + } + defer os.RemoveAll(s.root) + b, err := os.ReadFile(filepath.Join(s.root, "home", ".dolt", "config_global.json")) + if err != nil { + t.Fatal(err) + } + for _, k := range []string{`"metrics.disabled":"true"`, `"versioncheck.disabled":"true"`} { + if !strings.Contains(string(b), k) { + t.Errorf("server dolt global config lacks %s: %s", k, b) + } + } +} + +func TestDoltBackendSelection(t *testing.T) { + for _, tc := range []struct { + val, srcdir string + local bool + bad bool + }{ + {"", "", false, false}, + {"", "/runfiles", false, false}, // bazel: local only when a target or --test_env asks + {"local", "/runfiles", true, false}, + {"container", "/runfiles", false, false}, + {"local", "", true, false}, + {"docker", "", false, true}, + } { + t.Run(tc.val+"/"+strconv.Quote(tc.srcdir), func(t *testing.T) { + t.Setenv(EnvDoltServerBackend, tc.val) + t.Setenv("TEST_SRCDIR", tc.srcdir) + if got := useLocalDoltServer(); got != tc.local { + t.Errorf("useLocalDoltServer() = %v, want %v", got, tc.local) + } + if got := doltBackendErr() != nil; got != tc.bad { + t.Errorf("doltBackendErr() != nil = %v, want %v", got, tc.bad) + } + }) + } +} diff --git a/internal/testutil/testdoltcommon.go b/internal/testutil/testdoltcommon.go index 3658bfcc5..3b8c0a959 100644 --- a/internal/testutil/testdoltcommon.go +++ b/internal/testutil/testdoltcommon.go @@ -15,6 +15,28 @@ import ( // DoltDockerImage is the Docker image used for Dolt test containers. const DoltDockerImage = "dolthub/dolt-sql-server:2.2.0" +// EnvRequireDoltContainer, set to "1", turns an unavailable Dolt test server +// (either backend, see BEADS_TEST_DOLT_SERVER) into a failure instead of a +// skip: in RequireDoltContainer and StartIsolatedDoltContainer(Handle), and in +// every TestMain through DoltUnavailableForTestMain. Lanes that exist to run +// the Dolt server suites set it so they cannot pass green having run nothing +// (and, under Bazel, have that vacuous pass cached and shared). +const EnvRequireDoltContainer = "BEADS_TEST_REQUIRE_DOLT_CONTAINER" + +// DoltUnavailableForTestMain handles an EnsureDoltContainerForTestMain error +// in a TestMain and reports whether the TestMain must exit non-zero. By +// default it prints the historical warning and returns false, so the package +// runs and its Dolt tests skip. With BEADS_TEST_REQUIRE_DOLT_CONTAINER=1 it +// prints a FATAL line and returns true. +func DoltUnavailableForTestMain(err error) bool { + if os.Getenv(EnvRequireDoltContainer) == "1" { + fmt.Fprintf(os.Stderr, "FATAL: %v, but %s=1; this lane must not skip its Dolt tests\n", err, EnvRequireDoltContainer) + return true + } + fmt.Fprintf(os.Stderr, "WARN: %v, skipping Dolt tests\n", err) + return false +} + // RequireDoltBinary ensures the `dolt` CLI binary is available, and honors // BEADS_TEST_SKIP=dolt for tests that also depend on the shared // containerized Dolt SQL server. The test is skipped locally when dolt is diff --git a/internal/testutil/testdoltserver.go b/internal/testutil/testdoltserver.go index 367a0be70..c8a8616fa 100644 --- a/internal/testutil/testdoltserver.go +++ b/internal/testutil/testdoltserver.go @@ -24,6 +24,7 @@ import ( // doltServer represents a running test Dolt container instance. type doltServer struct { container *dolt.DoltContainer + local *localDoltServer // set instead of container by the local backend } // serverStartTimeout is the max time to wait for the test Dolt server to accept connections. @@ -55,8 +56,12 @@ const ( doltWrongVersion // image exists but wrong tag doltSkipped // explicit opt-out via BEADS_TEST_SKIP doltReady // ready to start containers + doltNoLocalCLI // local backend: pinned dolt CLI missing or wrong version, or a bad backend name ) +// doltLocalCLIErr explains doltNoLocalCLI. +var doltLocalCLIErr error + func (d doltReadiness) String() string { switch d { case doltNoDocker: @@ -69,6 +74,8 @@ func (d doltReadiness) String() string { return "Dolt tests skipped (BEADS_TEST_SKIP=dolt)" case doltReady: return "Dolt ready" + case doltNoLocalCLI: + return fmt.Sprintf("local Dolt test server backend unavailable: %v", doltLocalCLIErr) default: return fmt.Sprintf("unknown dolt readiness state: %d", int(d)) } @@ -98,13 +105,6 @@ func hasTestSkip(service string) bool { return false } -// EnvRequireDoltContainer, set to "1", turns a missing Dolt test container into -// a test failure instead of a skip in RequireDoltContainer and -// StartIsolatedDoltContainer(Handle). Lanes that exist to run the container -// suites (Bazel's requires-docker variants) set it so they cannot pass green -// having run nothing. -const EnvRequireDoltContainer = "BEADS_TEST_REQUIRE_DOLT_CONTAINER" - func skipOrFailDoltUnavailable(t *testing.T, state doltReadiness) { t.Helper() if os.Getenv(EnvRequireDoltContainer) == "1" { @@ -123,6 +123,20 @@ func checkDolt() doltReadiness { doltCached = doltSkipped return } + if err := doltBackendErr(); err != nil { + doltLocalCLIErr = err + doltCached = doltNoLocalCLI + return + } + if useLocalDoltServer() { + if _, err := resolveLocalDoltBinary(); err != nil { + doltLocalCLIErr = err + doltCached = doltNoLocalCLI + return + } + doltCached = doltReady + return + } if !isDockerAvailable() { return // doltCached zero value is doltNoDocker } @@ -154,6 +168,15 @@ func isDoltRepoImageCached() bool { // startDoltContainer starts the singleton Dolt container. func startDoltContainer() error { + if useLocalDoltServer() { + s, err := startLocalDoltServer() + if err != nil { + return fmt.Errorf("starting local Dolt server: %w", err) + } + doltTestPort = strconv.Itoa(s.Port()) + doltSingletonSrv = &doltServer{local: s} + return nil + } ctx, cancel := context.WithTimeout(context.Background(), serverStartTimeout) defer cancel() @@ -253,6 +276,10 @@ func terminateSharedContainer() { // holds doltServerMu. It does not consume doltTerminateOnce, so a later // replacement can still be removed by TerminateDoltContainer. func stopSharedContainerLocked() { + if doltSingletonSrv != nil && doltSingletonSrv.local != nil { + doltSingletonSrv.local.terminate() + doltSingletonSrv.local = nil + } if doltSingletonSrv != nil && doltSingletonSrv.container != nil { _ = testcontainers.TerminateContainer(doltSingletonSrv.container) doltSingletonSrv.container = nil @@ -313,11 +340,15 @@ type IsolatedDoltContainer struct { // Port is the mapped host port of the container's Dolt server. Port string - ctr *dolt.DoltContainer + ctr *dolt.DoltContainer + local *localDoltServer // local backend instead of ctr } // Stop pauses the container while retaining its data volume. func (c *IsolatedDoltContainer) Stop(ctx context.Context) error { + if c != nil && c.local != nil { + return c.local.stop() + } if c == nil || c.ctr == nil { return fmt.Errorf("no Dolt container running") } @@ -326,6 +357,9 @@ func (c *IsolatedDoltContainer) Stop(ctx context.Context) error { // Start resumes a container previously stopped with Stop. func (c *IsolatedDoltContainer) Start(ctx context.Context) error { + if c != nil && c.local != nil { + return c.local.restart() + } if c == nil || c.ctr == nil { return fmt.Errorf("no Dolt container running") } @@ -335,6 +369,9 @@ func (c *IsolatedDoltContainer) Start(ctx context.Context) error { // CurrentPort returns the container's currently mapped SQL port. Container // runtimes may assign a new host port after Stop/Start. func (c *IsolatedDoltContainer) CurrentPort(ctx context.Context) (string, error) { + if c != nil && c.local != nil { + return strconv.Itoa(c.local.Port()), nil + } if c == nil || c.ctr == nil { return "", fmt.Errorf("no Dolt container running") } @@ -346,8 +383,14 @@ func (c *IsolatedDoltContainer) CurrentPort(ctx context.Context) (string, error) } // Exec runs cmd inside the container and returns its exit code and combined -// output, demultiplexed (see containerExec). +// output, demultiplexed (see containerExec). Its working directory is the +// server's data directory (the image's WORKDIR, /var/lib/dolt); with the local +// backend cmd runs on the host in the local server's data directory, so +// commands must address the data through relative paths. func (c *IsolatedDoltContainer) Exec(ctx context.Context, cmd []string) (int, string, error) { + if c != nil && c.local != nil { + return c.local.execInDataDir(ctx, cmd) + } if c == nil || c.ctr == nil { return 0, "", fmt.Errorf("no Dolt container running") } @@ -367,6 +410,28 @@ func StartIsolatedDoltContainerHandle(t *testing.T) *IsolatedDoltContainer { skipOrFailDoltUnavailable(t, state) } + if useLocalDoltServer() { + return startIsolatedLocalDoltServer(t) + } + return startIsolatedDoltContainer(t) +} + +// startIsolatedLocalDoltServer is StartIsolatedDoltContainerHandle for the +// local backend, without the readiness gate. +func startIsolatedLocalDoltServer(t *testing.T) *IsolatedDoltContainer { + t.Helper() + s, err := startLocalDoltServer() + if err != nil { + t.Fatalf("starting local Dolt server: %v", err) + } + t.Cleanup(s.terminate) + return &IsolatedDoltContainer{Port: strconv.Itoa(s.Port()), local: s} +} + +// startIsolatedDoltContainer is StartIsolatedDoltContainerHandle for the +// container backend, without the readiness gate. +func startIsolatedDoltContainer(t *testing.T) *IsolatedDoltContainer { + t.Helper() ctx, cancel := context.WithTimeout(context.Background(), serverStartTimeout) defer cancel() ctr, err := dolt.Run(ctx, DoltDockerImage, @@ -469,6 +534,10 @@ func TerminateDoltContainer() { // DoltContainerCrashed returns true if the shared container has exited unexpectedly. // Returns false if no container was started. func DoltContainerCrashed() bool { + if doltSingletonSrv != nil && doltSingletonSrv.local != nil { + gone, _ := doltSingletonSrv.local.exited() + return gone + } if doltSingletonSrv == nil || doltSingletonSrv.container == nil { return false } @@ -482,6 +551,12 @@ func DoltContainerCrashed() bool { // DoltContainerCrashError returns an error if the shared container has exited // unexpectedly, nil otherwise. func DoltContainerCrashError() error { + if doltSingletonSrv != nil && doltSingletonSrv.local != nil { + if gone, err := doltSingletonSrv.local.exited(); gone { + return fmt.Errorf("local Dolt server exited (%v); log: %s", err, doltSingletonSrv.local.logSince(0)) + } + return nil + } if doltSingletonSrv == nil || doltSingletonSrv.container == nil { return nil } diff --git a/internal/testutil/testdoltserver_env_test.go b/internal/testutil/testdoltserver_env_test.go index 17aa1afa7..6810489d7 100644 --- a/internal/testutil/testdoltserver_env_test.go +++ b/internal/testutil/testdoltserver_env_test.go @@ -1,3 +1,5 @@ +//go:build !windows + package testutil import ( diff --git a/internal/tracker/testmain_test.go b/internal/tracker/testmain_test.go index 4bf75441f..dcff65085 100644 --- a/internal/tracker/testmain_test.go +++ b/internal/tracker/testmain_test.go @@ -41,11 +41,9 @@ func testMainInner(m *testing.M) int { // AD-01 (be-c5p): allow tracker tests to connect to the test container. os.Setenv("BEADS_TEST_SERVER", "1") if err := testutil.EnsureDoltContainerForTestMain(); err != nil { - if os.Getenv(testutil.EnvRequireDoltContainer) == "1" { - fmt.Fprintf(os.Stderr, "FATAL: %v, but %s=1: this lane must not skip the Dolt tests\n", err, testutil.EnvRequireDoltContainer) + if testutil.DoltUnavailableForTestMain(err) { return 1 } - fmt.Fprintf(os.Stderr, "WARN: %v, skipping Dolt tests\n", err) } else { defer testutil.TerminateDoltContainer() testServerPort = testutil.DoltContainerPortInt() diff --git a/internal/utils/testmain_test.go b/internal/utils/testmain_test.go index ea9634337..4f2370666 100644 --- a/internal/utils/testmain_test.go +++ b/internal/utils/testmain_test.go @@ -3,7 +3,6 @@ package utils import ( - "fmt" "os" "testing" @@ -24,7 +23,9 @@ func testMainInner(m *testing.M) int { // AD-01 (be-c5p): allow utils tests to connect to the test container. os.Setenv("BEADS_TEST_SERVER", "1") if err := testutil.EnsureDoltContainerForTestMain(); err != nil { - fmt.Fprintf(os.Stderr, "WARN: %v, skipping Dolt tests\n", err) + if testutil.DoltUnavailableForTestMain(err) { + return 1 + } } else { defer testutil.TerminateDoltContainer() DoltTestServerPort = testutil.DoltContainerPortInt() diff --git a/scripts/ci_workflow_test.go b/scripts/ci_workflow_test.go index c74de88de..bd842d395 100644 --- a/scripts/ci_workflow_test.go +++ b/scripts/ci_workflow_test.go @@ -323,6 +323,14 @@ func TestStorageDomainUOWJobsUseNestedTimeoutBudgets(t *testing.T) { }) } + // pr.yml's job is the one lane with both the image and the pinned dolt + // CLI, so it is where the container and local test servers are compared. + job := readCIWorkflow(t, "pr.yml").job(t, "test-domain-uow") + const fingerprintStep = "Test Dolt server fingerprint (container + local)" + assertStepRunsExactly(t, job, fingerprintStep, + "go test -tags gms_pure_go -count=1 -timeout 5m -v -run '^TestDoltServerFingerprint$' ./internal/testutil/") + assertStepEnvValue(t, job, fingerprintStep, "BEADS_TEST_REQUIRE_DOLT_CONTAINER", "1") + gate := readCIWorkflow(t, "pr.yml").job(t, "ci-gate") gateEnv := gate.step(t, "Evaluate CI gate").Env if !contains(gate.Needs, "test-domain-uow") { diff --git a/tests/regression/regression_test.go b/tests/regression/regression_test.go index feb9d5339..6683ee982 100644 --- a/tests/regression/regression_test.go +++ b/tests/regression/regression_test.go @@ -69,7 +69,9 @@ func testMainInner(m *testing.M) int { // AD-01 (be-c5p): allow regression tests to connect to the test container. os.Setenv("BEADS_TEST_SERVER", "1") if err := testutil.EnsureDoltContainerForTestMain(); err != nil { - fmt.Fprintf(os.Stderr, "WARN: %v, skipping Dolt tests\n", err) + if testutil.DoltUnavailableForTestMain(err) { + return 1 + } } else { defer testutil.TerminateDoltContainer() testDoltServerPort = testutil.DoltContainerPortInt() diff --git a/tools/bazel/test_env.sh b/tools/bazel/test_env.sh index fec7b5313..59a074362 100755 --- a/tools/bazel/test_env.sh +++ b/tools/bazel/test_env.sh @@ -23,10 +23,17 @@ mkdir -p "$root/home" "$root/xdg-config" "$root/dolt-root" "$root/tmp" : >"$root/gitconfig" # Dolt identity, as beads_test_env_enter sets with `dolt config --global`: # tests that shell out to dolt commit need an author. Written directly so the -# wrapper does not depend on a dolt binary. +# wrapper does not depend on a dolt binary. The two *.disabled keys stop every +# dolt command from checking for a newer release and from sending usage +# events over the network: calls a hermetic test must not depend on (they +# change no behavior, only drop a "newer version available" stderr line). +# They do not stop dolt from forking a detached `dolt send-metrics` child on +# exit, which outlives the command and re-creates $DOLT_ROOT_PATH/.dolt under +# a test that already removed it; DOLT_DISABLE_EVENT_FLUSH does. mkdir -p "$root/dolt-root/.dolt" -printf '%s\n' '{"user.email":"test@beads.local","user.name":"beads-test"}' \ +printf '%s\n' '{"user.email":"test@beads.local","user.name":"beads-test","metrics.disabled":"true","versioncheck.disabled":"true"}' \ >"$root/dolt-root/.dolt/config_global.json" +export DOLT_DISABLE_EVENT_FLUSH=1 export HOME="$root/home" export USERPROFILE="$root/home"