beads/internal/testutil/integration/diagnostics.go
Harry 37355db018 test: add multi-process concurrency and server lifecycle integration tests
Implement 5 test groups for GH#2763, targeting production failure modes
that single-process goroutine tests cannot catch:

T1: Server lifecycle (lifecycle_integration_test.go)
  - Start/stop cycle with state file verification
  - Crash recovery (SIGKILL -> restart -> data survives)
  - Restart data persistence across stop/start cycles
  - Multi-repo isolation (KillStaleServers cross-repo safety)
  - Concurrent IsRunning+Start atomicity
  - PID reuse detection (isDoltProcess check)

T2: Multi-process schema init (initschema_multiprocess_test.go)
  - 8 subprocesses racing initSchemaOnDB on fresh DB
  - Journal corruption detection via dolt verify
  - Schema integrity validation (all tables/indexes present)

T3: Multi-DoltStore operations (multistore_multiprocess_test.go)
  - Layer 1: 5 independent sql.DB pools, in-process concurrent ops
  - Layer 2: 5 subprocesses each with own DoltStore
  - Close isolation (one pool close doesnt cascade)

T4: Dirty state recovery (dirty_state_test.go)
  - Stale PID (alive non-dolt, dead, reused)
  - Stale port file, corrupt PID, truncated metadata.json
  - Port 0 regression (GH#2598)
  - Combined stale PID+port, orphan noms LOCK

T5: Port allocation race (port_race_test.go)
  - Concurrent Start() exactly-one-wins via flock
  - Ephemeral port retry with occupied port
  - Loopback-only binding verification
  - Multi-repo port collision prevention

Shared harness (internal/testutil/integration/):
  - ProcessRegistry: SIGTERM->SIGKILL escalation with t.Cleanup
  - SubprocessRunner: sync.Once binary build, env allowlist, Setpgid
  - StateCorruptor: dirty state fixture helpers
  - Diagnostics: failure artifact capture

All tests use //go:build integration && !windows and run in nightly CI
with -race flag. Existing test suite unaffected.

Fixes #2763

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
2026-03-22 15:01:45 -07:00

88 lines
2.1 KiB
Go

//go:build integration && !windows
package integration
import (
"fmt"
"os"
"path/filepath"
"strings"
"testing"
)
// Diagnostics captures state information for debugging test failures.
type Diagnostics struct {
t *testing.T
beadsDir string
}
// NewDiagnostics creates a diagnostics collector for the given .beads directory.
func NewDiagnostics(t *testing.T, beadsDir string) *Diagnostics {
return &Diagnostics{t: t, beadsDir: beadsDir}
}
// CaptureOnFailure registers a t.Cleanup that dumps diagnostic info if the
// test has failed. Call early in the test setup.
func (d *Diagnostics) CaptureOnFailure() {
d.t.Helper()
d.t.Cleanup(func() {
if !d.t.Failed() {
return
}
d.t.Log("=== DIAGNOSTICS (test failed) ===")
d.dumpStateFiles()
d.dumpProcessList()
})
}
// dumpStateFiles logs the content of PID and port files if they exist.
func (d *Diagnostics) dumpStateFiles() {
for _, name := range []string{"dolt-server.pid", "dolt-server.port", "dolt-server.log"} {
path := filepath.Join(d.beadsDir, name)
data, err := os.ReadFile(path)
if err != nil {
d.t.Logf(" %s: not found", name)
continue
}
content := strings.TrimSpace(string(data))
if len(content) > 500 {
content = content[:500] + "... (truncated)"
}
d.t.Logf(" %s: %s", name, content)
}
}
// dumpProcessList logs any dolt processes visible via /proc or ps.
func (d *Diagnostics) dumpProcessList() {
entries, err := os.ReadDir("/proc")
if err != nil {
d.t.Log(" /proc: not available")
return
}
var doltProcs []string
for _, entry := range entries {
if !entry.IsDir() {
continue
}
// Only numeric directories (PIDs).
pid := entry.Name()
if pid[0] < '0' || pid[0] > '9' {
continue
}
cmdline, err := os.ReadFile(filepath.Join("/proc", pid, "cmdline"))
if err != nil {
continue
}
if strings.Contains(string(cmdline), "dolt") {
doltProcs = append(doltProcs, fmt.Sprintf("PID %s: %s", pid, strings.ReplaceAll(string(cmdline), "\x00", " ")))
}
}
if len(doltProcs) == 0 {
d.t.Log(" dolt processes: none found")
} else {
d.t.Logf(" dolt processes (%d):", len(doltProcs))
for _, p := range doltProcs {
d.t.Logf(" %s", p)
}
}
}