Implement 5 test groups for GH#2763, targeting production failure modes that single-process goroutine tests cannot catch: T1: Server lifecycle (lifecycle_integration_test.go) - Start/stop cycle with state file verification - Crash recovery (SIGKILL -> restart -> data survives) - Restart data persistence across stop/start cycles - Multi-repo isolation (KillStaleServers cross-repo safety) - Concurrent IsRunning+Start atomicity - PID reuse detection (isDoltProcess check) T2: Multi-process schema init (initschema_multiprocess_test.go) - 8 subprocesses racing initSchemaOnDB on fresh DB - Journal corruption detection via dolt verify - Schema integrity validation (all tables/indexes present) T3: Multi-DoltStore operations (multistore_multiprocess_test.go) - Layer 1: 5 independent sql.DB pools, in-process concurrent ops - Layer 2: 5 subprocesses each with own DoltStore - Close isolation (one pool close doesnt cascade) T4: Dirty state recovery (dirty_state_test.go) - Stale PID (alive non-dolt, dead, reused) - Stale port file, corrupt PID, truncated metadata.json - Port 0 regression (GH#2598) - Combined stale PID+port, orphan noms LOCK T5: Port allocation race (port_race_test.go) - Concurrent Start() exactly-one-wins via flock - Ephemeral port retry with occupied port - Loopback-only binding verification - Multi-repo port collision prevention Shared harness (internal/testutil/integration/): - ProcessRegistry: SIGTERM->SIGKILL escalation with t.Cleanup - SubprocessRunner: sync.Once binary build, env allowlist, Setpgid - StateCorruptor: dirty state fixture helpers - Diagnostics: failure artifact capture All tests use //go:build integration && !windows and run in nightly CI with -race flag. Existing test suite unaffected. Fixes #2763 Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
88 lines
2.1 KiB
Go
88 lines
2.1 KiB
Go
//go:build integration && !windows
|
|
|
|
package integration
|
|
|
|
import (
|
|
"fmt"
|
|
"os"
|
|
"path/filepath"
|
|
"strings"
|
|
"testing"
|
|
)
|
|
|
|
// Diagnostics captures state information for debugging test failures.
|
|
type Diagnostics struct {
|
|
t *testing.T
|
|
beadsDir string
|
|
}
|
|
|
|
// NewDiagnostics creates a diagnostics collector for the given .beads directory.
|
|
func NewDiagnostics(t *testing.T, beadsDir string) *Diagnostics {
|
|
return &Diagnostics{t: t, beadsDir: beadsDir}
|
|
}
|
|
|
|
// CaptureOnFailure registers a t.Cleanup that dumps diagnostic info if the
|
|
// test has failed. Call early in the test setup.
|
|
func (d *Diagnostics) CaptureOnFailure() {
|
|
d.t.Helper()
|
|
d.t.Cleanup(func() {
|
|
if !d.t.Failed() {
|
|
return
|
|
}
|
|
d.t.Log("=== DIAGNOSTICS (test failed) ===")
|
|
d.dumpStateFiles()
|
|
d.dumpProcessList()
|
|
})
|
|
}
|
|
|
|
// dumpStateFiles logs the content of PID and port files if they exist.
|
|
func (d *Diagnostics) dumpStateFiles() {
|
|
for _, name := range []string{"dolt-server.pid", "dolt-server.port", "dolt-server.log"} {
|
|
path := filepath.Join(d.beadsDir, name)
|
|
data, err := os.ReadFile(path)
|
|
if err != nil {
|
|
d.t.Logf(" %s: not found", name)
|
|
continue
|
|
}
|
|
content := strings.TrimSpace(string(data))
|
|
if len(content) > 500 {
|
|
content = content[:500] + "... (truncated)"
|
|
}
|
|
d.t.Logf(" %s: %s", name, content)
|
|
}
|
|
}
|
|
|
|
// dumpProcessList logs any dolt processes visible via /proc or ps.
|
|
func (d *Diagnostics) dumpProcessList() {
|
|
entries, err := os.ReadDir("/proc")
|
|
if err != nil {
|
|
d.t.Log(" /proc: not available")
|
|
return
|
|
}
|
|
var doltProcs []string
|
|
for _, entry := range entries {
|
|
if !entry.IsDir() {
|
|
continue
|
|
}
|
|
// Only numeric directories (PIDs).
|
|
pid := entry.Name()
|
|
if pid[0] < '0' || pid[0] > '9' {
|
|
continue
|
|
}
|
|
cmdline, err := os.ReadFile(filepath.Join("/proc", pid, "cmdline"))
|
|
if err != nil {
|
|
continue
|
|
}
|
|
if strings.Contains(string(cmdline), "dolt") {
|
|
doltProcs = append(doltProcs, fmt.Sprintf("PID %s: %s", pid, strings.ReplaceAll(string(cmdline), "\x00", " ")))
|
|
}
|
|
}
|
|
if len(doltProcs) == 0 {
|
|
d.t.Log(" dolt processes: none found")
|
|
} else {
|
|
d.t.Logf(" dolt processes (%d):", len(doltProcs))
|
|
for _, p := range doltProcs {
|
|
d.t.Logf(" %s", p)
|
|
}
|
|
}
|
|
}
|