beads/internal/testutil/integration/corruption.go
Harry 37355db018 test: add multi-process concurrency and server lifecycle integration tests
Implement 5 test groups for GH#2763, targeting production failure modes
that single-process goroutine tests cannot catch:

T1: Server lifecycle (lifecycle_integration_test.go)
  - Start/stop cycle with state file verification
  - Crash recovery (SIGKILL -> restart -> data survives)
  - Restart data persistence across stop/start cycles
  - Multi-repo isolation (KillStaleServers cross-repo safety)
  - Concurrent IsRunning+Start atomicity
  - PID reuse detection (isDoltProcess check)

T2: Multi-process schema init (initschema_multiprocess_test.go)
  - 8 subprocesses racing initSchemaOnDB on fresh DB
  - Journal corruption detection via dolt verify
  - Schema integrity validation (all tables/indexes present)

T3: Multi-DoltStore operations (multistore_multiprocess_test.go)
  - Layer 1: 5 independent sql.DB pools, in-process concurrent ops
  - Layer 2: 5 subprocesses each with own DoltStore
  - Close isolation (one pool close doesnt cascade)

T4: Dirty state recovery (dirty_state_test.go)
  - Stale PID (alive non-dolt, dead, reused)
  - Stale port file, corrupt PID, truncated metadata.json
  - Port 0 regression (GH#2598)
  - Combined stale PID+port, orphan noms LOCK

T5: Port allocation race (port_race_test.go)
  - Concurrent Start() exactly-one-wins via flock
  - Ephemeral port retry with occupied port
  - Loopback-only binding verification
  - Multi-repo port collision prevention

Shared harness (internal/testutil/integration/):
  - ProcessRegistry: SIGTERM->SIGKILL escalation with t.Cleanup
  - SubprocessRunner: sync.Once binary build, env allowlist, Setpgid
  - StateCorruptor: dirty state fixture helpers
  - Diagnostics: failure artifact capture

All tests use //go:build integration && !windows and run in nightly CI
with -race flag. Existing test suite unaffected.

Fixes #2763

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
2026-03-22 15:01:45 -07:00

101 lines
3 KiB
Go

//go:build integration && !windows
package integration
import (
"fmt"
"os"
"path/filepath"
"testing"
)
// StateCorruptor creates dirty server state for testing recovery scenarios.
type StateCorruptor struct {
BeadsDir string
t *testing.T
}
// NewStateCorruptor creates a corruptor targeting the given .beads directory.
func NewStateCorruptor(t *testing.T, beadsDir string) *StateCorruptor {
t.Helper()
return &StateCorruptor{BeadsDir: beadsDir, t: t}
}
// WriteStalePID writes a PID file containing the given PID.
func (c *StateCorruptor) WriteStalePID(pid int) {
c.t.Helper()
path := filepath.Join(c.BeadsDir, "dolt-server.pid")
if err := os.WriteFile(path, []byte(fmt.Sprintf("%d", pid)), 0600); err != nil {
c.t.Fatalf("WriteStalePID: %v", err)
}
}
// WriteStalePort writes a port file containing the given port number.
func (c *StateCorruptor) WriteStalePort(port int) {
c.t.Helper()
path := filepath.Join(c.BeadsDir, "dolt-server.port")
if err := os.WriteFile(path, []byte(fmt.Sprintf("%d", port)), 0600); err != nil {
c.t.Fatalf("WriteStalePort: %v", err)
}
}
// WriteCorruptPID writes a non-numeric value to the PID file.
func (c *StateCorruptor) WriteCorruptPID() {
c.t.Helper()
path := filepath.Join(c.BeadsDir, "dolt-server.pid")
if err := os.WriteFile(path, []byte("not-a-pid"), 0600); err != nil {
c.t.Fatalf("WriteCorruptPID: %v", err)
}
}
// WriteTruncatedMetadata writes partial (invalid) JSON to metadata.json.
func (c *StateCorruptor) WriteTruncatedMetadata() {
c.t.Helper()
path := filepath.Join(c.BeadsDir, "metadata.json")
if err := os.WriteFile(path, []byte(`{"backend": "dolt", "dolt_da`), 0600); err != nil {
c.t.Fatalf("WriteTruncatedMetadata: %v", err)
}
}
// WritePortZero writes "0" to the port file (GH#2598 regression).
func (c *StateCorruptor) WritePortZero() {
c.t.Helper()
c.WriteStalePort(0)
}
// CreateOrphanNomsLock creates a zero-byte LOCK file at .dolt/noms/LOCK
// to simulate a power-loss scenario.
func (c *StateCorruptor) CreateOrphanNomsLock(doltDataDir string) {
c.t.Helper()
nomsDir := filepath.Join(doltDataDir, "noms")
if err := os.MkdirAll(nomsDir, 0700); err != nil {
c.t.Fatalf("CreateOrphanNomsLock: mkdir: %v", err)
}
lockPath := filepath.Join(nomsDir, "LOCK")
if err := os.WriteFile(lockPath, nil, 0600); err != nil {
c.t.Fatalf("CreateOrphanNomsLock: write: %v", err)
}
}
// CreateCombinedStaleState writes both stale PID and stale port files.
func (c *StateCorruptor) CreateCombinedStaleState(pid, port int) {
c.t.Helper()
c.WriteStalePID(pid)
c.WriteStalePort(port)
}
// PIDFilePath returns the expected PID file path.
func (c *StateCorruptor) PIDFilePath() string {
return filepath.Join(c.BeadsDir, "dolt-server.pid")
}
// PortFilePath returns the expected port file path.
func (c *StateCorruptor) PortFilePath() string {
return filepath.Join(c.BeadsDir, "dolt-server.port")
}
// FileExists returns true if the given path exists.
func FileExists(path string) bool {
_, err := os.Stat(path)
return err == nil
}