revert: restore embedded Dolt mode (was only supposed to be removed from Gas Town)

Revert commits e5330641 and 0df41523 which removed embedded Dolt mode
from the beads codebase. The embedded Dolt removal was only intended for
Gas Town, not for Beads.

Restores:
- dolthub/driver dependency and embedded connection path
- Advisory flock (AccessLock) for concurrent process protection
- Embedded connector lifecycle management in DoltStore
- All CGO build tags across ~45 files
- Vendored go-icu-regex library
- nocgo stub files for non-CGO builds
- Bootstrap and embedded_uow helpers
- factory_dolt_test.go and dolt_singleprocess_test.go

Merge conflict resolutions:
- store.go: TCP fail-fast check wrapped inside if cfg.ServerMode
- store.go: backtick-quoting for embedded CREATE DATABASE (prefix-based names)
- perf_dolt.go: port references updated from 3306 to 3307
- ci.yml: restored continue-on-error for Nix (dolthub/driver requires Go 1.25.6+)

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>

Executed-By: beads/crew/emma
Rig: beads
Role: crew
This commit is contained in:
beads/crew/emma 2026-02-07 21:21:24 -08:00 • committed by Steve Yegge
commit 2137693dad
98 changed files with 5804 additions and 311 deletions

View file

@ -193,6 +193,9 @@ jobs:
test-nix:
name: Test Nix Flake
runs-on: ubuntu-latest
# Allow failure until nixpkgs updates Go to 1.25.6+
# (dolthub/driver requires go 1.25.6, nixpkgs-unstable has go 1.25.1)
continue-on-error: true
steps:
- uses: actions/checkout@v6
- uses: cachix/install-nix-action@v31

View file

@ -9,11 +9,26 @@ BINARY := bd
BUILD_DIR := .
INSTALL_DIR := $(HOME)/.local/bin
# Dolt backend requires CGO for embedded database support.
# Without CGO, builds will fail with "dolt backend requires CGO".
export CGO_ENABLED := 1
# ICU4C is keg-only on macOS (Homebrew doesn't symlink it into /opt/homebrew).
# Dolt's go-icu-regex dependency needs these paths to compile and link.
ifeq ($(shell uname),Darwin)
ICU_PREFIX := $(shell brew --prefix icu4c 2>/dev/null)
ifneq ($(ICU_PREFIX),)
export CGO_CFLAGS += -I$(ICU_PREFIX)/include
export CGO_CPPFLAGS += -I$(ICU_PREFIX)/include
export CGO_LDFLAGS += -L$(ICU_PREFIX)/lib
endif
endif
# Build the bd binary
build:
@echo "Building bd..."
ifeq ($(OS),Windows_NT)
go build -ldflags="-X main.Build=$$(git rev-parse --short HEAD)" -o $(BUILD_DIR)/$(BINARY) ./cmd/bd
go build -tags gms_pure_go -ldflags="-X main.Build=$$(git rev-parse --short HEAD)" -o $(BUILD_DIR)/$(BINARY) ./cmd/bd
else
go build -ldflags="-X main.Build=$$(git rev-parse --short HEAD)" -o $(BUILD_DIR)/$(BINARY) ./cmd/bd
ifeq ($(shell uname),Darwin)

View file

@ -19,6 +19,14 @@ Repository: https://github.com/dolthub/go-mysql-server
License: Apache License 2.0
Copyright: Copyright 2019-2026 DoltHub, Inc.
--------------------------------------------------------------------------------
dolthub/driver
--------------------------------------------------------------------------------
Repository: https://github.com/dolthub/driver
License: Apache License 2.0
Copyright: Copyright 2019-2026 DoltHub, Inc.
================================================================================
Apache License 2.0
================================================================================

View file

@ -366,7 +366,8 @@ func runDaemonLoop(interval time.Duration, autoCommit, autoPush, autoPull, local
backend = configfile.BackendSQLite
}
// Daemon is not supported with single-process backends
// Daemon is not supported with single-process backends (e.g., embedded Dolt)
// Note: Dolt server mode supports multi-process, so check capabilities not backend type
cfg, cfgErr := configfile.Load(beadsDir)
if cfgErr == nil && cfg != nil && cfg.GetCapabilities().SingleProcessOnly {
errMsg := fmt.Sprintf(`DAEMON NOT SUPPORTED WITH %s BACKEND
@ -492,7 +493,8 @@ The daemon will now exit.`, strings.ToUpper(backend))
"sql_port", doltServer.SQLPort(),
"remotesapi_port", doltServer.RemotesAPIPort())
// Configure factory with server connection details
// Configure factory to use server mode
factoryOpts.ServerMode = true
factoryOpts.ServerHost = doltServer.Host()
factoryOpts.ServerPort = doltServer.SQLPort()
}
@ -510,8 +512,10 @@ The daemon will now exit.`, strings.ToUpper(backend))
if sqliteStore, ok := store.(*sqlite.SQLiteStorage); ok {
sqliteStore.EnableFreshnessChecking()
log.Info("database opened", "path", store.Path(), "backend", "sqlite", "freshness_checking", true)
} else if federation {
log.Info("database opened", "path", store.Path(), "backend", "dolt", "mode", "federation/server")
} else {
log.Info("database opened", "path", store.Path(), "backend", "dolt", "mode", "server")
log.Info("database opened", "path", store.Path(), "backend", "dolt", "mode", "embedded")
}
// Auto-upgrade .beads/.gitignore if outdated

View file

@ -78,7 +78,7 @@ func isDoltBackendImpl() bool {
}
// singleProcessOnlyBackend returns true if the current workspace backend is configured
// as single-process-only.
// as single-process-only (currently Dolt embedded).
// This is a wrapper that calls singleProcessOnlyBackendFn for testability.
func singleProcessOnlyBackend() bool {
return singleProcessOnlyBackendFn()

View file

@ -20,13 +20,13 @@ func singleProcessBackendHelp(backend string) string {
}
// guardDaemonStartForDolt blocks daemon start/restart commands when the current
// workspace backend is single-process-only, unless --federation is specified.
// workspace backend is Dolt, unless --federation is specified.
//
// Rationale: some backends may be single-writer at the OS-process level. The
// Rationale: embedded Dolt is effectively single-writer at the OS-process level. The
// daemon architecture relies on multiple processes (CLI + daemon + helper spawns),
// which can trigger lock contention.
// which can trigger lock contention and transient "read-only" failures.
//
// Exception: --federation flag enables multi-writer mode.
// Exception: --federation flag enables dolt sql-server mode which is multi-writer.
//
// Note: This guard should only be attached to commands that START a daemon process
// (start, restart). Read-only commands (status, stop, logs, health, list) are allowed

View file

@ -377,8 +377,8 @@ func startDaemon(interval time.Duration, autoCommit, autoPush, autoPull, localMo
os.Exit(1)
}
// Guardrail: single-process backends must never spawn a daemon process.
// Exception: federation mode enables multi-writer support.
// Guardrail: single-process backends (e.g., Dolt embedded) must never spawn a daemon process.
// Exception: federation mode runs dolt sql-server which enables multi-writer support.
// This should already be blocked by command guards, but keep it defensive.
if singleProcessOnlyBackend() && !federation {
fmt.Fprintf(os.Stderr, "Error: daemon mode is not supported for single-process backends (e.g., dolt). Hint: use sqlite backend for daemon mode, use --federation for dolt server mode, or run commands in direct mode\n")

View file

@ -258,7 +258,7 @@ Stops the daemon gracefully, then starts a new one.`,
}
workspace := targetDaemon.WorkspacePath
// Guardrail: don't (re)start daemons for single-process backends.
// Guardrail: don't (re)start daemons for single-process backends (e.g., embedded Dolt).
// This command may be run from a different workspace, so check the target workspace.
// Note: Dolt server mode supports multi-process, so GetCapabilities() is used.
targetBeadsDir := beads.FollowRedirect(filepath.Join(workspace, ".beads"))

View file

@ -55,7 +55,7 @@ func disableDaemonForFallback(reason string) {
}
// ensureStoreActive guarantees that a storage backend is initialized and tracked.
// Uses the factory to respect metadata.json backend configuration (SQLite or Dolt server).
// Uses the factory to respect metadata.json backend configuration (SQLite, Dolt embedded, or Dolt server).
func ensureStoreActive() error {
lockStore()
active := isStoreActive() && getStore() != nil
@ -91,7 +91,7 @@ func ensureStoreActive() error {
}
}
// Use factory to create the appropriate backend (SQLite or Dolt server)
// Use factory to create the appropriate backend (SQLite, Dolt embedded, or Dolt server)
// based on metadata.json configuration
store, err := factory.NewFromConfig(getRootContext(), beadsDir)
if err != nil {

View file

@ -488,7 +488,7 @@ func runDiagnostics(path string) doctorResult {
result.OverallOK = false // Unresolved conflicts are a real problem
}
// Check 8h: Dolt server mode configuration
// Check 8h: Dolt init vs embedded mode mismatch
doltModeCheck := convertWithCategory(doctor.CheckDoltServerModeMismatch(path), doctor.CategoryFederation)
result.Checks = append(result.Checks, doltModeCheck)

View file

@ -1,3 +1,5 @@
//go:build cgo
package doctor
import (
@ -7,12 +9,21 @@ import (
"os"
"path/filepath"
_ "github.com/go-sql-driver/mysql"
// Import Dolt driver for direct connection
_ "github.com/dolthub/driver"
"github.com/steveyegge/beads/internal/configfile"
"github.com/steveyegge/beads/internal/storage/doltutil"
"github.com/steveyegge/beads/internal/storage/factory"
)
// closeDoltDBWithTimeout closes a sql.DB with a timeout to prevent indefinite hangs.
// This is needed because embedded Dolt can hang on close.
func closeDoltDBWithTimeout(db *sql.DB) {
// Use the shared helper; ignore errors since we're just cleaning up
_ = doltutil.CloseWithTimeout("db", db.Close)
}
// GetBackend returns the configured backend type from configuration.
// It checks config.yaml first (storage-backend key), then falls back to metadata.json.
// Returns "sqlite" (default) or "dolt".
@ -26,41 +37,6 @@ func IsDoltBackend(beadsDir string) bool {
return GetBackend(beadsDir) == configfile.BackendDolt
}
// openDoltServerDB opens a connection to the Dolt server via MySQL protocol.
// Returns the connection and any error.
func openDoltServerDB(beadsDir string) (*sql.DB, error) {
cfg, err := configfile.Load(beadsDir)
if err != nil || cfg == nil {
// Fall back to defaults
cfg = configfile.DefaultConfig()
}
host := cfg.GetDoltServerHost()
if host == "" {
host = "127.0.0.1"
}
port := cfg.GetDoltServerPort()
user := cfg.GetDoltServerUser()
if user == "" {
user = "root"
}
database := cfg.GetDoltDatabase()
if database == "" {
database = "beads"
}
connStr := fmt.Sprintf("%s@tcp(%s:%d)/%s?parseTime=true", user, host, port, database)
db, err := sql.Open("mysql", connStr)
if err != nil {
return nil, fmt.Errorf("failed to open Dolt server connection: %w", err)
}
db.SetMaxOpenConns(5)
db.SetMaxIdleConns(2)
return db, nil
}
// CheckDoltConnection verifies connectivity to the Dolt database.
func CheckDoltConnection(path string) DoctorCheck {
beadsDir := resolveBeadsDir(filepath.Join(path, ".beads"))
@ -88,8 +64,11 @@ func CheckDoltConnection(path string) DoctorCheck {
}
}
// Try to connect to Dolt server
db, err := openDoltServerDB(beadsDir)
// Try to connect to Dolt
doltDir := filepath.Join(beadsDir, "dolt")
connStr := fmt.Sprintf("file://%s?commitname=beads&commitemail=beads@local", doltDir)
db, err := sql.Open("dolt", connStr)
if err != nil {
return DoctorCheck{
Name: "Dolt Connection",
@ -99,17 +78,26 @@ func CheckDoltConnection(path string) DoctorCheck {
Category: CategoryCore,
}
}
defer db.Close()
defer closeDoltDBWithTimeout(db)
// Ping
// Switch to beads database and ping
ctx := context.Background()
if _, err := db.ExecContext(ctx, "USE beads"); err != nil {
return DoctorCheck{
Name: "Dolt Connection",
Status: StatusError,
Message: "Failed to switch to beads database",
Detail: err.Error(),
Category: CategoryCore,
}
}
if err := db.PingContext(ctx); err != nil {
return DoctorCheck{
Name: "Dolt Connection",
Status: StatusError,
Message: "Failed to ping Dolt database",
Detail: err.Error(),
Fix: "Ensure dolt sql-server is running",
Category: CategoryCore,
}
}
@ -118,7 +106,7 @@ func CheckDoltConnection(path string) DoctorCheck {
Name: "Dolt Connection",
Status: StatusOK,
Message: "Connected successfully",
Detail: "Storage: Dolt (server mode)",
Detail: "Storage: Dolt",
Category: CategoryCore,
}
}
@ -137,7 +125,10 @@ func CheckDoltSchema(path string) DoctorCheck {
}
}
db, err := openDoltServerDB(beadsDir)
doltDir := filepath.Join(beadsDir, "dolt")
connStr := fmt.Sprintf("file://%s?commitname=beads&commitemail=beads@local", doltDir)
db, err := sql.Open("dolt", connStr)
if err != nil {
return DoctorCheck{
Name: "Dolt Schema",
@ -147,9 +138,18 @@ func CheckDoltSchema(path string) DoctorCheck {
Category: CategoryCore,
}
}
defer db.Close()
defer closeDoltDBWithTimeout(db)
ctx := context.Background()
if _, err := db.ExecContext(ctx, "USE beads"); err != nil {
return DoctorCheck{
Name: "Dolt Schema",
Status: StatusError,
Message: "Failed to switch to beads database",
Detail: err.Error(),
Category: CategoryCore,
}
}
// Check required tables
requiredTables := []string{"issues", "dependencies", "config", "labels", "events"}
@ -213,8 +213,11 @@ func CheckDoltIssueCount(path string) DoctorCheck {
}
}
// Get Dolt count via server
db, err := openDoltServerDB(beadsDir)
// Get Dolt count
doltDir := filepath.Join(beadsDir, "dolt")
connStr := fmt.Sprintf("file://%s?commitname=beads&commitemail=beads@local", doltDir)
db, err := sql.Open("dolt", connStr)
if err != nil {
return DoctorCheck{
Name: "Dolt-JSONL Sync",
@ -224,9 +227,19 @@ func CheckDoltIssueCount(path string) DoctorCheck {
Category: CategoryData,
}
}
defer db.Close()
defer closeDoltDBWithTimeout(db)
ctx := context.Background()
if _, err := db.ExecContext(ctx, "USE beads"); err != nil {
return DoctorCheck{
Name: "Dolt-JSONL Sync",
Status: StatusError,
Message: "Failed to switch to beads database",
Detail: err.Error(),
Category: CategoryData,
}
}
var doltCount int
err = db.QueryRowContext(ctx, "SELECT COUNT(*) FROM issues").Scan(&doltCount)
if err != nil {
@ -271,7 +284,10 @@ func CheckDoltStatus(path string) DoctorCheck {
}
}
db, err := openDoltServerDB(beadsDir)
doltDir := filepath.Join(beadsDir, "dolt")
connStr := fmt.Sprintf("file://%s?commitname=beads&commitemail=beads@local", doltDir)
db, err := sql.Open("dolt", connStr)
if err != nil {
return DoctorCheck{
Name: "Dolt Status",
@ -281,9 +297,18 @@ func CheckDoltStatus(path string) DoctorCheck {
Category: CategoryData,
}
}
defer db.Close()
defer closeDoltDBWithTimeout(db)
ctx := context.Background()
if _, err := db.ExecContext(ctx, "USE beads"); err != nil {
return DoctorCheck{
Name: "Dolt Status",
Status: StatusWarning,
Message: "Could not switch to beads database",
Detail: err.Error(),
Category: CategoryData,
}
}
// Check dolt_status for uncommitted changes
rows, err := db.QueryContext(ctx, "SELECT table_name, staged, status FROM dolt_status")

View file

@ -1,3 +1,4 @@
//go:build cgo
package doctor
import (

View file

@ -0,0 +1,54 @@
//go:build !cgo
package doctor
// CheckFederationRemotesAPI returns N/A when CGO is not available.
func CheckFederationRemotesAPI(path string) DoctorCheck {
return DoctorCheck{
Name: "Federation remotesapi",
Status: StatusOK,
Message: "N/A (requires CGO)",
Category: CategoryFederation,
}
}
// CheckFederationPeerConnectivity returns N/A when CGO is not available.
func CheckFederationPeerConnectivity(path string) DoctorCheck {
return DoctorCheck{
Name: "Peer Connectivity",
Status: StatusOK,
Message: "N/A (requires CGO)",
Category: CategoryFederation,
}
}
// CheckFederationSyncStaleness returns N/A when CGO is not available.
func CheckFederationSyncStaleness(path string) DoctorCheck {
return DoctorCheck{
Name: "Sync Staleness",
Status: StatusOK,
Message: "N/A (requires CGO)",
Category: CategoryFederation,
}
}
// CheckFederationConflicts returns N/A when CGO is not available.
func CheckFederationConflicts(path string) DoctorCheck {
return DoctorCheck{
Name: "Federation Conflicts",
Status: StatusOK,
Message: "N/A (requires CGO)",
Category: CategoryFederation,
}
}
// CheckDoltServerModeMismatch returns N/A when CGO is not available.
func CheckDoltServerModeMismatch(path string) DoctorCheck {
return DoctorCheck{
Name: "Dolt Mode",
Status: StatusOK,
Message: "N/A (requires CGO)",
Category: CategoryFederation,
}
}

View file

@ -1,3 +1,5 @@
//go:build cgo
package doctor
import (
@ -10,6 +12,9 @@ import (
"path/filepath"
"strings"
// Import Dolt driver for direct connection
_ "github.com/dolthub/driver"
"github.com/steveyegge/beads/internal/beads"
"github.com/steveyegge/beads/internal/configfile"
"github.com/steveyegge/beads/internal/storage"
@ -516,15 +521,21 @@ func compareDoltWithJSONL(ctx context.Context, store storage.Storage, jsonlIDs m
return missing
}
// checkDoltLocks checks for uncommitted changes in Dolt via server connection.
// checkDoltLocks checks for uncommitted changes in Dolt.
func checkDoltLocks(beadsDir string) (bool, string) {
db, err := openDoltServerDB(beadsDir)
doltDir := filepath.Join(beadsDir, "dolt")
connStr := fmt.Sprintf("file://%s?commitname=beads&commitemail=beads@local", doltDir)
db, err := sql.Open("dolt", connStr)
if err != nil {
return false, ""
}
defer db.Close()
ctx := context.Background()
if _, err := db.ExecContext(ctx, "USE beads"); err != nil {
return false, ""
}
// Check dolt_status for uncommitted changes
rows, err := db.QueryContext(ctx, "SELECT table_name, staged, status FROM dolt_status")

View file

@ -0,0 +1,64 @@
//go:build !cgo
package doctor
// MigrationValidationResult provides machine-parseable migration validation output.
// This stub exists for non-CGO builds where Dolt is not available.
type MigrationValidationResult struct {
Phase string `json:"phase"`
Ready bool `json:"ready"`
Backend string `json:"backend"`
JSONLCount int `json:"jsonl_count"`
SQLiteCount int `json:"sqlite_count"`
DoltCount int `json:"dolt_count"`
MissingInDB []string `json:"missing_in_db"`
MissingInJSONL []string `json:"missing_in_jsonl"`
Errors []string `json:"errors"`
Warnings []string `json:"warnings"`
JSONLValid bool `json:"jsonl_valid"`
JSONLMalformed int `json:"jsonl_malformed"`
DoltHealthy bool `json:"dolt_healthy"`
DoltLocked bool `json:"dolt_locked"`
SchemaValid bool `json:"schema_valid"`
RecommendedFix string `json:"recommended_fix"`
}
// CheckMigrationReadiness returns N/A when CGO is not available.
func CheckMigrationReadiness(path string) (DoctorCheck, MigrationValidationResult) {
return DoctorCheck{
Name: "Migration Readiness",
Status: StatusOK,
Message: "N/A (requires CGO for Dolt)",
Category: CategoryMaintenance,
}, MigrationValidationResult{
Phase: "pre-migration",
Ready: false,
Backend: "unknown",
Errors: []string{"Dolt migration requires CGO build"},
}
}
// CheckMigrationCompletion returns N/A when CGO is not available.
func CheckMigrationCompletion(path string) (DoctorCheck, MigrationValidationResult) {
return DoctorCheck{
Name: "Migration Completion",
Status: StatusOK,
Message: "N/A (requires CGO for Dolt)",
Category: CategoryMaintenance,
}, MigrationValidationResult{
Phase: "post-migration",
Ready: false,
Backend: "unknown",
Errors: []string{"Dolt migration requires CGO build"},
}
}
// CheckDoltLocks returns N/A when CGO is not available.
func CheckDoltLocks(path string) DoctorCheck {
return DoctorCheck{
Name: "Dolt Locks",
Status: StatusOK,
Message: "N/A (requires CGO for Dolt)",
Category: CategoryMaintenance,
}
}

View file

@ -0,0 +1,64 @@
//go:build !cgo
package doctor
import (
"testing"
)
func TestCheckMigrationReadiness_NoCGO(t *testing.T) {
check, result := CheckMigrationReadiness("/tmp/nonexistent")
if check.Status != StatusOK {
t.Errorf("status = %q, want %q", check.Status, StatusOK)
}
if check.Message != "N/A (requires CGO for Dolt)" {
t.Errorf("message = %q, want %q", check.Message, "N/A (requires CGO for Dolt)")
}
if result.Ready {
t.Error("expected Ready = false in nocgo mode")
}
if len(result.Errors) == 0 {
t.Error("expected error message about CGO")
}
}
func TestCheckMigrationCompletion_NoCGO(t *testing.T) {
check, result := CheckMigrationCompletion("/tmp/nonexistent")
if check.Status != StatusOK {
t.Errorf("status = %q, want %q", check.Status, StatusOK)
}
if result.Ready {
t.Error("expected Ready = false in nocgo mode")
}
}
func TestCheckDoltLocks_NoCGO(t *testing.T) {
check := CheckDoltLocks("/tmp/nonexistent")
if check.Status != StatusOK {
t.Errorf("status = %q, want %q", check.Status, StatusOK)
}
if check.Message != "N/A (requires CGO for Dolt)" {
t.Errorf("message = %q, want %q", check.Message, "N/A (requires CGO for Dolt)")
}
}
func TestMigrationValidationResult_NoCGO(t *testing.T) {
// Test that the struct is properly defined even without CGO
result := MigrationValidationResult{
Phase: "test",
Ready: false,
Backend: "unknown",
}
if result.Phase != "test" {
t.Errorf("Phase = %q, want %q", result.Phase, "test")
}
}

View file

@ -1,3 +1,5 @@
//go:build cgo
package doctor
import (

View file

@ -1,3 +1,5 @@
//go:build cgo
package doctor
import (
@ -11,15 +13,15 @@ import (
"strings"
"time"
_ "github.com/dolthub/driver"
_ "github.com/go-sql-driver/mysql"
"github.com/steveyegge/beads/internal/configfile"
)
// DoltPerfMetrics holds performance metrics for Dolt operations
type DoltPerfMetrics struct {
Backend string // "dolt-server"
ServerStatus string // "running" or "not running"
Backend string // "dolt-embedded" or "dolt-server"
ServerMode bool // Whether connected via sql-server
ServerStatus string // "running", "not running", or N/A
Platform string // OS/arch
GoVersion string // Go runtime version
DoltVersion string // Dolt version if available
@ -55,17 +57,9 @@ func RunDoltPerformanceDiagnostics(path string, enableProfiling bool) (*DoltPerf
GoVersion: runtime.Version(),
}
// Load config for server connection settings
cfg, err := configfile.Load(beadsDir)
if err != nil || cfg == nil {
cfg = configfile.DefaultConfig()
}
host := cfg.GetDoltServerHost()
port := cfg.GetDoltServerPort()
// Check if server is running
// Check if server mode is available
doltDir := filepath.Join(beadsDir, "dolt")
serverRunning := isDoltServerRunning(host, port)
serverRunning := isDoltServerRunning("127.0.0.1", 3307)
if serverRunning {
metrics.ServerStatus = "running"
} else {
@ -83,13 +77,18 @@ func RunDoltPerformanceDiagnostics(path string, enableProfiling bool) (*DoltPerf
}
}
// Run server mode diagnostics
if !serverRunning {
return metrics, fmt.Errorf("dolt sql-server is not running; start it with: dolt sql-server")
}
if err := runDoltServerDiagnostics(metrics, host, port); err != nil {
return metrics, fmt.Errorf("server mode diagnostics failed: %w", err)
// Connect and run diagnostics - try server mode first if available
if serverRunning {
if err := runDoltServerDiagnostics(metrics, "127.0.0.1", 3307); err != nil {
fmt.Fprintf(os.Stderr, "Server mode diagnostics failed, falling back to embedded: %v\n", err)
if err := runDoltEmbeddedDiagnostics(metrics, doltDir); err != nil {
return metrics, fmt.Errorf("embedded mode diagnostics failed: %w", err)
}
}
} else {
if err := runDoltEmbeddedDiagnostics(metrics, doltDir); err != nil {
return metrics, fmt.Errorf("embedded mode diagnostics failed: %w", err)
}
}
// Calculate database size
@ -101,6 +100,7 @@ func RunDoltPerformanceDiagnostics(path string, enableProfiling bool) (*DoltPerf
// runDoltServerDiagnostics runs diagnostics via dolt sql-server
func runDoltServerDiagnostics(metrics *DoltPerfMetrics, host string, port int) error {
metrics.Backend = "dolt-server"
metrics.ServerMode = true
dsn := fmt.Sprintf("root:@tcp(%s:%d)/beads?parseTime=true", host, port)
@ -126,6 +126,41 @@ func runDoltServerDiagnostics(metrics *DoltPerfMetrics, host string, port int) e
return runDoltDiagnosticQueries(ctx, db, metrics)
}
// runDoltEmbeddedDiagnostics runs diagnostics via embedded Dolt
func runDoltEmbeddedDiagnostics(metrics *DoltPerfMetrics, doltDir string) error {
metrics.Backend = "dolt-embedded"
metrics.ServerMode = false
connStr := fmt.Sprintf("file://%s?commitname=beads&commitemail=beads@local", doltDir)
// Measure connection time (includes bootstrap overhead)
start := time.Now()
db, err := sql.Open("dolt", connStr)
if err != nil {
return fmt.Errorf("failed to open Dolt database: %w", err)
}
defer closeDoltDBWithTimeout(db)
// Single connection for embedded mode
db.SetMaxOpenConns(1)
db.SetMaxIdleConns(1)
ctx := context.Background()
// Switch to beads database
if _, err := db.ExecContext(ctx, "USE beads"); err != nil {
return fmt.Errorf("failed to switch to beads database: %w", err)
}
if err := db.PingContext(ctx); err != nil {
return fmt.Errorf("failed to ping database: %w", err)
}
metrics.ConnectionTime = time.Since(start).Milliseconds()
// Run all diagnostics
return runDoltDiagnosticQueries(ctx, db, metrics)
}
// runDoltDiagnosticQueries runs the diagnostic queries and populates metrics
func runDoltDiagnosticQueries(ctx context.Context, db *sql.DB, metrics *DoltPerfMetrics) error {
// Get issue counts
@ -275,7 +310,7 @@ func PrintDoltPerfReport(metrics *DoltPerfMetrics) {
fmt.Println(strings.Repeat("=", 50))
fmt.Printf("\nBackend: %s\n", metrics.Backend)
fmt.Printf("Server Status: %s\n", metrics.ServerStatus)
fmt.Printf("Server Mode: %v (status: %s)\n", metrics.ServerMode, metrics.ServerStatus)
fmt.Printf("Platform: %s\n", metrics.Platform)
fmt.Printf("Go: %s\n", metrics.GoVersion)
fmt.Printf("Dolt: %s\n", metrics.DoltVersion)
@ -288,7 +323,7 @@ func PrintDoltPerfReport(metrics *DoltPerfMetrics) {
fmt.Printf(" Database size: %s\n", metrics.DatabaseSize)
fmt.Printf("\nOperation Performance (ms):\n")
fmt.Printf(" Connection: %s\n", formatTiming(metrics.ConnectionTime))
fmt.Printf(" Connection/Bootstrap: %s\n", formatTiming(metrics.ConnectionTime))
fmt.Printf(" bd ready (GetReadyWork): %s\n", formatTiming(metrics.ReadyWorkTime))
fmt.Printf(" bd list --status=open: %s\n", formatTiming(metrics.ListOpenTime))
fmt.Printf(" bd show <issue>: %s\n", formatTiming(metrics.ShowIssueTime))
@ -319,6 +354,17 @@ func assessDoltPerformance(metrics *DoltPerfMetrics) {
var warnings []string
var recommendations []string
// Check connection time (embedded mode bootstrap overhead)
if !metrics.ServerMode && metrics.ConnectionTime > 500 {
warnings = append(warnings, fmt.Sprintf("High bootstrap time (%dms) in embedded mode", metrics.ConnectionTime))
recommendations = append(recommendations, "Consider using server mode: set BEADS_DOLT_SERVER_MODE=1")
}
// Check if server mode is available but not being used
if !metrics.ServerMode && metrics.ServerStatus == "running" {
recommendations = append(recommendations, "Server is running but not being used. Enable server mode for better performance.")
}
// Check ready work query time
if metrics.ReadyWorkTime > 200 {
warnings = append(warnings, fmt.Sprintf("Slow ready-work query (%dms)", metrics.ReadyWorkTime))
@ -381,7 +427,7 @@ func CheckDoltPerformance(path string) DoctorCheck {
var issues []string
if metrics.ConnectionTime > 1000 {
issues = append(issues, fmt.Sprintf("slow connection (%dms)", metrics.ConnectionTime))
issues = append(issues, fmt.Sprintf("slow bootstrap (%dms)", metrics.ConnectionTime))
}
if metrics.ReadyWorkTime > 500 {
@ -389,19 +435,84 @@ func CheckDoltPerformance(path string) DoctorCheck {
}
if len(issues) > 0 {
fix := "Run 'bd doctor perf-dolt' for detailed analysis"
if !metrics.ServerMode && metrics.ServerStatus != "running" {
fix = "Consider enabling server mode: BEADS_DOLT_SERVER_MODE=1"
}
return DoctorCheck{
Name: "Dolt Performance",
Status: StatusWarning,
Message: strings.Join(issues, "; "),
Fix: "Run 'bd doctor perf-dolt' for detailed analysis",
Fix: fix,
Category: CategoryPerformance,
}
}
mode := "embedded"
if metrics.ServerMode {
mode = "server"
}
return DoctorCheck{
Name: "Dolt Performance",
Status: StatusOK,
Message: fmt.Sprintf("OK (server mode, connect: %dms, ready: %dms)", metrics.ConnectionTime, metrics.ReadyWorkTime),
Message: fmt.Sprintf("OK (mode: %s, connect: %dms, ready: %dms)", mode, metrics.ConnectionTime, metrics.ReadyWorkTime),
Category: CategoryPerformance,
}
}
// CompareDoltModes runs diagnostics in both embedded and server mode for comparison
func CompareDoltModes(path string) error {
beadsDir := resolveBeadsDir(filepath.Join(path, ".beads"))
if !IsDoltBackend(beadsDir) {
return fmt.Errorf("not a Dolt backend")
}
doltDir := filepath.Join(beadsDir, "dolt")
serverRunning := isDoltServerRunning("127.0.0.1", 3307)
fmt.Println("\nDolt Mode Comparison")
fmt.Println(strings.Repeat("=", 50))
// Run embedded mode diagnostics
fmt.Println("\n[Embedded Mode]")
embeddedMetrics := &DoltPerfMetrics{}
if err := runDoltEmbeddedDiagnostics(embeddedMetrics, doltDir); err != nil {
fmt.Printf(" Error: %v\n", err)
} else {
fmt.Printf(" Connection time: %dms\n", embeddedMetrics.ConnectionTime)
fmt.Printf(" Ready-work query: %dms\n", embeddedMetrics.ReadyWorkTime)
fmt.Printf(" List open query: %dms\n", embeddedMetrics.ListOpenTime)
fmt.Printf(" Complex query: %dms\n", embeddedMetrics.ComplexQueryTime)
}
// Run server mode diagnostics if available
if serverRunning {
fmt.Println("\n[Server Mode]")
serverMetrics := &DoltPerfMetrics{}
if err := runDoltServerDiagnostics(serverMetrics, "127.0.0.1", 3307); err != nil {
fmt.Printf(" Error: %v\n", err)
} else {
fmt.Printf(" Connection time: %dms\n", serverMetrics.ConnectionTime)
fmt.Printf(" Ready-work query: %dms\n", serverMetrics.ReadyWorkTime)
fmt.Printf(" List open query: %dms\n", serverMetrics.ListOpenTime)
fmt.Printf(" Complex query: %dms\n", serverMetrics.ComplexQueryTime)
// Print comparison
if embeddedMetrics.ConnectionTime > 0 && serverMetrics.ConnectionTime > 0 {
fmt.Println("\n[Comparison]")
fmt.Printf(" Connection speedup: %.1fx faster in server mode\n",
float64(embeddedMetrics.ConnectionTime)/float64(serverMetrics.ConnectionTime))
if embeddedMetrics.ReadyWorkTime > 0 && serverMetrics.ReadyWorkTime > 0 {
fmt.Printf(" Ready-work speedup: %.1fx\n",
float64(embeddedMetrics.ReadyWorkTime)/float64(serverMetrics.ReadyWorkTime))
}
}
}
} else {
fmt.Println("\n[Server Mode]")
fmt.Println(" Not available (start with: dolt sql-server)")
}
fmt.Println()
return nil
}

View file

@ -69,7 +69,19 @@ func RunServerHealthChecks(path string) ServerHealthResult {
return result
}
// Run server health checks
// Check if server mode is configured
if !cfg.IsDoltServerMode() {
result.Checks = append(result.Checks, DoctorCheck{
Name: "Server Config",
Status: StatusOK,
Message: fmt.Sprintf("Dolt mode is '%s' (embedded is the default)", cfg.GetDoltMode()),
Detail: "Server health checks only apply when dolt_mode is explicitly set to 'server'",
Category: CategoryFederation,
})
return result
}
// Server mode is configured - run health checks
host := cfg.GetDoltServerHost()
port := cfg.GetDoltServerPort()

View file

@ -1,3 +1,4 @@
//go:build cgo
package main
import (
@ -6,7 +7,7 @@ import (
"github.com/steveyegge/beads/internal/storage/dolt"
)
// DoltServerHandle wraps a dolt.Server
// DoltServerHandle wraps a dolt.Server for CGO builds
type DoltServerHandle struct {
server *dolt.Server
}
@ -66,7 +67,7 @@ func (h *DoltServerHandle) Host() string {
return ""
}
// DoltServerAvailable returns true (Dolt server is always available)
// DoltServerAvailable returns true when CGO is available
func DoltServerAvailable() bool {
return true
}

View file

@ -0,0 +1,51 @@
//go:build !cgo
package main
import (
"context"
"errors"
)
// DoltServerHandle is a stub for non-CGO builds
type DoltServerHandle struct{}
// DoltDefaultSQLPort is the default SQL port for dolt server
const DoltDefaultSQLPort = 3306
// DoltDefaultRemotesAPIPort is the default remotesapi port for dolt server
const DoltDefaultRemotesAPIPort = 50051
// ErrDoltRequiresCGO is returned when dolt features are requested without CGO
var ErrDoltRequiresCGO = errors.New("dolt backend requires CGO; use pre-built binaries from GitHub releases or enable CGO")
// StartDoltServer returns an error in non-CGO builds
func StartDoltServer(ctx context.Context, dataDir, logFile string, sqlPort, remotePort int) (*DoltServerHandle, error) {
return nil, ErrDoltRequiresCGO
}
// Stop is a no-op stub
func (h *DoltServerHandle) Stop() error {
return nil
}
// SQLPort returns 0 in non-CGO builds
func (h *DoltServerHandle) SQLPort() int {
return 0
}
// RemotesAPIPort returns 0 in non-CGO builds
func (h *DoltServerHandle) RemotesAPIPort() int {
return 0
}
// Host returns empty string in non-CGO builds
func (h *DoltServerHandle) Host() string {
return ""
}
// DoltServerAvailable returns false when CGO is not available
func DoltServerAvailable() bool {
return false
}

View file

@ -0,0 +1,135 @@
package main
import (
"os"
"os/exec"
"path/filepath"
"strings"
"testing"
"time"
)
func writeDoltWorkspace(t *testing.T, workspaceDir string) (beadsDir string, doltDir string) {
t.Helper()
beadsDir = filepath.Join(workspaceDir, ".beads")
doltDir = filepath.Join(beadsDir, "dolt")
if err := os.MkdirAll(doltDir, 0o700); err != nil {
t.Fatalf("mkdir: %v", err)
}
metadata := `{
"database": "dolt",
"backend": "dolt"
}`
if err := os.WriteFile(filepath.Join(beadsDir, "metadata.json"), []byte(metadata), 0o600); err != nil {
t.Fatalf("write metadata.json: %v", err)
}
return beadsDir, doltDir
}
func TestDoltSingleProcess_ShouldAutoStartDaemonFalse(t *testing.T) {
oldDBPath := dbPath
t.Cleanup(func() { dbPath = oldDBPath })
dbPath = ""
ws := t.TempDir()
beadsDir, _ := writeDoltWorkspace(t, ws)
t.Setenv("BEADS_DIR", beadsDir)
// Ensure the finder sees a workspace root (and not the repo running tests).
oldWD, _ := os.Getwd()
_ = os.Chdir(ws)
t.Cleanup(func() { _ = os.Chdir(oldWD) })
if shouldAutoStartDaemon() {
t.Fatalf("expected shouldAutoStartDaemon() to be false for dolt backend")
}
}
func TestDoltSingleProcess_TryAutoStartDoesNotCreateStartlock(t *testing.T) {
oldDBPath := dbPath
t.Cleanup(func() { dbPath = oldDBPath })
dbPath = ""
ws := t.TempDir()
beadsDir, _ := writeDoltWorkspace(t, ws)
t.Setenv("BEADS_DIR", beadsDir)
socketPath := filepath.Join(ws, "bd.sock")
lockPath := socketPath + ".startlock"
ok := tryAutoStartDaemon(socketPath)
if ok {
t.Fatalf("expected tryAutoStartDaemon() to return false for dolt backend")
}
if _, err := os.Stat(lockPath); err == nil {
t.Fatalf("expected startlock not to be created for dolt backend: %s", lockPath)
} else if !os.IsNotExist(err) {
t.Fatalf("stat startlock: %v", err)
}
}
func TestDoltSingleProcess_DaemonGuardBlocksStartCommand(t *testing.T) {
oldDBPath := dbPath
t.Cleanup(func() { dbPath = oldDBPath })
dbPath = ""
ws := t.TempDir()
beadsDir, _ := writeDoltWorkspace(t, ws)
t.Setenv("BEADS_DIR", beadsDir)
// Use daemonStartCmd which has the guard attached.
// Ensure help and federation flags exist (cobra adds them during execution).
cmd := daemonStartCmd
cmd.Flags().Bool("help", false, "help")
// Note: federation flag is already registered in init()
err := guardDaemonStartForDolt(cmd, nil)
if err == nil {
t.Fatalf("expected daemon guard error for dolt backend without --federation")
}
// Guardrail wording may evolve; assert a stable intent.
if !strings.Contains(err.Error(), "daemon mode is not supported") {
t.Fatalf("expected error to mention daemon mode unsupported, got: %v", err)
}
}
// This test uses a helper subprocess because startDaemon calls os.Exit on failure.
func TestDoltSingleProcess_StartDaemonGuardrailExitsNonZero(t *testing.T) {
if os.Getenv("BD_TEST_HELPER_STARTDAEMON") == "1" {
// Helper mode: set up environment and invoke startDaemon (should os.Exit(1)).
ws := os.Getenv("BD_TEST_WORKSPACE")
_, doltDir := writeDoltWorkspace(t, ws)
// Ensure FindDatabasePath can resolve.
_ = os.Chdir(ws)
_ = os.Setenv("BEADS_DB", doltDir)
dbPath = ""
pidFile := filepath.Join(ws, ".beads", "daemon.pid")
startDaemon(5*time.Second, false, false, false, false, false, "", pidFile, "info", false, false, 0, 0)
return
}
ws := t.TempDir()
// Pre-create workspace structure so helper can just use it.
_, doltDir := writeDoltWorkspace(t, ws)
exe, err := os.Executable()
if err != nil {
t.Fatalf("os.Executable: %v", err)
}
cmd := exec.Command(exe, "-test.run", "^TestDoltSingleProcess_StartDaemonGuardrailExitsNonZero$", "-test.v")
cmd.Env = append(os.Environ(),
"BD_TEST_HELPER_STARTDAEMON=1",
"BD_TEST_WORKSPACE="+ws,
"BEADS_DB="+doltDir,
)
out, err := cmd.CombinedOutput()
if err == nil {
t.Fatalf("expected non-zero exit; output:\n%s", string(out))
}
if !strings.Contains(string(out), "daemon mode is not supported") {
t.Fatalf("expected output to mention daemon unsupported; got:\n%s", string(out))
}
}

View file

@ -1,3 +1,4 @@
//go:build cgo
package main
import (

View file

@ -0,0 +1,35 @@
//go:build !cgo
package main
import (
"fmt"
"github.com/spf13/cobra"
)
var federationCmd = &cobra.Command{
Use: "federation",
GroupID: "sync",
Short: "Manage peer-to-peer federation (requires CGO)",
Long: `Federation commands require CGO and the Dolt storage backend.
This binary was built without CGO support. To use federation features:
1. Use pre-built binaries from GitHub releases, or
2. Build from source with CGO enabled
Federation enables synchronized issue tracking across multiple Gas Towns,
each maintaining their own Dolt database while sharing updates via remotes.`,
Run: func(cmd *cobra.Command, args []string) {
fmt.Println("Federation requires CGO and Dolt backend.")
fmt.Println("")
fmt.Println("This binary was built without CGO support. To use federation:")
fmt.Println(" 1. Download pre-built binaries from GitHub releases")
fmt.Println(" 2. Or build from source with CGO enabled")
},
}
func init() {
rootCmd.AddCommand(federationCmd)
}

View file

@ -753,8 +753,8 @@ func init() {
initCmd.Flags().Bool("force", false, "Force re-initialization even if JSONL already has issues (may cause data loss)")
initCmd.Flags().Bool("from-jsonl", false, "Import from current .beads/issues.jsonl file instead of git history (preserves manual cleanups)")
// Dolt server mode flags
initCmd.Flags().Bool("server", false, "Configure Dolt server connection settings (server mode is always used)")
// Dolt server mode flags (bd-dolt.2.2)
initCmd.Flags().Bool("server", false, "Explicitly configure Dolt in server mode for high-concurrency (default: embedded)")
initCmd.Flags().String("server-host", "", "Dolt server host (default: 127.0.0.1)")
initCmd.Flags().Int("server-port", 0, "Dolt server port (default: 3307)")
initCmd.Flags().String("server-user", "", "Dolt server MySQL user (default: root)")

View file

@ -641,6 +641,17 @@ var rootCmd = &cobra.Command{
debug.Logf("wisp operation detected, using direct mode")
}
// Embedded Dolt is single-process-only; never use daemon/RPC.
// (Dolt server mode supports multi-process and won't trigger this.)
// This must be checked after dbPath is resolved.
if !noDaemon && singleProcessOnlyBackend() {
noDaemon = true
daemonStatus.AutoStartEnabled = false
daemonStatus.FallbackReason = FallbackSingleProcessOnly
daemonStatus.Detail = "backend is single-process-only (embedded dolt): daemon mode disabled; using direct mode"
debug.Logf("single-process backend detected, using direct mode")
}
// Try to connect to daemon first (unless --no-daemon flag is set or worktree safety check fails)
if noDaemon {
// Only set FallbackFlagNoDaemon if not already set by auto-bypass logic
@ -843,12 +854,22 @@ var rootCmd = &cobra.Command{
}
if backend == configfile.BackendDolt {
// Set advisory lock timeout for dolt embedded mode.
// Reads get a shorter timeout (shared lock, less contention expected).
// Writes get a longer timeout (exclusive lock, may need to wait for readers).
if useReadOnly {
opts.OpenTimeout = 5 * time.Second
} else {
opts.OpenTimeout = 15 * time.Second
}
// For Dolt, use the dolt subdirectory
doltPath := filepath.Join(beadsDir, "dolt")
// Load server connection config from metadata.json
// Check if server mode is configured in metadata.json
cfg, cfgErr := configfile.Load(beadsDir)
if cfgErr == nil && cfg != nil {
if cfgErr == nil && cfg != nil && cfg.IsDoltServerMode() {
opts.ServerMode = true
opts.ServerHost = cfg.GetDoltServerHost()
opts.ServerPort = cfg.GetDoltServerPort()
if cfg.Database != "" {

View file

@ -1,3 +1,4 @@
//go:build cgo
package main
import (

View file

@ -1,3 +1,5 @@
//go:build cgo
package main
import (

View file

@ -0,0 +1,35 @@
//go:build !cgo
package main
import (
"fmt"
"os"
"github.com/spf13/cobra"
)
var migrateDoltCmd = &cobra.Command{
Use: "dolt",
Short: "Migrate from SQLite to Dolt backend",
Long: `Migrate the current beads installation from SQLite to Dolt backend. (Requires CGO)`,
Run: func(cmd *cobra.Command, args []string) {
if jsonOutput {
outputJSON(map[string]interface{}{
"error": "dolt_not_available",
"message": "Dolt backend requires CGO. This binary was built without CGO support.",
})
} else {
fmt.Fprintf(os.Stderr, "Error: Dolt backend requires CGO\n")
fmt.Fprintf(os.Stderr, "This binary was built without CGO support.\n")
fmt.Fprintf(os.Stderr, "To use Dolt, rebuild with: CGO_ENABLED=1 go build\n")
}
os.Exit(1)
},
}
func init() {
migrateDoltCmd.Flags().Bool("dry-run", false, "Preview migration without making changes")
migrateDoltCmd.Flags().BoolP("yes", "y", false, "Skip confirmation prompt (for automation)")
migrateCmd.AddCommand(migrateDoltCmd)
}

View file

@ -0,0 +1,41 @@
//go:build !cgo
package main
import (
"fmt"
"os"
)
// handleToDoltMigration is a stub for non-cgo builds.
// Dolt requires CGO, so this migration is not available.
func handleToDoltMigration(dryRun bool, autoYes bool) {
if jsonOutput {
outputJSON(map[string]interface{}{
"error": "dolt_not_available",
"message": "Dolt backend requires CGO. This binary was built without CGO support.",
})
} else {
fmt.Fprintf(os.Stderr, "Error: Dolt backend requires CGO\n")
fmt.Fprintf(os.Stderr, "This binary was built without CGO support.\n")
fmt.Fprintf(os.Stderr, "To use Dolt, rebuild with: CGO_ENABLED=1 go build\n")
}
os.Exit(1)
}
// handleToSQLiteMigration is a stub for non-cgo builds.
// Dolt requires CGO, so this migration from Dolt is not available.
func handleToSQLiteMigration(dryRun bool, autoYes bool) {
if jsonOutput {
outputJSON(map[string]interface{}{
"error": "dolt_not_available",
"message": "Dolt backend requires CGO. This binary was built without CGO support.",
})
} else {
fmt.Fprintf(os.Stderr, "Error: Dolt backend requires CGO\n")
fmt.Fprintf(os.Stderr, "This binary was built without CGO support.\n")
fmt.Fprintf(os.Stderr, "To use Dolt, rebuild with: CGO_ENABLED=1 go build\n")
}
os.Exit(1)
}

View file

@ -34,7 +34,7 @@ func importFromJSONL(ctx context.Context, jsonlPath string, renameOnImport bool,
protectLeftSnapshot = opts[1]
}
// Guardrail: single-process backends must not spawn a helper `bd import`
// Guardrail: single-process backends (e.g., Dolt) must not spawn a helper `bd import`
// process while the parent holds an open store. Use inline import instead.
if singleProcessOnlyBackend() {
return importFromJSONLInline(ctx, jsonlPath, renameOnImport, noGitHistory, protectLeftSnapshot)

View file

@ -71,6 +71,31 @@ The installer will:
**TL;DR:** Use Homebrew if available. Use npm if you're in a Node.js environment. Use the script for quick one-off installs or CI.
## Build Dependencies (go install / from source)
If you install via `go install` or build from source, you need system dependencies for CGO:
macOS (Homebrew):
```bash
brew install icu4c zstd
```
Linux (Debian/Ubuntu):
```bash
sudo apt-get install -y libicu-dev libzstd-dev
```
Linux (Fedora/RHEL):
```bash
sudo dnf install -y libicu-devel libzstd-devel
```
If you see `unicode/uregex.h` missing on macOS, `icu4c` is keg-only. Use:
```bash
ICU_PREFIX="$(brew --prefix icu4c)"
CGO_CFLAGS="-I${ICU_PREFIX}/include" CGO_CPPFLAGS="-I${ICU_PREFIX}/include" CGO_LDFLAGS="-L${ICU_PREFIX}/lib" go install github.com/steveyegge/beads/cmd/bd@latest
```
## Platform-Specific Installation
### macOS
@ -150,13 +175,15 @@ irm https://raw.githubusercontent.com/steveyegge/beads/main/install.ps1 | iex
The script installs a prebuilt Windows release if available. Go is only required for `go install` or building from source.
**Dolt backend on Windows:** Supported. Connects to dolt sql-server via MySQL protocol.
**Dolt backend on Windows:** Supported via pure-Go regex backend. Windows builds automatically use Go's stdlib `regexp` instead of ICU regex to avoid CGO/header dependencies. If you need full ICU regex semantics, use Linux/macOS (or WSL) with ICU installed.
**Via go install**:
```pwsh
go install github.com/steveyegge/beads/cmd/bd@latest
```
The installer automatically applies the pure-Go regex backend on Windows.
**From source**:
```pwsh
git clone https://github.com/steveyegge/beads
@ -165,6 +192,8 @@ go build -o bd.exe ./cmd/bd
Move-Item bd.exe $env:USERPROFILE\AppData\Local\Microsoft\WindowsApps\
```
The build automatically applies the pure-Go regex backend on Windows via the `gms_pure_go` build tag. If you see `unicode/uregex.h` missing while building, this is normal—the build will skip it on Windows.
**Verify installation**:
```pwsh
bd version
@ -355,20 +384,21 @@ go install github.com/steveyegge/beads/cmd/bd@latest
### `zsh: killed bd` or crashes on macOS
Some users report crashes when running `bd init` or other commands on macOS. Try reinstalling:
Some users report crashes when running `bd init` or other commands on macOS. This is typically caused by CGO/SQLite compatibility issues.
**Workaround:**
```bash
# Via Homebrew (recommended)
brew reinstall beads
# Build with CGO enabled
CGO_ENABLED=1 go install github.com/steveyegge/beads/cmd/bd@latest
# Or rebuild from source
# Or if building from source
git clone https://github.com/steveyegge/beads
cd beads
go build -o bd ./cmd/bd
CGO_ENABLED=1 go build -o bd ./cmd/bd
sudo mv bd /usr/local/bin/
```
If you're still seeing crashes, please [file an issue](https://github.com/steveyegge/beads/issues).
If you installed via Homebrew, this shouldn't be necessary as the formula already enables CGO. If you're still seeing crashes with the Homebrew version, please [file an issue](https://github.com/steveyegge/beads/issues).
### Claude Code Plugin: MCP server fails to start

124
go.mod
View file

@ -1,6 +1,6 @@
module github.com/steveyegge/beads
go 1.25.5
go 1.25.6
require (
github.com/BurntSushi/toml v1.6.0
@ -9,6 +9,7 @@ require (
github.com/charmbracelet/glamour v0.10.0
github.com/charmbracelet/huh v0.8.0
github.com/charmbracelet/lipgloss v1.1.1-0.20250404203927-76690c660834
github.com/dolthub/driver v0.2.1-0.20260129210815-dc59f6dbac23
github.com/fsnotify/fsnotify v1.9.0
github.com/go-sql-driver/mysql v1.9.3
github.com/gofrs/flock v0.13.0
@ -26,14 +27,54 @@ require (
rsc.io/script v0.0.2
)
replace github.com/dolthub/go-icu-regex => ./internal/third_party/go-icu-regex
require (
cel.dev/expr v0.24.0 // indirect
cloud.google.com/go v0.120.0 // indirect
cloud.google.com/go/auth v0.16.2 // indirect
cloud.google.com/go/auth/oauth2adapt v0.2.8 // indirect
cloud.google.com/go/compute/metadata v0.7.0 // indirect
cloud.google.com/go/iam v1.5.2 // indirect
cloud.google.com/go/monitoring v1.24.2 // indirect
cloud.google.com/go/storage v1.50.0 // indirect
filippo.io/edwards25519 v1.1.0 // indirect
github.com/AlekSi/pointer v1.0.0 // indirect
github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.27.0 // indirect
github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.50.0 // indirect
github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.50.0 // indirect
github.com/HdrHistogram/hdrhistogram-go v1.1.2 // indirect
github.com/alecthomas/chroma/v2 v2.14.0 // indirect
github.com/aliyun/aliyun-oss-go-sdk v3.0.2+incompatible // indirect
github.com/apache/arrow/go/arrow v0.0.0-20211112161151-bc219186db40 // indirect
github.com/apache/thrift v0.19.0 // indirect
github.com/atotto/clipboard v0.1.4 // indirect
github.com/aws/aws-sdk-go-v2 v1.36.3 // indirect
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.10 // indirect
github.com/aws/aws-sdk-go-v2/config v1.29.8 // indirect
github.com/aws/aws-sdk-go-v2/credentials v1.17.61 // indirect
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.30 // indirect
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.64 // indirect
github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.34 // indirect
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.34 // indirect
github.com/aws/aws-sdk-go-v2/internal/ini v1.8.3 // indirect
github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.34 // indirect
github.com/aws/aws-sdk-go-v2/service/dynamodb v1.41.0 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.3 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.6.2 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/endpoint-discovery v1.10.15 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.15 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.15 // indirect
github.com/aws/aws-sdk-go-v2/service/s3 v1.78.0 // indirect
github.com/aws/aws-sdk-go-v2/service/sso v1.25.0 // indirect
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.29.0 // indirect
github.com/aws/aws-sdk-go-v2/service/sts v1.33.16 // indirect
github.com/aws/smithy-go v1.22.2 // indirect
github.com/aymanbagabas/go-osc52/v2 v2.0.1 // indirect
github.com/aymerick/douceur v0.2.0 // indirect
github.com/bcicen/jstream v1.0.1 // indirect
github.com/catppuccin/go v0.3.0 // indirect
github.com/cespare/xxhash/v2 v2.3.0 // indirect
github.com/charmbracelet/bubbles v0.21.1-0.20250623103423-23b8fd6302d7 // indirect
github.com/charmbracelet/bubbletea v1.3.6 // indirect
github.com/charmbracelet/colorprofile v0.2.3-0.20250311203215-f60798e515dc // indirect
@ -42,44 +83,121 @@ require (
github.com/charmbracelet/x/exp/slice v0.0.0-20250327172914-2fdc97757edf // indirect
github.com/charmbracelet/x/exp/strings v0.0.0-20240722160745-212f7b056ed0 // indirect
github.com/charmbracelet/x/term v0.2.1 // indirect
github.com/cncf/xds/go v0.0.0-20250501225837-2ac532fd4443 // indirect
github.com/denisbrodbeck/machineid v1.0.1 // indirect
github.com/dlclark/regexp2 v1.11.0 // indirect
github.com/dolthub/aws-sdk-go-ini-parser v0.0.0-20250305001723-2821c37f6c12 // indirect
github.com/dolthub/dolt/go v0.40.5-0.20260129204643-460dc5864c05 // indirect
github.com/dolthub/eventsapi_schema v0.0.0-20250915094920-eadfd39051ca // indirect
github.com/dolthub/flatbuffers/v23 v23.3.3-dh.2 // indirect
github.com/dolthub/fslock v0.0.3 // indirect
github.com/dolthub/go-icu-regex v0.0.0-20250916051405-78a38d478790 // indirect
github.com/dolthub/go-mysql-server v0.20.1-0.20260128201837-5b7ec92cc6e9 // indirect
github.com/dolthub/gozstd v0.0.0-20240423170813-23a2903bca63 // indirect
github.com/dolthub/jsonpath v0.0.2-0.20240227200619-19675ab05c71 // indirect
github.com/dolthub/vitess v0.0.0-20260128180459-bd171d35a7e2 // indirect
github.com/dustin/go-humanize v1.0.1 // indirect
github.com/edsrzf/mmap-go v1.2.0 // indirect
github.com/envoyproxy/go-control-plane/envoy v1.32.4 // indirect
github.com/envoyproxy/protoc-gen-validate v1.2.1 // indirect
github.com/erikgeiser/coninput v0.0.0-20211004153227-1c3628e74d0f // indirect
github.com/esote/minmaxheap v1.0.0 // indirect
github.com/fatih/color v1.16.0 // indirect
github.com/felixge/httpsnoop v1.0.4 // indirect
github.com/go-jose/go-jose/v4 v4.0.5 // indirect
github.com/go-logr/logr v1.4.3 // indirect
github.com/go-logr/stdr v1.2.2 // indirect
github.com/go-viper/mapstructure/v2 v2.4.0 // indirect
github.com/google/go-cmp v0.7.0 // indirect
github.com/goccy/go-json v0.10.2 // indirect
github.com/gocraft/dbr/v2 v2.7.6 // indirect
github.com/golang/snappy v0.0.4 // indirect
github.com/google/btree v1.1.2 // indirect
github.com/google/s2a-go v0.1.9 // indirect
github.com/google/uuid v1.6.0 // indirect
github.com/googleapis/enterprise-certificate-proxy v0.3.6 // indirect
github.com/googleapis/gax-go/v2 v2.14.2 // indirect
github.com/gorilla/css v1.0.1 // indirect
github.com/hashicorp/golang-lru v1.0.2 // indirect
github.com/hashicorp/golang-lru/v2 v2.0.7 // indirect
github.com/inconshreveable/mousetrap v1.1.0 // indirect
github.com/juju/gnuflag v1.0.0 // indirect
github.com/kch42/buzhash v0.0.0-20160816060738-9bdec3dec7c6 // indirect
github.com/klauspost/compress v1.18.0 // indirect
github.com/klauspost/cpuid/v2 v2.2.6 // indirect
github.com/kylelemons/godebug v1.1.0 // indirect
github.com/lestrrat-go/strftime v1.0.6 // indirect
github.com/lucasb-eyer/go-colorful v1.2.0 // indirect
github.com/mattn/go-colorable v0.1.13 // indirect
github.com/mattn/go-isatty v0.0.20 // indirect
github.com/mattn/go-localereader v0.0.1 // indirect
github.com/mattn/go-runewidth v0.0.16 // indirect
github.com/microcosm-cc/bluemonday v1.0.27 // indirect
github.com/mitchellh/hashstructure/v2 v2.0.2 // indirect
github.com/mohae/uvarint v0.0.0-20160208145430-c3f9e62bf2b0 // indirect
github.com/muesli/ansi v0.0.0-20230316100256-276c6243b2f6 // indirect
github.com/muesli/cancelreader v0.2.2 // indirect
github.com/muesli/reflow v0.3.0 // indirect
github.com/ncruces/julianday v1.0.0 // indirect
github.com/oracle/oci-go-sdk/v65 v65.55.0 // indirect
github.com/pelletier/go-toml/v2 v2.2.4 // indirect
github.com/pierrec/lz4/v4 v4.1.21 // indirect
github.com/pkg/errors v0.9.1 // indirect
github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 // indirect
github.com/pmezard/go-difflib v1.0.0 // indirect
github.com/rivo/uniseg v0.4.7 // indirect
github.com/rogpeppe/go-internal v1.13.1 // indirect
github.com/sagikazarmark/locafero v0.11.0 // indirect
github.com/sergi/go-diff v1.3.1 // indirect
github.com/shopspring/decimal v1.4.0 // indirect
github.com/silvasur/buzhash v0.0.0-20160816060738-9bdec3dec7c6 // indirect
github.com/sirupsen/logrus v1.9.3 // indirect
github.com/sony/gobreaker v0.5.0 // indirect
github.com/sourcegraph/conc v0.3.1-0.20240121214520-5f936abd7ae8 // indirect
github.com/spf13/afero v1.15.0 // indirect
github.com/spf13/cast v1.10.0 // indirect
github.com/spf13/pflag v1.0.10 // indirect
github.com/spiffe/go-spiffe/v2 v2.5.0 // indirect
github.com/subosito/gotenv v1.6.0 // indirect
github.com/tidwall/gjson v1.18.0 // indirect
github.com/tidwall/match v1.1.1 // indirect
github.com/tidwall/pretty v1.2.1 // indirect
github.com/tidwall/sjson v1.2.5 // indirect
github.com/vbauerster/mpb/v8 v8.7.2 // indirect
github.com/xitongsys/parquet-go v1.6.2 // indirect
github.com/xitongsys/parquet-go-source v0.0.0-20240122235623-d6294584ab18 // indirect
github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e // indirect
github.com/yuin/goldmark v1.7.8 // indirect
github.com/yuin/goldmark-emoji v1.0.5 // indirect
github.com/zeebo/errs v1.4.0 // indirect
github.com/zeebo/xxh3 v1.0.2 // indirect
go.opentelemetry.io/auto/sdk v1.1.0 // indirect
go.opentelemetry.io/contrib/detectors/gcp v1.36.0 // indirect
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.61.0 // indirect
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.61.0 // indirect
go.opentelemetry.io/otel v1.36.0 // indirect
go.opentelemetry.io/otel/metric v1.36.0 // indirect
go.opentelemetry.io/otel/sdk v1.36.0 // indirect
go.opentelemetry.io/otel/sdk/metric v1.36.0 // indirect
go.opentelemetry.io/otel/trace v1.36.0 // indirect
go.uber.org/multierr v1.11.0 // indirect
go.uber.org/zap v1.27.0 // indirect
go.yaml.in/yaml/v3 v3.0.4 // indirect
golang.org/x/crypto v0.47.0 // indirect
golang.org/x/exp v0.0.0-20240205201215-2c58cdc269a3 // indirect
golang.org/x/net v0.48.0 // indirect
golang.org/x/oauth2 v0.30.0 // indirect
golang.org/x/sync v0.19.0 // indirect
golang.org/x/telemetry v0.0.0-20251203150158-8fff8a5912fc // indirect
golang.org/x/text v0.33.0 // indirect
golang.org/x/time v0.12.0 // indirect
golang.org/x/tools v0.40.0 // indirect
golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 // indirect
google.golang.org/api v0.241.0 // indirect
google.golang.org/genproto v0.0.0-20250505200425-f936aa4a68b2 // indirect
google.golang.org/genproto/googleapis/api v0.0.0-20250528174236-200df99c418a // indirect
google.golang.org/genproto/googleapis/rpc v0.0.0-20250707201910-8d1bb00bc6a7 // indirect
google.golang.org/grpc v1.74.2 // indirect
google.golang.org/protobuf v1.36.8 // indirect
gopkg.in/go-jose/go-jose.v2 v2.6.3 // indirect
gopkg.in/src-d/go-errors.v1 v1.0.0 // indirect
gopkg.in/yaml.v2 v2.4.0 // indirect
)

1430
go.sum

File diff suppressed because it is too large Load diff

View file

@ -233,7 +233,14 @@ func findDatabaseInBeadsDir(beadsDir string, warnOnIssues bool) string {
backend := cfg.GetBackend()
if backend == configfile.BackendDolt {
// For Dolt server mode, database is on the server - no local directory required
return cfg.DatabasePath(beadsDir)
if cfg.IsDoltServerMode() {
return cfg.DatabasePath(beadsDir)
}
// For embedded Dolt, check if the configured database directory exists
doltPath := cfg.DatabasePath(beadsDir)
if info, err := os.Stat(doltPath); err == nil && info.IsDir() {
return doltPath
}
} else {
// For SQLite, check if the .db file exists
dbPath := cfg.DatabasePath(beadsDir)

View file

@ -19,10 +19,10 @@ type Config struct {
// Deletions configuration
DeletionsRetentionDays int `json:"deletions_retention_days,omitempty"` // 0 means use default (3 days)
// Dolt connection mode configuration
// Server mode connects to an external dolt sql-server via MySQL protocol.
// The "embedded" value is accepted for backward compatibility but treated as "server".
DoltMode string `json:"dolt_mode,omitempty"` // "server" (only mode supported)
// Dolt connection mode configuration (bd-dolt.2.2)
// Default is "embedded" (in-process). Server mode ("server") connects to an
// external dolt sql-server and should only be used for high-concurrency scenarios.
DoltMode string `json:"dolt_mode,omitempty"` // "embedded" (default) or "server"
DoltServerHost string `json:"dolt_server_host,omitempty"` // Server host (default: 127.0.0.1)
DoltServerPort int `json:"dolt_server_port,omitempty"` // Server port (default: 3307)
DoltServerUser string `json:"dolt_server_user,omitempty"` // MySQL user (default: root)
@ -170,6 +170,11 @@ const (
//
// This is intentionally small and stable: callers should use these flags to decide
// whether to enable features like daemon/RPC/autostart and process spawning.
//
// NOTE: The embedded Dolt driver is effectively single-writer at the OS-process level.
// Even if multiple goroutines are safe within one process, multiple processes opening
// the same Dolt directory concurrently can cause lock contention and transient
// "read-only" failures. Therefore, Dolt is treated as single-process-only.
type BackendCapabilities struct {
// SingleProcessOnly indicates the backend must not be accessed from multiple
// Beads OS processes concurrently (no daemon mode, no RPC client/server split,
@ -179,21 +184,33 @@ type BackendCapabilities struct {
// CapabilitiesForBackend returns capabilities for a backend string.
// Unknown backends are treated conservatively as single-process-only.
//
// Note: For Dolt, this returns SingleProcessOnly=true for embedded mode.
// Use Config.GetCapabilities() when you have the full config to properly
// handle server mode (which supports multi-process access).
func CapabilitiesForBackend(backend string) BackendCapabilities {
switch strings.TrimSpace(strings.ToLower(backend)) {
case "", BackendSQLite:
return BackendCapabilities{SingleProcessOnly: false}
case BackendDolt:
// Dolt uses server mode which supports multi-writer access
return BackendCapabilities{SingleProcessOnly: false}
// Embedded Dolt is single-process-only.
// Server mode is handled by Config.GetCapabilities().
return BackendCapabilities{SingleProcessOnly: true}
default:
return BackendCapabilities{SingleProcessOnly: true}
}
}
// GetCapabilities returns the backend capabilities for this config.
// Unlike CapabilitiesForBackend(string), this considers Dolt server mode
// which supports multi-process access.
func (c *Config) GetCapabilities() BackendCapabilities {
return CapabilitiesForBackend(c.GetBackend())
backend := c.GetBackend()
if backend == BackendDolt && c.IsDoltServerMode() {
// Server mode supports multi-writer, so NOT single-process-only
return BackendCapabilities{SingleProcessOnly: false}
}
return CapabilitiesForBackend(backend)
}
// GetBackend returns the configured backend type, defaulting to SQLite.
@ -219,14 +236,22 @@ const (
)
// IsDoltServerMode returns true if Dolt should connect via sql-server.
// Always returns true for Dolt backends (embedded mode has been removed).
// Server mode is opt-in for high-concurrency scenarios; embedded is the default.
// Checks the BEADS_DOLT_SERVER_MODE env var first, then falls back to the
// dolt_mode field in metadata.json. Only applies when backend is "dolt".
func (c *Config) IsDoltServerMode() bool {
return c.GetBackend() == BackendDolt
if os.Getenv("BEADS_DOLT_SERVER_MODE") == "1" && c.GetBackend() == BackendDolt {
return true
}
return c.GetBackend() == BackendDolt && strings.ToLower(c.DoltMode) == DoltModeServer
}
// GetDoltMode returns the Dolt connection mode. Always returns "server".
// GetDoltMode returns the Dolt connection mode, defaulting to embedded.
func (c *Config) GetDoltMode() string {
return DoltModeServer
if c.DoltMode == "" {
return DoltModeEmbedded
}
return c.DoltMode
}
// GetDoltServerHost returns the Dolt server host.

View file

@ -224,20 +224,20 @@ func TestDoltServerMode(t *testing.T) {
want: false,
},
{
name: "dolt always server mode",
cfg: &Config{Backend: BackendDolt},
want: true,
},
{
name: "dolt with legacy embedded config still returns true",
name: "dolt embedded mode",
cfg: &Config{Backend: BackendDolt, DoltMode: DoltModeEmbedded},
want: true,
want: false,
},
{
name: "dolt explicit server mode",
name: "dolt server mode",
cfg: &Config{Backend: BackendDolt, DoltMode: DoltModeServer},
want: true,
},
{
name: "dolt default mode",
cfg: &Config{Backend: BackendDolt},
want: false, // Default is embedded
},
}
for _, tt := range tests {
@ -251,21 +251,20 @@ func TestDoltServerMode(t *testing.T) {
})
t.Run("GetDoltMode", func(t *testing.T) {
// GetDoltMode always returns "server" regardless of config
tests := []struct {
name string
cfg *Config
want string
}{
{
name: "empty defaults to server",
name: "empty defaults to embedded",
cfg: &Config{},
want: DoltModeServer,
want: DoltModeEmbedded,
},
{
name: "legacy embedded treated as server",
name: "explicit embedded",
cfg: &Config{DoltMode: DoltModeEmbedded},
want: DoltModeServer,
want: DoltModeEmbedded,
},
{
name: "explicit server",
@ -369,19 +368,28 @@ func TestDoltServerMode(t *testing.T) {
})
}
// TestIsDoltServerModeAlwaysTrue tests that IsDoltServerMode is always true for Dolt
func TestIsDoltServerModeAlwaysTrue(t *testing.T) {
t.Run("dolt backend always server mode", func(t *testing.T) {
// TestIsDoltServerModeEnvVar tests env var overrides for IsDoltServerMode
func TestIsDoltServerModeEnvVar(t *testing.T) {
t.Run("env var override with dolt backend", func(t *testing.T) {
t.Setenv("BEADS_DOLT_SERVER_MODE", "1")
cfg := &Config{Backend: BackendDolt}
if !cfg.IsDoltServerMode() {
t.Error("IsDoltServerMode() = false, want true for dolt backend")
t.Error("IsDoltServerMode() = false, want true when env var set with dolt backend")
}
})
t.Run("sqlite backend not server mode", func(t *testing.T) {
t.Run("env var ignored for sqlite backend", func(t *testing.T) {
t.Setenv("BEADS_DOLT_SERVER_MODE", "1")
cfg := &Config{Backend: BackendSQLite}
if cfg.IsDoltServerMode() {
t.Error("IsDoltServerMode() = true, want false for sqlite backend")
t.Error("IsDoltServerMode() = true, want false when env var set but sqlite backend")
}
})
t.Run("env var not set", func(t *testing.T) {
cfg := &Config{Backend: BackendDolt}
if cfg.IsDoltServerMode() {
t.Error("IsDoltServerMode() = true, want false when no config or env var")
}
})
}
@ -399,8 +407,18 @@ func TestGetCapabilities(t *testing.T) {
wantSingleProc: false,
},
{
name: "dolt is multi-process (server mode)",
name: "dolt embedded is single-process",
cfg: &Config{Backend: BackendDolt, DoltMode: DoltModeEmbedded},
wantSingleProc: true,
},
{
name: "dolt default (empty) is single-process",
cfg: &Config{Backend: BackendDolt},
wantSingleProc: true,
},
{
name: "dolt server mode is multi-process",
cfg: &Config{Backend: BackendDolt, DoltMode: DoltModeServer},
wantSingleProc: false,
},
}

View file

@ -0,0 +1,97 @@
package dolt
import (
"errors"
"fmt"
"os"
"path/filepath"
"time"
"github.com/steveyegge/beads/internal/lockfile"
)
// AccessLock coordinates access to the embedded Dolt database using flock.
// Shared locks allow concurrent readers; exclusive locks ensure single-writer.
// The lock file lives in .beads/dolt-access.lock (alongside daemon.lock).
type AccessLock struct {
file *os.File
path string
}
const (
// accessLockFile is the name of the advisory lock file in .beads/.
accessLockFile = "dolt-access.lock"
// lockPollInterval is how often we retry acquiring the lock.
lockPollInterval = 50 * time.Millisecond
)
// AcquireAccessLock acquires an advisory flock on the dolt-access.lock file.
// If exclusive is true, acquires an exclusive lock (for writes); otherwise shared (for reads).
// Polls with lockPollInterval until timeout expires. Returns ErrLockBusy on timeout.
//
// The doltDir parameter is the path to the dolt data directory (e.g., .beads/dolt).
// The lock file is placed in the parent directory (.beads/) alongside daemon.lock.
func AcquireAccessLock(doltDir string, exclusive bool, timeout time.Duration) (*AccessLock, error) {
// Lock file goes in parent of dolt dir (i.e., .beads/)
beadsDir := filepath.Dir(doltDir)
lockPath := filepath.Join(beadsDir, accessLockFile)
// Ensure parent directory exists
if err := os.MkdirAll(beadsDir, 0o750); err != nil {
return nil, fmt.Errorf("create lock dir: %w", err)
}
// Open or create lock file
// #nosec G304 - controlled path derived from database configuration
f, err := os.OpenFile(lockPath, os.O_CREATE|os.O_RDWR, 0o600)
if err != nil {
return nil, fmt.Errorf("open access lock: %w", err)
}
// Pick the right lock function
lockFn := lockfile.FlockSharedNonBlock
if exclusive {
lockFn = lockfile.FlockExclusiveNonBlock
}
// Try once immediately
if err := lockFn(f); err == nil {
return &AccessLock{file: f, path: lockPath}, nil
} else if !errors.Is(err, lockfile.ErrLockBusy) {
_ = f.Close()
return nil, fmt.Errorf("access lock: %w", err)
}
// Poll until timeout
deadline := time.Now().Add(timeout)
for time.Now().Before(deadline) {
time.Sleep(lockPollInterval)
if err := lockFn(f); err == nil {
return &AccessLock{file: f, path: lockPath}, nil
} else if !errors.Is(err, lockfile.ErrLockBusy) {
_ = f.Close()
return nil, fmt.Errorf("access lock: %w", err)
}
}
_ = f.Close()
kind := "shared"
if exclusive {
kind = "exclusive"
}
return nil, fmt.Errorf("dolt access lock timeout (%s, %v): another bd process is using the database: %w",
kind, timeout, lockfile.ErrLockBusy)
}
// Release releases the advisory lock and closes the underlying file.
// Safe to call multiple times (idempotent).
func (l *AccessLock) Release() {
if l == nil || l.file == nil {
return
}
_ = lockfile.FlockUnlock(l.file)
_ = l.file.Close()
l.file = nil
}

View file

@ -1,3 +1,4 @@
//go:build cgo
package dolt
import (

View file

@ -0,0 +1,590 @@
//go:build cgo
package dolt
import (
"bufio"
"context"
"encoding/json"
"fmt"
"os"
"path/filepath"
"strings"
"time"
"github.com/steveyegge/beads/internal/audit"
"github.com/steveyegge/beads/internal/lockfile"
"github.com/steveyegge/beads/internal/routing"
"github.com/steveyegge/beads/internal/types"
"github.com/steveyegge/beads/internal/utils"
)
// BootstrapResult contains statistics about the bootstrap operation
type BootstrapResult struct {
IssuesImported int
IssuesSkipped int
RoutesImported int
InteractionsImported int
ParseErrors []ParseError
PrefixDetected string
}
// ParseError describes a JSONL parsing error
type ParseError struct {
Line int
Message string
Snippet string
}
// BootstrapConfig controls bootstrap behavior
type BootstrapConfig struct {
BeadsDir string // Path to .beads directory
DoltPath string // Path to dolt subdirectory
LockTimeout time.Duration // Timeout waiting for bootstrap lock
}
// Bootstrap checks if Dolt DB needs bootstrapping from JSONL and performs it if needed.
// This is called during store creation to handle the cold-start scenario where
// JSONL files exist (from git clone) but no Dolt database exists yet.
//
// Returns:
// - true, result, nil: Bootstrap was performed successfully
// - false, nil, nil: No bootstrap needed (Dolt already exists or no JSONL)
// - false, nil, err: Bootstrap failed
func Bootstrap(ctx context.Context, cfg BootstrapConfig) (bool, *BootstrapResult, error) {
if cfg.LockTimeout == 0 {
cfg.LockTimeout = 30 * time.Second
}
// Check if Dolt database already exists and is ready
if doltExists(cfg.DoltPath) && schemaReady(ctx, cfg.DoltPath) {
return false, nil, nil
}
// Check if JSONL exists to bootstrap from
jsonlPath := findJSONLPath(cfg.BeadsDir)
if jsonlPath == "" {
// No JSONL to bootstrap from - let normal init handle it
return false, nil, nil
}
// Acquire bootstrap lock to prevent concurrent bootstraps
lockPath := cfg.DoltPath + ".bootstrap.lock"
lockFile, err := acquireBootstrapLock(lockPath, cfg.LockTimeout)
if err != nil {
return false, nil, fmt.Errorf("bootstrap lock timeout: %w", err)
}
defer releaseBootstrapLock(lockFile, lockPath)
// Double-check after acquiring lock - another process may have bootstrapped
if doltExists(cfg.DoltPath) && schemaReady(ctx, cfg.DoltPath) {
return false, nil, nil
}
// Perform bootstrap
result, err := performBootstrap(ctx, cfg, jsonlPath)
if err != nil {
return false, nil, err
}
return true, result, nil
}
// doltExists checks if a Dolt database directory exists
func doltExists(doltPath string) bool {
// The embedded Dolt driver creates the database in a subdirectory
// named after the database (default: "beads"), with .dolt inside that.
// So we check for any subdirectory containing a .dolt directory.
entries, err := os.ReadDir(doltPath)
if err != nil {
return false
}
for _, entry := range entries {
// Use os.Stat to follow symlinks - entry.IsDir() returns false for symlinks
fullPath := filepath.Join(doltPath, entry.Name())
info, err := os.Stat(fullPath)
if err != nil {
continue
}
if info.IsDir() {
doltDir := filepath.Join(fullPath, ".dolt")
if doltInfo, err := os.Stat(doltDir); err == nil && doltInfo.IsDir() {
return true
}
}
}
return false
}
// schemaReady checks if the Dolt database has the required schema
// This is a simple check based on the existence of expected files.
// We avoid opening a connection here since the caller will do that.
func schemaReady(_ context.Context, doltPath string) bool {
// The embedded Dolt driver stores databases in subdirectories.
// Check for the expected database name's config.json which indicates
// the database was initialized.
configPath := filepath.Join(doltPath, "beads", ".dolt", "config.json")
_, err := os.Stat(configPath)
return err == nil
}
// findJSONLPath looks for JSONL files in the beads directory
func findJSONLPath(beadsDir string) string {
// Check in order of preference
candidates := []string{
filepath.Join(beadsDir, "issues.jsonl"),
filepath.Join(beadsDir, "beads.jsonl"), // Legacy name
}
for _, path := range candidates {
if info, err := os.Stat(path); err == nil && !info.IsDir() {
return path
}
}
return ""
}
// staleLockAge is the maximum age of a lock file before it's considered stale.
// Bootstrap operations should complete well within this window.
const staleLockAge = 5 * time.Minute
// acquireBootstrapLock acquires an exclusive lock for bootstrap operations.
// Uses non-blocking flock with polling to respect the timeout deadline.
// Detects and cleans up stale lock files from crashed processes.
func acquireBootstrapLock(lockPath string, timeout time.Duration) (*os.File, error) {
// Check for stale lock file before attempting to acquire.
// If the lock file is very old, the holding process likely crashed
// without cleanup. Remove it so we can proceed.
if info, err := os.Stat(lockPath); err == nil {
age := time.Since(info.ModTime())
if age > staleLockAge {
fmt.Fprintf(os.Stderr, "Bootstrap: removing stale lock file (age: %s)\n", age.Round(time.Second))
_ = os.Remove(lockPath)
}
}
// Create lock file
// #nosec G304 - controlled path
f, err := os.OpenFile(lockPath, os.O_CREATE|os.O_RDWR, 0600)
if err != nil {
return nil, fmt.Errorf("failed to create lock file: %w", err)
}
// Try to acquire lock with non-blocking flock and polling.
// The previous implementation used FlockExclusiveBlocking which blocks
// indefinitely, making the timeout unreachable.
deadline := time.Now().Add(timeout)
for {
err := lockfile.FlockExclusiveNonBlocking(f)
if err == nil {
// Lock acquired - update modification time for stale detection
return f, nil
}
if !lockfile.IsLocked(err) {
// Unexpected error (not contention)
_ = f.Close()
return nil, fmt.Errorf("failed to acquire bootstrap lock: %w", err)
}
if time.Now().After(deadline) {
_ = f.Close()
return nil, fmt.Errorf("timeout after %s waiting for bootstrap lock (another bootstrap may be running)", timeout)
}
// Wait briefly before retrying
time.Sleep(100 * time.Millisecond)
}
}
// releaseBootstrapLock releases the bootstrap lock and removes the lock file
func releaseBootstrapLock(f *os.File, lockPath string) {
if f != nil {
_ = lockfile.FlockUnlock(f)
_ = f.Close()
}
// Clean up lock file
_ = os.Remove(lockPath)
}
// performBootstrap performs the actual bootstrap from JSONL files.
// Import order: routes -> issues -> interactions (dependencies require issues to exist)
func performBootstrap(ctx context.Context, cfg BootstrapConfig, jsonlPath string) (*BootstrapResult, error) {
result := &BootstrapResult{}
// Parse issues JSONL with graceful error handling
issues, parseErrors := parseJSONLWithErrors(jsonlPath)
result.ParseErrors = parseErrors
if len(parseErrors) > 0 {
fmt.Fprintf(os.Stderr, "Bootstrap: Skipped %d malformed lines during bootstrap:\n", len(parseErrors))
maxShow := 5
if len(parseErrors) < maxShow {
maxShow = len(parseErrors)
}
for i := 0; i < maxShow; i++ {
e := parseErrors[i]
fmt.Fprintf(os.Stderr, " Line %d: %s\n", e.Line, e.Message)
}
if len(parseErrors) > maxShow {
fmt.Fprintf(os.Stderr, " ... and %d more\n", len(parseErrors)-maxShow)
}
}
if len(issues) == 0 {
return nil, fmt.Errorf("no valid issues found in JSONL file %s", jsonlPath)
}
// Detect prefix from issues
result.PrefixDetected = detectPrefixFromIssues(issues)
// Create Dolt store (this initializes schema)
store, err := New(ctx, &Config{Path: cfg.DoltPath})
if err != nil {
return nil, fmt.Errorf("failed to create Dolt store: %w", err)
}
defer func() { _ = store.Close() }()
// Set issue prefix
if result.PrefixDetected != "" {
if err := store.SetConfig(ctx, "issue_prefix", result.PrefixDetected); err != nil {
return nil, fmt.Errorf("failed to set issue_prefix: %w", err)
}
}
// Import routes first (no dependencies)
routesImported, err := importRoutesBootstrap(ctx, store, cfg.BeadsDir)
if err != nil {
// Non-fatal - routes.jsonl may not exist
fmt.Fprintf(os.Stderr, "Bootstrap: warning: failed to import routes: %v\n", err)
}
result.RoutesImported = routesImported
// Import issues in a transaction
imported, skipped, err := importIssuesBootstrap(ctx, store, issues)
if err != nil {
return nil, fmt.Errorf("failed to import issues: %w", err)
}
result.IssuesImported = imported
result.IssuesSkipped = skipped
// Import interactions (after issues, since interactions may reference issue_id)
interactionsPath := filepath.Join(cfg.BeadsDir, "interactions.jsonl")
interactionsImported, err := importInteractionsBootstrap(ctx, store, interactionsPath)
if err != nil {
// Non-fatal - interactions.jsonl may not exist
fmt.Fprintf(os.Stderr, "Bootstrap: warning: failed to import interactions: %v\n", err)
}
result.InteractionsImported = interactionsImported
// Commit the bootstrap
if err := store.Commit(ctx, "Bootstrap from JSONL"); err != nil {
// Non-fatal - data is still in the database
fmt.Fprintf(os.Stderr, "Bootstrap: warning: failed to create Dolt commit: %v\n", err)
}
return result, nil
}
// parseJSONLWithErrors parses JSONL, collecting errors instead of failing
func parseJSONLWithErrors(jsonlPath string) ([]*types.Issue, []ParseError) {
// #nosec G304 - controlled path
f, err := os.Open(jsonlPath)
if err != nil {
return nil, []ParseError{{Line: 0, Message: fmt.Sprintf("failed to open file: %v", err)}}
}
defer func() { _ = f.Close() }()
var issues []*types.Issue
var parseErrors []ParseError
scanner := bufio.NewScanner(f)
scanner.Buffer(make([]byte, 0, 1024), 2*1024*1024) // 2MB buffer for large lines
lineNo := 0
for scanner.Scan() {
lineNo++
line := scanner.Text()
if strings.TrimSpace(line) == "" {
continue
}
// Skip Git merge conflict markers
trimmed := strings.TrimSpace(line)
if strings.HasPrefix(trimmed, "<<<<<<< ") ||
trimmed == "=======" ||
strings.HasPrefix(trimmed, ">>>>>>> ") {
parseErrors = append(parseErrors, ParseError{
Line: lineNo,
Message: "Git merge conflict marker",
Snippet: truncateSnippet(line, 50),
})
continue
}
var issue types.Issue
if err := json.Unmarshal([]byte(line), &issue); err != nil {
parseErrors = append(parseErrors, ParseError{
Line: lineNo,
Message: err.Error(),
Snippet: truncateSnippet(line, 50),
})
continue
}
// Apply defaults for omitted fields
issue.SetDefaults()
// Fix closed_at invariant
if issue.Status == types.StatusClosed && issue.ClosedAt == nil {
now := time.Now()
issue.ClosedAt = &now
}
issues = append(issues, &issue)
}
if err := scanner.Err(); err != nil {
parseErrors = append(parseErrors, ParseError{
Line: lineNo,
Message: fmt.Sprintf("scanner error: %v", err),
})
}
return issues, parseErrors
}
// truncateSnippet truncates a string for display
func truncateSnippet(s string, maxLen int) string {
if len(s) > maxLen {
return s[:maxLen] + "..."
}
return s
}
// detectPrefixFromIssues detects the most common prefix from issues
func detectPrefixFromIssues(issues []*types.Issue) string {
prefixCounts := make(map[string]int)
for _, issue := range issues {
if issue.ID == "" {
continue
}
prefix := utils.ExtractIssuePrefix(issue.ID)
if prefix != "" {
prefixCounts[prefix]++
}
}
// Find most common prefix
var maxPrefix string
var maxCount int
for prefix, count := range prefixCounts {
if count > maxCount {
maxPrefix = prefix
maxCount = count
}
}
return maxPrefix
}
// importIssuesBootstrap imports issues during bootstrap
// Returns (imported, skipped, error)
func importIssuesBootstrap(ctx context.Context, store *DoltStore, issues []*types.Issue) (int, int, error) {
// Skip validation during bootstrap since we're importing existing data
// The data was already validated when originally created
tx, err := store.db.BeginTx(ctx, nil)
if err != nil {
return 0, 0, fmt.Errorf("failed to begin transaction: %w", err)
}
defer func() { _ = tx.Rollback() }()
imported := 0
skipped := 0
seenIDs := make(map[string]bool)
for _, issue := range issues {
// Skip duplicates within batch
if seenIDs[issue.ID] {
skipped++
continue
}
seenIDs[issue.ID] = true
// Set timestamps if missing
now := time.Now().UTC()
if issue.CreatedAt.IsZero() {
issue.CreatedAt = now
}
if issue.UpdatedAt.IsZero() {
issue.UpdatedAt = now
}
// Compute content hash if missing
if issue.ContentHash == "" {
issue.ContentHash = issue.ComputeContentHash()
}
// Insert issue
if err := insertIssue(ctx, tx, issue); err != nil {
// Check for duplicate key (issue already exists)
if strings.Contains(err.Error(), "Duplicate entry") ||
strings.Contains(err.Error(), "UNIQUE constraint") {
skipped++
continue
}
return imported, skipped, fmt.Errorf("failed to insert issue %s: %w", issue.ID, err)
}
// Import labels
for _, label := range issue.Labels {
_, err := tx.ExecContext(ctx, `
INSERT INTO labels (issue_id, label)
VALUES (?, ?)
`, issue.ID, label)
if err != nil && !strings.Contains(err.Error(), "Duplicate entry") {
return imported, skipped, fmt.Errorf("failed to insert label for %s: %w", issue.ID, err)
}
}
imported++
}
// Import dependencies in a second pass (after all issues exist)
for _, issue := range issues {
for _, dep := range issue.Dependencies {
// Check if both issues exist
var exists int
err := tx.QueryRowContext(ctx, "SELECT 1 FROM issues WHERE id = ?", dep.DependsOnID).Scan(&exists)
if err != nil {
// Target doesn't exist, skip dependency
continue
}
_, err = tx.ExecContext(ctx, `
INSERT INTO dependencies (issue_id, depends_on_id, type, created_by, created_at)
VALUES (?, ?, ?, ?, ?)
ON DUPLICATE KEY UPDATE type = type
`, dep.IssueID, dep.DependsOnID, dep.Type, "bootstrap", time.Now().UTC())
if err != nil && !strings.Contains(err.Error(), "Duplicate entry") {
// Non-fatal for dependencies
fmt.Fprintf(os.Stderr, "Bootstrap: warning: failed to import dependency %s -> %s: %v\n",
dep.IssueID, dep.DependsOnID, err)
}
}
}
if err := tx.Commit(); err != nil {
return imported, skipped, fmt.Errorf("failed to commit: %w", err)
}
return imported, skipped, nil
}
// importRoutesBootstrap imports routes from routes.jsonl during bootstrap
// Returns the number of routes imported
func importRoutesBootstrap(ctx context.Context, store *DoltStore, beadsDir string) (int, error) {
routes, err := routing.LoadRoutes(beadsDir)
if err != nil {
return 0, err
}
if len(routes) == 0 {
return 0, nil // No routes to import
}
tx, err := store.db.BeginTx(ctx, nil)
if err != nil {
return 0, fmt.Errorf("failed to begin transaction: %w", err)
}
defer func() { _ = tx.Rollback() }()
imported := 0
for _, route := range routes {
_, err := tx.ExecContext(ctx, `
INSERT INTO routes (prefix, path, created_at)
VALUES (?, ?, ?)
ON DUPLICATE KEY UPDATE path = VALUES(path)
`, route.Prefix, route.Path, time.Now().UTC())
if err != nil {
return imported, fmt.Errorf("failed to insert route %s: %w", route.Prefix, err)
}
imported++
}
if err := tx.Commit(); err != nil {
return imported, fmt.Errorf("failed to commit routes: %w", err)
}
return imported, nil
}
// importInteractionsBootstrap imports interactions from interactions.jsonl during bootstrap
// Returns the number of interactions imported
func importInteractionsBootstrap(ctx context.Context, store *DoltStore, interactionsPath string) (int, error) {
// #nosec G304 - controlled path
f, err := os.Open(interactionsPath)
if err != nil {
if os.IsNotExist(err) {
return 0, nil // No interactions file is not an error
}
return 0, err
}
defer func() { _ = f.Close() }()
tx, err := store.db.BeginTx(ctx, nil)
if err != nil {
return 0, fmt.Errorf("failed to begin transaction: %w", err)
}
defer func() { _ = tx.Rollback() }()
imported := 0
scanner := bufio.NewScanner(f)
scanner.Buffer(make([]byte, 0, 1024), 2*1024*1024) // 2MB buffer for large lines
for scanner.Scan() {
line := scanner.Text()
if strings.TrimSpace(line) == "" {
continue
}
var entry audit.Entry
if err := json.Unmarshal([]byte(line), &entry); err != nil {
// Skip malformed lines during bootstrap
continue
}
// Convert extra map to JSON (default to empty object for valid JSON)
extraJSON := []byte("{}")
if entry.Extra != nil {
extraJSON, _ = json.Marshal(entry.Extra)
}
_, err := tx.ExecContext(ctx, `
INSERT INTO interactions (id, kind, created_at, actor, issue_id, model, prompt, response, error, tool_name, exit_code, parent_id, label, reason, extra)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
ON DUPLICATE KEY UPDATE kind = kind
`, entry.ID, entry.Kind, entry.CreatedAt, entry.Actor, entry.IssueID, entry.Model, entry.Prompt, entry.Response, entry.Error, entry.ToolName, entry.ExitCode, entry.ParentID, entry.Label, entry.Reason, extraJSON)
if err != nil && !strings.Contains(err.Error(), "Duplicate entry") {
// Non-fatal - skip individual failures
fmt.Fprintf(os.Stderr, "Bootstrap: warning: failed to import interaction %s: %v\n", entry.ID, err)
continue
}
imported++
}
if err := scanner.Err(); err != nil {
return imported, fmt.Errorf("scanner error: %w", err)
}
if err := tx.Commit(); err != nil {
return imported, fmt.Errorf("failed to commit interactions: %w", err)
}
return imported, nil
}

View file

@ -0,0 +1,503 @@
//go:build cgo
package dolt
import (
"context"
"encoding/json"
"os"
"path/filepath"
"strings"
"testing"
"time"
"github.com/steveyegge/beads/internal/types"
)
func TestBootstrapFromJSONL(t *testing.T) {
// Create temp directory structure
tmpDir := t.TempDir()
beadsDir := filepath.Join(tmpDir, ".beads")
doltDir := filepath.Join(beadsDir, "dolt")
if err := os.MkdirAll(beadsDir, 0755); err != nil {
t.Fatalf("failed to create beads dir: %v", err)
}
// Create test JSONL file with issues
jsonlPath := filepath.Join(beadsDir, "issues.jsonl")
issues := []types.Issue{
{
ID: "test-001",
Title: "First issue",
Description: "Test description 1",
Status: types.StatusOpen,
Priority: 2,
IssueType: types.TypeTask,
CreatedAt: time.Now().Add(-time.Hour),
UpdatedAt: time.Now(),
},
{
ID: "test-002",
Title: "Second issue",
Description: "Test description 2",
Status: types.StatusInProgress,
Priority: 1,
IssueType: types.TypeBug,
CreatedAt: time.Now().Add(-2 * time.Hour),
UpdatedAt: time.Now().Add(-30 * time.Minute),
Labels: []string{"urgent", "backend"},
},
{
ID: "test-003",
Title: "Closed issue",
Status: types.StatusClosed,
Priority: 3,
IssueType: types.TypeTask,
},
}
var jsonlContent strings.Builder
for _, issue := range issues {
data, err := json.Marshal(issue)
if err != nil {
t.Fatalf("failed to marshal issue: %v", err)
}
jsonlContent.Write(data)
jsonlContent.WriteString("\n")
}
if err := os.WriteFile(jsonlPath, []byte(jsonlContent.String()), 0644); err != nil {
t.Fatalf("failed to write JSONL: %v", err)
}
// Perform bootstrap
ctx := context.Background()
bootstrapped, result, err := Bootstrap(ctx, BootstrapConfig{
BeadsDir: beadsDir,
DoltPath: doltDir,
LockTimeout: 10 * time.Second,
})
if err != nil {
t.Fatalf("bootstrap failed: %v", err)
}
if !bootstrapped {
t.Fatal("expected bootstrap to be performed")
}
if result == nil {
t.Fatal("expected non-nil result")
}
// Verify results
if result.IssuesImported != 3 {
t.Errorf("expected 3 issues imported, got %d", result.IssuesImported)
}
if result.PrefixDetected != "test" {
t.Errorf("expected prefix 'test', got '%s'", result.PrefixDetected)
}
// Open store and verify issues were imported
store, err := New(ctx, &Config{Path: doltDir})
if err != nil {
t.Fatalf("failed to open store: %v", err)
}
defer store.Close()
// Check prefix was set
prefix, err := store.GetConfig(ctx, "issue_prefix")
if err != nil {
t.Fatalf("failed to get config: %v", err)
}
if prefix != "test" {
t.Errorf("expected prefix 'test', got '%s'", prefix)
}
// Check issues exist
issue1, err := store.GetIssue(ctx, "test-001")
if err != nil {
t.Fatalf("failed to get issue: %v", err)
}
if issue1 == nil {
t.Fatal("expected issue test-001 to exist")
}
if issue1.Title != "First issue" {
t.Errorf("expected title 'First issue', got '%s'", issue1.Title)
}
// Check labels were imported
issue2, err := store.GetIssue(ctx, "test-002")
if err != nil {
t.Fatalf("failed to get issue: %v", err)
}
if issue2 == nil {
t.Fatal("expected issue test-002 to exist")
}
if len(issue2.Labels) != 2 {
t.Errorf("expected 2 labels, got %d", len(issue2.Labels))
}
// Verify closed_at was set for closed issue
issue3, err := store.GetIssue(ctx, "test-003")
if err != nil {
t.Fatalf("failed to get issue: %v", err)
}
if issue3 == nil {
t.Fatal("expected issue test-003 to exist")
}
if issue3.ClosedAt == nil {
t.Error("expected closed_at to be set for closed issue")
}
}
func TestBootstrapNoOpWhenDoltExists(t *testing.T) {
// Create temp directory structure with existing Dolt DB
tmpDir := t.TempDir()
beadsDir := filepath.Join(tmpDir, ".beads")
doltDir := filepath.Join(beadsDir, "dolt")
if err := os.MkdirAll(beadsDir, 0755); err != nil {
t.Fatalf("failed to create beads dir: %v", err)
}
// Create a Dolt store first
ctx := context.Background()
store, err := New(ctx, &Config{Path: doltDir})
if err != nil {
t.Fatalf("failed to create store: %v", err)
}
if err := store.SetConfig(ctx, "issue_prefix", "existing"); err != nil {
t.Fatalf("failed to set config: %v", err)
}
store.Close()
// Create JSONL file
jsonlPath := filepath.Join(beadsDir, "issues.jsonl")
issue := types.Issue{
ID: "new-001",
Title: "New issue",
Status: types.StatusOpen,
}
data, _ := json.Marshal(issue)
if err := os.WriteFile(jsonlPath, append(data, '\n'), 0644); err != nil {
t.Fatalf("failed to write JSONL: %v", err)
}
// Attempt bootstrap - should be no-op
bootstrapped, result, err := Bootstrap(ctx, BootstrapConfig{
BeadsDir: beadsDir,
DoltPath: doltDir,
LockTimeout: 10 * time.Second,
})
if err != nil {
t.Fatalf("bootstrap failed: %v", err)
}
if bootstrapped {
t.Error("expected no bootstrap when Dolt already exists")
}
if result != nil {
t.Error("expected nil result when no bootstrap performed")
}
// Verify original prefix preserved
store, err = New(ctx, &Config{Path: doltDir})
if err != nil {
t.Fatalf("failed to reopen store: %v", err)
}
defer store.Close()
prefix, _ := store.GetConfig(ctx, "issue_prefix")
if prefix != "existing" {
t.Errorf("expected prefix 'existing', got '%s'", prefix)
}
}
func TestBootstrapNoOpWhenNoJSONL(t *testing.T) {
// Create temp directory structure without JSONL
tmpDir := t.TempDir()
beadsDir := filepath.Join(tmpDir, ".beads")
doltDir := filepath.Join(beadsDir, "dolt")
if err := os.MkdirAll(beadsDir, 0755); err != nil {
t.Fatalf("failed to create beads dir: %v", err)
}
// Attempt bootstrap - should be no-op
ctx := context.Background()
bootstrapped, result, err := Bootstrap(ctx, BootstrapConfig{
BeadsDir: beadsDir,
DoltPath: doltDir,
LockTimeout: 10 * time.Second,
})
if err != nil {
t.Fatalf("bootstrap failed: %v", err)
}
if bootstrapped {
t.Error("expected no bootstrap when no JSONL exists")
}
if result != nil {
t.Error("expected nil result when no bootstrap performed")
}
}
func TestBootstrapGracefulDegradation(t *testing.T) {
// Create temp directory structure
tmpDir := t.TempDir()
beadsDir := filepath.Join(tmpDir, ".beads")
doltDir := filepath.Join(beadsDir, "dolt")
if err := os.MkdirAll(beadsDir, 0755); err != nil {
t.Fatalf("failed to create beads dir: %v", err)
}
// Create JSONL with some malformed lines
jsonlPath := filepath.Join(beadsDir, "issues.jsonl")
goodIssue := types.Issue{
ID: "test-001",
Title: "Good issue",
Status: types.StatusOpen,
}
goodData, _ := json.Marshal(goodIssue)
content := string(goodData) + "\n" +
"{invalid json}\n" +
"<<<<<<< HEAD\n" + // Git conflict marker
string(goodData) + "\n" // Duplicate - will be skipped
if err := os.WriteFile(jsonlPath, []byte(content), 0644); err != nil {
t.Fatalf("failed to write JSONL: %v", err)
}
// Perform bootstrap
ctx := context.Background()
bootstrapped, result, err := Bootstrap(ctx, BootstrapConfig{
BeadsDir: beadsDir,
DoltPath: doltDir,
LockTimeout: 10 * time.Second,
})
if err != nil {
t.Fatalf("bootstrap failed: %v", err)
}
if !bootstrapped {
t.Fatal("expected bootstrap to be performed")
}
// Should have parse errors for malformed lines
if len(result.ParseErrors) != 2 {
t.Errorf("expected 2 parse errors, got %d", len(result.ParseErrors))
}
// Should have imported the good issue
if result.IssuesImported != 1 {
t.Errorf("expected 1 issue imported, got %d", result.IssuesImported)
}
// Duplicate should be skipped (not errored)
if result.IssuesSkipped != 1 {
t.Errorf("expected 1 issue skipped, got %d", result.IssuesSkipped)
}
}
func TestParseJSONLWithErrors(t *testing.T) {
// Create temp file with mixed content
tmpDir := t.TempDir()
jsonlPath := filepath.Join(tmpDir, "test.jsonl")
goodIssue := types.Issue{
ID: "test-001",
Title: "Good issue",
Status: types.StatusOpen,
}
goodData, _ := json.Marshal(goodIssue)
content := string(goodData) + "\n" +
"\n" + // Empty line - should be skipped
" \n" + // Whitespace line - should be skipped
"{broken\n" + // Invalid JSON
"<<<<<<< HEAD\n" + // Git conflict
"=======\n" + // Git conflict
">>>>>>> branch\n" + // Git conflict
string(goodData) + "\n" // Another good line
if err := os.WriteFile(jsonlPath, []byte(content), 0644); err != nil {
t.Fatalf("failed to write file: %v", err)
}
issues, errors := parseJSONLWithErrors(jsonlPath)
if len(issues) != 2 {
t.Errorf("expected 2 issues, got %d", len(issues))
}
// Should have 4 parse errors: 1 invalid JSON + 3 conflict markers
if len(errors) != 4 {
t.Errorf("expected 4 parse errors, got %d: %+v", len(errors), errors)
}
}
func TestDetectPrefixFromIssues(t *testing.T) {
issues := []*types.Issue{
{ID: "proj-001"},
{ID: "proj-002"},
{ID: "proj-003"},
{ID: "other-001"},
}
prefix := detectPrefixFromIssues(issues)
if prefix != "proj" {
t.Errorf("expected prefix 'proj', got '%s'", prefix)
}
}
func TestBootstrapWithRoutesAndInteractions(t *testing.T) {
// Create temp directory structure
tmpDir := t.TempDir()
beadsDir := filepath.Join(tmpDir, ".beads")
doltDir := filepath.Join(beadsDir, "dolt")
if err := os.MkdirAll(beadsDir, 0755); err != nil {
t.Fatalf("failed to create beads dir: %v", err)
}
// Create test issues JSONL
jsonlPath := filepath.Join(beadsDir, "issues.jsonl")
issue := types.Issue{
ID: "test-001",
Title: "Test issue",
Status: types.StatusOpen,
}
data, _ := json.Marshal(issue)
if err := os.WriteFile(jsonlPath, append(data, '\n'), 0644); err != nil {
t.Fatalf("failed to write issues JSONL: %v", err)
}
// Create test routes JSONL
routesPath := filepath.Join(beadsDir, "routes.jsonl")
routesContent := `{"prefix":"test-","path":"."}
{"prefix":"other-","path":"other/rig"}
`
if err := os.WriteFile(routesPath, []byte(routesContent), 0644); err != nil {
t.Fatalf("failed to write routes JSONL: %v", err)
}
// Create test interactions JSONL
interactionsPath := filepath.Join(beadsDir, "interactions.jsonl")
interactionsContent := `{"id":"int-001","kind":"llm_call","created_at":"2025-01-20T10:00:00Z","actor":"test-agent","model":"claude-3"}
{"id":"int-002","kind":"tool_call","created_at":"2025-01-20T10:01:00Z","actor":"test-agent","tool_name":"bash","exit_code":0}
`
if err := os.WriteFile(interactionsPath, []byte(interactionsContent), 0644); err != nil {
t.Fatalf("failed to write interactions JSONL: %v", err)
}
// Perform bootstrap
ctx := context.Background()
bootstrapped, result, err := Bootstrap(ctx, BootstrapConfig{
BeadsDir: beadsDir,
DoltPath: doltDir,
LockTimeout: 10 * time.Second,
})
if err != nil {
t.Fatalf("bootstrap failed: %v", err)
}
if !bootstrapped {
t.Fatal("expected bootstrap to be performed")
}
if result == nil {
t.Fatal("expected non-nil result")
}
// Verify issues imported
if result.IssuesImported != 1 {
t.Errorf("expected 1 issue imported, got %d", result.IssuesImported)
}
// Verify routes imported
if result.RoutesImported != 2 {
t.Errorf("expected 2 routes imported, got %d", result.RoutesImported)
}
// Verify interactions imported
if result.InteractionsImported != 2 {
t.Errorf("expected 2 interactions imported, got %d", result.InteractionsImported)
}
// Open store and verify data
store, err := New(ctx, &Config{Path: doltDir})
if err != nil {
t.Fatalf("failed to open store: %v", err)
}
defer store.Close()
// Verify routes table
var routeCount int
err = store.db.QueryRowContext(ctx, "SELECT COUNT(*) FROM routes").Scan(&routeCount)
if err != nil {
t.Fatalf("failed to count routes: %v", err)
}
if routeCount != 2 {
t.Errorf("expected 2 routes in table, got %d", routeCount)
}
// Verify interactions table
var interactionCount int
err = store.db.QueryRowContext(ctx, "SELECT COUNT(*) FROM interactions").Scan(&interactionCount)
if err != nil {
t.Fatalf("failed to count interactions: %v", err)
}
if interactionCount != 2 {
t.Errorf("expected 2 interactions in table, got %d", interactionCount)
}
}
func TestBootstrapWithoutOptionalFiles(t *testing.T) {
// Test that bootstrap succeeds when routes.jsonl and interactions.jsonl don't exist
tmpDir := t.TempDir()
beadsDir := filepath.Join(tmpDir, ".beads")
doltDir := filepath.Join(beadsDir, "dolt")
if err := os.MkdirAll(beadsDir, 0755); err != nil {
t.Fatalf("failed to create beads dir: %v", err)
}
// Create only issues JSONL
jsonlPath := filepath.Join(beadsDir, "issues.jsonl")
issue := types.Issue{
ID: "test-001",
Title: "Test issue",
Status: types.StatusOpen,
}
data, _ := json.Marshal(issue)
if err := os.WriteFile(jsonlPath, append(data, '\n'), 0644); err != nil {
t.Fatalf("failed to write issues JSONL: %v", err)
}
// Perform bootstrap - should succeed even without routes.jsonl and interactions.jsonl
ctx := context.Background()
bootstrapped, result, err := Bootstrap(ctx, BootstrapConfig{
BeadsDir: beadsDir,
DoltPath: doltDir,
LockTimeout: 10 * time.Second,
})
if err != nil {
t.Fatalf("bootstrap failed: %v", err)
}
if !bootstrapped {
t.Fatal("expected bootstrap to be performed")
}
if result == nil {
t.Fatal("expected non-nil result")
}
// Verify issues imported
if result.IssuesImported != 1 {
t.Errorf("expected 1 issue imported, got %d", result.IssuesImported)
}
// Routes and interactions should be 0 (files don't exist)
if result.RoutesImported != 0 {
t.Errorf("expected 0 routes imported, got %d", result.RoutesImported)
}
if result.InteractionsImported != 0 {
t.Errorf("expected 0 interactions imported, got %d", result.InteractionsImported)
}
}

View file

@ -1,4 +1,4 @@
// Package dolt provides concurrency tests for Dolt with multiple writers.
// Package dolt provides concurrency tests for embedded Dolt with multiple writers.
//
// These tests validate that Gas Town can safely run multiple polecats concurrently,
// all writing to the same Dolt DB for creating issues, updating status,

View file

@ -1,3 +1,4 @@
//go:build cgo
package dolt
import (

View file

@ -1,3 +1,4 @@
//go:build cgo
package dolt
import (

View file

@ -1,3 +1,4 @@
//go:build cgo
package dolt
import (
@ -83,7 +84,7 @@ func (s *DoltStore) GetDependenciesWithMetadata(ctx context.Context, issueID str
}
// Collect dep metadata first, then close rows before fetching issues.
// This avoids connection pool deadlock when MaxOpenConns=1.
// This avoids connection pool deadlock when MaxOpenConns=1 (embedded dolt).
type depMeta struct {
depID, depType string
}
@ -149,7 +150,7 @@ func (s *DoltStore) GetDependentsWithMetadata(ctx context.Context, issueID strin
}
// Collect dep metadata first, then close rows before fetching issues.
// This avoids connection pool deadlock when MaxOpenConns=1.
// This avoids connection pool deadlock when MaxOpenConns=1 (embedded dolt).
type depMeta struct {
depID, depType string
}

View file

@ -1,3 +1,4 @@
//go:build cgo
package dolt
import (
@ -11,10 +12,10 @@ import (
// =============================================================================
func TestGetDependenciesWithMetadata(t *testing.T) {
// Note: This test is skipped because GetDependenciesWithMetadata
// Note: This test is skipped in embedded Dolt mode because GetDependenciesWithMetadata
// makes nested GetIssue calls inside a rows cursor, which can cause connection issues.
// This is a known limitation of the current implementation (see bd-tdgo.3).
t.Skip("Skipping: GetDependenciesWithMetadata has nested query issue in Dolt mode")
t.Skip("Skipping: GetDependenciesWithMetadata has nested query issue in embedded Dolt mode")
}
func TestGetDependenciesWithMetadata_NoResults(t *testing.T) {
@ -51,10 +52,10 @@ func TestGetDependenciesWithMetadata_NoResults(t *testing.T) {
// =============================================================================
func TestGetDependentsWithMetadata(t *testing.T) {
// Note: This test is skipped because GetDependentsWithMetadata
// Note: This test is skipped in embedded Dolt mode because GetDependentsWithMetadata
// makes nested GetIssue calls inside a rows cursor, which can cause connection issues.
// This is a known limitation of the current implementation (see bd-tdgo.3).
t.Skip("Skipping: GetDependentsWithMetadata has nested query issue in Dolt mode")
t.Skip("Skipping: GetDependentsWithMetadata has nested query issue in embedded Dolt mode")
}
// =============================================================================

View file

@ -1,3 +1,4 @@
//go:build cgo
package dolt
import (

View file

@ -1,3 +1,5 @@
//go:build cgo
// Package dolt provides performance benchmarks for the Dolt storage backend.
// Run with: go test -bench=. -benchmem ./internal/storage/dolt/...
//
@ -69,9 +71,9 @@ func setupBenchStore(b *testing.B) (*DoltStore, func()) {
// Bootstrap & Connection Benchmarks
// =============================================================================
// BenchmarkBootstrap measures store initialization time.
// BenchmarkBootstrapEmbedded measures store initialization time in embedded mode.
// This is the critical path for CLI commands that open/close the store each time.
func BenchmarkBootstrap(b *testing.B) {
func BenchmarkBootstrapEmbedded(b *testing.B) {
if _, err := os.LookupEnv("DOLT_PATH"); err != false {
if _, err := os.Stat("/usr/local/bin/dolt"); os.IsNotExist(err) {
if _, err := os.Stat("/usr/bin/dolt"); os.IsNotExist(err) {
@ -94,6 +96,7 @@ func BenchmarkBootstrap(b *testing.B) {
CommitterName: "bench",
CommitterEmail: "bench@example.com",
Database: "benchdb",
ServerMode: false, // Force embedded mode
}
initStore, err := New(ctx, cfg)
@ -141,6 +144,7 @@ func BenchmarkColdStart(b *testing.B) {
CommitterName: "bench",
CommitterEmail: "bench@example.com",
Database: "benchdb",
ServerMode: false, // Force embedded mode
}
b.ResetTimer()
@ -224,6 +228,7 @@ func BenchmarkCLIWorkflow(b *testing.B) {
CommitterName: "bench",
CommitterEmail: "bench@example.com",
Database: "benchdb",
ServerMode: false,
}
b.ResetTimer()

View file

@ -15,6 +15,9 @@ import (
)
// testTimeout is the maximum time for any single test operation.
// The embedded Dolt driver can be slow, especially for complex JOIN queries.
// If tests are timing out, it may indicate an issue with the embedded Dolt
// driver's async operations rather than with the DoltStore implementation.
const testTimeout = 30 * time.Second
// testContext returns a context with timeout for test operations
@ -70,8 +73,7 @@ func setupTestStore(t *testing.T) (*DoltStore, func()) {
store, err := New(ctx, cfg)
if err != nil {
os.RemoveAll(tmpDir)
// Server mode requires a running Dolt server - skip if unavailable
t.Skipf("failed to create Dolt store: %v", err)
t.Fatalf("failed to create Dolt store: %v", err)
}
// Set up issue prefix

View file

@ -0,0 +1,90 @@
//go:build cgo
package dolt
import (
"context"
"database/sql"
"errors"
embedded "github.com/dolthub/driver"
"github.com/steveyegge/beads/internal/storage/doltutil"
)
func ignoreContextCanceled(err error) error {
if err == nil {
return nil
}
// Dolt engine close paths may surface context.Canceled from background goroutines / shutdown plumbing.
// For unit-of-work helpers, we treat that as non-fatal cleanup noise.
if errors.Is(err, context.Canceled) {
return nil
}
return err
}
// withEmbeddedDolt executes exactly one unit of work using a single embedded Dolt connector.
//
// Lifecycle:
// 1) ParseDSN
// 2) configure (e.g. enable retries by setting cfg.BackOff)
// 3) NewConnector
// 4) sql.OpenDB(connector)
// 5) PingContext(ctx) to force open (and retries)
// 6) fn(ctx, db)
// 7) db.Close()
// 8) connector.Close() to release filesystem locks
//
// IMPORTANT: This helper does not wrap or modify ctx (no timeouts). The embedded driver derives
// a session context from the Connect(ctx) context and reuses it across statements.
func withEmbeddedDolt(
ctx context.Context,
dsn string,
configure func(cfg *embedded.Config),
fn func(ctx context.Context, db *sql.DB) error,
) (err error) {
if ctx == nil {
ctx = context.Background()
}
if fn == nil {
return errors.New("withEmbeddedDolt: fn is required")
}
cfg, err := embedded.ParseDSN(dsn)
if err != nil {
return err
}
if configure != nil {
configure(&cfg)
}
connector, err := embedded.NewConnector(cfg)
if err != nil {
return err
}
db := sql.OpenDB(connector)
defer func() {
// Close DB first (stops pool activity), then close the connector to release engine locks.
// Use CloseWithTimeout to prevent indefinite hangs (same issue as DoltStore.Close).
cerr := errors.Join(
ignoreContextCanceled(doltutil.CloseWithTimeout("db", db.Close)),
ignoreContextCanceled(doltutil.CloseWithTimeout("connector", connector.Close)),
)
if err == nil {
err = cerr
} else {
err = errors.Join(err, cerr)
}
}()
// Force open (and retries if configured) before entering the unit of work.
if err := db.PingContext(ctx); err != nil {
return err
}
return fn(ctx, db)
}

View file

@ -1,3 +1,4 @@
//go:build cgo
package dolt
import (

View file

@ -1,3 +1,4 @@
//go:build cgo
package dolt
import (

View file

@ -1,3 +1,4 @@
//go:build cgo
package dolt
import (

View file

@ -1,3 +1,4 @@
//go:build cgo
package dolt
import (

View file

@ -489,6 +489,7 @@ func setupHubWithNSpokes(t *testing.T, ctx context.Context, n int) (*HubSpokeSet
hubStore, err := New(ctx, &Config{
Path: hubDir,
Database: "beads",
ServerMode: true,
ServerHost: "127.0.0.1",
ServerPort: 14307,
CommitterName: "hub-mayor",
@ -560,6 +561,7 @@ func setupHubWithNSpokes(t *testing.T, ctx context.Context, n int) (*HubSpokeSet
spokeStore, err := New(ctx, &Config{
Path: spokeDir,
Database: "beads",
ServerMode: true,
ServerHost: "127.0.0.1",
ServerPort: 14308 + i,
CommitterName: spokeName,

View file

@ -1,3 +1,4 @@
//go:build cgo
package dolt
import (

View file

@ -1,3 +1,4 @@
//go:build cgo
package dolt
import (

View file

@ -1,3 +1,4 @@
//go:build cgo
package dolt
import (
@ -127,9 +128,9 @@ func TestGetLabelsForIssues_NoLabels(t *testing.T) {
func TestGetIssuesByLabel(t *testing.T) {
// Skip: GetIssuesByLabel makes nested queries (GetIssue calls inside a rows cursor)
// which can cause connection issues with single-connection pools.
// which can cause connection issues in embedded Dolt mode.
// This is a known limitation that should be fixed in bd-tdgo.3.
t.Skip("Skipping: GetIssuesByLabel has nested query issue in Dolt mode")
t.Skip("Skipping: GetIssuesByLabel has nested query issue in embedded Dolt mode")
}
func TestGetIssuesByLabel_NoMatches(t *testing.T) {

View file

@ -1,3 +1,4 @@
//go:build cgo
package dolt
import (

View file

@ -1,3 +1,4 @@
//go:build cgo
package migrations
import (

View file

@ -1,3 +1,4 @@
//go:build cgo
package migrations
import (

View file

@ -1,3 +1,4 @@
//go:build cgo
package migrations
import (

View file

@ -1,85 +1,70 @@
//go:build cgo
package migrations
import (
"database/sql"
"fmt"
"os"
"os/exec"
"path/filepath"
"testing"
"time"
_ "github.com/go-sql-driver/mysql"
embedded "github.com/dolthub/driver"
"github.com/steveyegge/beads/internal/storage/doltutil"
)
// openTestDolt creates a temporary Dolt database via sql-server for testing.
// openTestDolt creates a temporary embedded Dolt database for testing.
func openTestDolt(t *testing.T) *sql.DB {
t.Helper()
// Skip if dolt is not available
if _, err := exec.LookPath("dolt"); err != nil {
t.Skip("dolt not installed, skipping test")
}
tmpDir := t.TempDir()
dbPath := filepath.Join(tmpDir, "testdb")
if err := os.MkdirAll(dbPath, 0755); err != nil {
t.Fatalf("failed to create db dir: %v", err)
}
// Initialize dolt repo
cmd := exec.Command("dolt", "init")
cmd.Dir = dbPath
if err := cmd.Run(); err != nil {
t.Fatalf("failed to init dolt repo: %v", err)
}
// Start dolt sql-server on a random-ish port
port := 13400 + os.Getpid()%100
serverCmd := exec.Command("dolt", "sql-server",
"--host", "127.0.0.1",
"--port", fmt.Sprintf("%d", port),
"--no-auto-commit",
)
serverCmd.Dir = dbPath
serverCmd.Stdout = os.Stderr
serverCmd.Stderr = os.Stderr
if err := serverCmd.Start(); err != nil {
t.Fatalf("failed to start dolt sql-server: %v", err)
}
t.Cleanup(func() {
if serverCmd.Process != nil {
_ = serverCmd.Process.Kill()
_ = serverCmd.Wait()
}
})
// Wait for server to be ready
dsn := fmt.Sprintf("root:@tcp(127.0.0.1:%d)/?parseTime=true", port)
var db *sql.DB
var err error
for i := 0; i < 30; i++ {
db, err = sql.Open("mysql", dsn)
if err == nil {
if pingErr := db.Ping(); pingErr == nil {
break
}
_ = db.Close()
}
time.Sleep(100 * time.Millisecond)
}
absPath, err := filepath.Abs(dbPath)
if err != nil {
t.Fatalf("failed to connect to dolt sql-server: %v", err)
t.Fatalf("failed to get abs path: %v", err)
}
// Create database and schema
if _, err := db.Exec("CREATE DATABASE IF NOT EXISTS beads"); err != nil {
// First connect without database to create it
initDSN := fmt.Sprintf("file://%s?commitname=test&commitemail=test@test.com", absPath)
initCfg, err := embedded.ParseDSN(initDSN)
if err != nil {
t.Fatalf("failed to parse init DSN: %v", err)
}
initConnector, err := embedded.NewConnector(initCfg)
if err != nil {
t.Fatalf("failed to create init connector: %v", err)
}
initDB := sql.OpenDB(initConnector)
_, err = initDB.Exec("CREATE DATABASE IF NOT EXISTS beads")
if err != nil {
_ = doltutil.CloseWithTimeout("initDB", initDB.Close)
_ = doltutil.CloseWithTimeout("initConnector", initConnector.Close)
t.Fatalf("failed to create database: %v", err)
}
if _, err := db.Exec("USE beads"); err != nil {
t.Fatalf("failed to use database: %v", err)
_ = doltutil.CloseWithTimeout("initDB", initDB.Close)
_ = doltutil.CloseWithTimeout("initConnector", initConnector.Close)
// Now connect with database specified
dsn := fmt.Sprintf("file://%s?commitname=test&commitemail=test@test.com&database=beads", absPath)
cfg, err := embedded.ParseDSN(dsn)
if err != nil {
t.Fatalf("failed to parse DSN: %v", err)
}
connector, err := embedded.NewConnector(cfg)
if err != nil {
t.Fatalf("failed to create connector: %v", err)
}
t.Cleanup(func() { _ = doltutil.CloseWithTimeout("connector", connector.Close) })
db := sql.OpenDB(connector)
t.Cleanup(func() { _ = doltutil.CloseWithTimeout("db", db.Close) })
// Create minimal issues table without wisp_type (simulating old schema)
_, err = db.Exec(`CREATE TABLE IF NOT EXISTS issues (
id VARCHAR(255) PRIMARY KEY,
@ -92,15 +77,6 @@ func openTestDolt(t *testing.T) *sql.DB {
t.Fatalf("failed to create issues table: %v", err)
}
// Reconnect with database in DSN
_ = db.Close()
dbDSN := fmt.Sprintf("root:@tcp(127.0.0.1:%d)/beads?parseTime=true", port)
db, err = sql.Open("mysql", dbDSN)
if err != nil {
t.Fatalf("failed to connect to beads database: %v", err)
}
t.Cleanup(func() { _ = db.Close() })
return db
}
@ -175,3 +151,5 @@ func TestTableExists(t *testing.T) {
t.Fatal("nonexistent table should not exist")
}
}

View file

@ -65,6 +65,7 @@ func TestServerModeNestedQueries(t *testing.T) {
store, err := New(ctx, &Config{
Path: tmpDir,
Database: "beads",
ServerMode: true,
ServerHost: "127.0.0.1",
ServerPort: 13308,
})
@ -193,6 +194,7 @@ func TestServerModeDependencyQueries(t *testing.T) {
store, err := New(ctx, &Config{
Path: tmpDir,
Database: "beads",
ServerMode: true,
ServerHost: "127.0.0.1",
ServerPort: 13309,
})
@ -348,6 +350,7 @@ func TestServerModeTransactionSearchIssues(t *testing.T) {
store, err := New(ctx, &Config{
Path: tmpDir,
Database: "beads",
ServerMode: true,
ServerHost: "127.0.0.1",
ServerPort: 13310,
})
@ -473,6 +476,7 @@ func TestServerModeScanIssueIDs(t *testing.T) {
store, err := New(ctx, &Config{
Path: tmpDir,
Database: "beads",
ServerMode: true,
ServerHost: "127.0.0.1",
ServerPort: 13311,
})

View file

@ -599,6 +599,7 @@ func setupTwoTowns(t *testing.T, ctx context.Context) (*TownSetup, *TownSetup) {
alphaStore, err := New(ctx, &Config{
Path: alphaDir,
Database: "beads",
ServerMode: true,
ServerHost: "127.0.0.1",
ServerPort: 13307,
CommitterName: "alpha-town",
@ -661,6 +662,7 @@ func setupTwoTowns(t *testing.T, ctx context.Context) (*TownSetup, *TownSetup) {
betaStore, err := New(ctx, &Config{
Path: betaDir,
Database: "beads",
ServerMode: true,
ServerHost: "127.0.0.1",
ServerPort: 13308,
CommitterName: "beta-town",

View file

@ -1,4 +1,4 @@
//go:build !windows
//go:build cgo && !windows
package dolt

View file

@ -1,4 +1,4 @@
//go:build windows
//go:build cgo && windows
package dolt

View file

@ -1,4 +1,4 @@
//go:build !windows
//go:build cgo && !windows
package dolt

View file

@ -1,4 +1,4 @@
//go:build windows
//go:build cgo && windows
package dolt

View file

@ -1,3 +1,4 @@
//go:build cgo
package dolt
import (

View file

@ -1,3 +1,4 @@
//go:build cgo
package dolt
import (

View file

@ -1,3 +1,4 @@
//go:build cgo
package dolt
import (
@ -69,8 +70,26 @@ func TestIsRetryableError(t *testing.T) {
}
}
func TestWithRetry_Success(t *testing.T) {
store := &DoltStore{}
func TestWithRetry_EmbeddedMode(t *testing.T) {
// In embedded mode (serverMode=false), withRetry should just call the operation once
store := &DoltStore{serverMode: false}
callCount := 0
err := store.withRetry(context.Background(), func() error {
callCount++
return errors.New("driver: bad connection")
})
if err == nil {
t.Error("expected error, got nil")
}
if callCount != 1 {
t.Errorf("expected 1 call in embedded mode, got %d", callCount)
}
}
func TestWithRetry_ServerMode_Success(t *testing.T) {
store := &DoltStore{serverMode: true}
callCount := 0
err := store.withRetry(context.Background(), func() error {
@ -86,8 +105,8 @@ func TestWithRetry_Success(t *testing.T) {
}
}
func TestWithRetry_RetryOnBadConnection(t *testing.T) {
store := &DoltStore{}
func TestWithRetry_ServerMode_RetryOnBadConnection(t *testing.T) {
store := &DoltStore{serverMode: true}
callCount := 0
err := store.withRetry(context.Background(), func() error {
@ -106,8 +125,8 @@ func TestWithRetry_RetryOnBadConnection(t *testing.T) {
}
}
func TestWithRetry_NonRetryableError(t *testing.T) {
store := &DoltStore{}
func TestWithRetry_ServerMode_NonRetryableError(t *testing.T) {
store := &DoltStore{serverMode: true}
callCount := 0
err := store.withRetry(context.Background(), func() error {

View file

@ -1,3 +1,4 @@
//go:build cgo
package dolt
// schema defines the MySQL-compatible database schema for Dolt.

View file

@ -1,8 +1,9 @@
//go:build cgo
// Package dolt implements the storage interface using Dolt (versioned MySQL-compatible database).
//
// This file implements the dolt sql-server management for federation mode.
// When federation is enabled, we run dolt sql-server to enable multi-writer
// support and expose the remotesapi for peer-to-peer sync.
// When federation is enabled, we run dolt sql-server instead of the embedded driver
// to enable multi-writer support and expose the remotesapi for peer-to-peer sync.
package dolt
import (

View file

@ -57,6 +57,7 @@ func TestServerModeConnection(t *testing.T) {
store, err := New(ctx, &Config{
Path: tmpDir,
Database: "beads",
ServerMode: true,
ServerHost: "127.0.0.1",
ServerPort: 13307,
})

View file

@ -1,3 +1,5 @@
//go:build cgo
// Package dolt implements the storage interface using Dolt (versioned MySQL-compatible database).
//
// Dolt provides native version control for SQL data with cell-level merge, history queries,
@ -5,11 +7,15 @@
// by making the database itself version-controlled.
//
// Key differences from SQLite backend:
// - Connects to dolt sql-server via MySQL protocol
// - Uses github.com/dolthub/driver for embedded Dolt access
// - Supports version control operations (commit, push, pull, branch, merge)
// - History queries via AS OF and dolt_history_* tables
// - Cell-level merge instead of line-level JSONL merge
// - Multi-writer support via server mode
//
// Connection modes:
// - Embedded: No server required, database/sql interface via dolthub/driver
// - Server: Connect to running dolt sql-server for multi-writer scenarios
//
package dolt
import (
@ -26,6 +32,7 @@ import (
"time"
"github.com/cenkalti/backoff/v4"
embedded "github.com/dolthub/driver"
// Import MySQL driver for server mode connections
_ "github.com/go-sql-driver/mysql"
@ -35,12 +42,18 @@ import (
// DoltStore implements the Storage interface using Dolt
type DoltStore struct {
db *sql.DB
dbPath string // Path to Dolt database directory
closed atomic.Bool // Tracks whether Close() has been called
connStr string // Connection string for reconnection
mu sync.RWMutex // Protects concurrent access
readOnly bool // True if opened in read-only mode
db *sql.DB
dbPath string // Path to Dolt database directory
closed atomic.Bool // Tracks whether Close() has been called
connStr string // Connection string for reconnection
mu sync.RWMutex // Protects concurrent access
readOnly bool // True if opened in read-only mode
serverMode bool // True if connected to dolt sql-server (vs embedded)
accessLock *AccessLock // Advisory flock preventing concurrent dolt LOCK contention
// embeddedConnector is non-nil only in embedded mode. It must be closed to release
// filesystem locks held by the embedded engine.
embeddedConnector *embedded.Connector
// Version control config
committerName string
@ -51,23 +64,35 @@ type DoltStore struct {
// Config holds Dolt database configuration
type Config struct {
Path string // Path to Dolt database directory
CommitterName string // Git-style committer name
CommitterEmail string // Git-style committer email
Remote string // Default remote name (e.g., "origin")
Database string // Database name within Dolt (default: "beads")
ReadOnly bool // Open in read-only mode (skip schema init)
Path string // Path to Dolt database directory
CommitterName string // Git-style committer name
CommitterEmail string // Git-style committer email
Remote string // Default remote name (e.g., "origin")
Database string // Database name within Dolt (default: "beads")
ReadOnly bool // Open in read-only mode (skip schema init)
OpenTimeout time.Duration // Advisory lock timeout (0 = no advisory lock)
// Server connection options
// Server mode options (federation)
ServerMode bool // Connect to dolt sql-server instead of embedded
ServerHost string // Server host (default: 127.0.0.1)
ServerPort int // Server port (default: 3307)
ServerUser string // MySQL user (default: root)
ServerPassword string // MySQL password (default: empty, can be set via BEADS_DOLT_PASSWORD)
}
// Server retry configuration.
// go-sql-driver/mysql doesn't have built-in retry. We add retry for transient
// connection errors (stale pool connections, brief network issues, server restarts).
const embeddedOpenMaxElapsed = 30 * time.Second
func newEmbeddedOpenBackoff() backoff.BackOff {
// BackOff implementations are stateful; always return a fresh instance.
bo := backoff.NewExponentialBackOff()
bo.MaxElapsedTime = embeddedOpenMaxElapsed
return bo
}
// Server mode retry configuration.
// Server mode uses go-sql-driver/mysql which doesn't have built-in retry like the
// embedded driver. We add retry for transient connection errors (stale pool connections,
// brief network issues, server restarts).
const serverRetryMaxElapsed = 30 * time.Second
func newServerRetryBackoff() backoff.BackOff {
@ -107,7 +132,12 @@ func isRetryableError(err error) bool {
}
// withRetry executes an operation with retry for transient errors.
// Only active in server mode; embedded mode has driver-level retry.
func (s *DoltStore) withRetry(ctx context.Context, op func() error) error {
if !s.serverMode {
return op()
}
bo := newServerRetryBackoff()
return backoff.Retry(func() error {
err := op()
@ -147,19 +177,21 @@ func New(ctx context.Context, cfg *Config) (*DoltStore, error) {
cfg.Remote = "origin"
}
// Server connection defaults
if cfg.ServerHost == "" {
cfg.ServerHost = "127.0.0.1"
}
if cfg.ServerPort == 0 {
cfg.ServerPort = DefaultSQLPort
}
if cfg.ServerUser == "" {
cfg.ServerUser = "root"
}
// Check environment variable for password (more secure than command-line)
if cfg.ServerPassword == "" {
cfg.ServerPassword = os.Getenv("BEADS_DOLT_PASSWORD")
// Server mode defaults
if cfg.ServerMode {
if cfg.ServerHost == "" {
cfg.ServerHost = "127.0.0.1"
}
if cfg.ServerPort == 0 {
cfg.ServerPort = DefaultSQLPort
}
if cfg.ServerUser == "" {
cfg.ServerUser = "root"
}
// Check environment variable for password (more secure than command-line)
if cfg.ServerPassword == "" {
cfg.ServerPassword = os.Getenv("BEADS_DOLT_PASSWORD")
}
}
// Ensure directory exists
@ -167,51 +199,142 @@ func New(ctx context.Context, cfg *Config) (*DoltStore, error) {
return nil, fmt.Errorf("failed to create database directory: %w", err)
}
// IMPORTANT: Use an absolute path for embedded DSNs.
//
// The embedded driver sets its internal filesystem working directory to Config.Directory
// and also passes the directory path through to lower layers. If we pass a relative path,
// the working-directory stacking can effectively double it (e.g. ".beads/dolt/.beads/dolt").
absPath, err := filepath.Abs(cfg.Path)
if err != nil {
return nil, fmt.Errorf("failed to get absolute path: %w", err)
}
// Fail-fast TCP check before MySQL protocol initialization.
// This gives an immediate, clear error if the Dolt server isn't running,
// rather than waiting for MySQL driver timeouts.
addr := net.JoinHostPort(cfg.ServerHost, fmt.Sprintf("%d", cfg.ServerPort))
conn, err := net.DialTimeout("tcp", addr, 500*time.Millisecond)
if err != nil {
return nil, fmt.Errorf("Dolt server unreachable at %s: %w\n\nThe Dolt server may not be running. Try:\n gt dolt start # If using Gas Town\n dolt sql-server # Manual start in database directory",
addr, err)
// Acquire advisory flock before opening dolt (embedded mode only).
// This prevents multiple bd processes from competing for dolt's internal LOCK file.
// Set BD_SKIP_ACCESS_LOCK=1 to bypass flock for testing whether Dolt's internal
// locking is sufficient. See bd-39gso for testing plan.
var accessLock *AccessLock
if !cfg.ServerMode && cfg.OpenTimeout > 0 && os.Getenv("BD_SKIP_ACCESS_LOCK") == "" {
exclusive := !cfg.ReadOnly
var lockErr error
accessLock, lockErr = AcquireAccessLock(absPath, exclusive, cfg.OpenTimeout)
if lockErr != nil {
return nil, fmt.Errorf("failed to acquire dolt access lock: %w", lockErr)
}
}
var db *sql.DB
var connStr string
var embeddedConnector *embedded.Connector
if cfg.ServerMode {
// Fail-fast TCP check before MySQL protocol initialization.
// This gives an immediate, clear error if the Dolt server isn't running,
// rather than waiting for MySQL driver timeouts.
addr := net.JoinHostPort(cfg.ServerHost, fmt.Sprintf("%d", cfg.ServerPort))
conn, err := net.DialTimeout("tcp", addr, 500*time.Millisecond)
if err != nil {
if accessLock != nil {
accessLock.Release()
}
return nil, fmt.Errorf("Dolt server unreachable at %s: %w\n\nThe Dolt server may not be running. Try:\n gt dolt start # If using Gas Town\n dolt sql-server # Manual start in database directory",
addr, err)
}
_ = conn.Close()
// Server mode: connect via MySQL protocol to dolt sql-server
db, connStr, err = openServerConnection(ctx, cfg)
} else {
// Embedded mode:
// - Perform initialization as explicit units of work (each with its own connector).
// - Then open a fresh connector/DB for the returned store instance.
initDSN := fmt.Sprintf(
"file://%s?commitname=%s&commitemail=%s",
absPath, cfg.CommitterName, cfg.CommitterEmail,
)
dbDSN := fmt.Sprintf(
"file://%s?commitname=%s&commitemail=%s&database=%s",
absPath, cfg.CommitterName, cfg.CommitterEmail, cfg.Database,
)
configureRetries := func(c *embedded.Config) {
// Enable driver open retries for embedded usage.
c.BackOff = newEmbeddedOpenBackoff()
}
// UOW 1: ensure database exists. Skip in read-only mode.
if !cfg.ReadOnly {
if err := withEmbeddedDolt(ctx, initDSN, configureRetries, func(ctx context.Context, db *sql.DB) error {
_, err := db.ExecContext(ctx, fmt.Sprintf("CREATE DATABASE IF NOT EXISTS `%s`", cfg.Database))
return err
}); err != nil {
return nil, fmt.Errorf("failed to create dolt database: %w", err)
}
}
// UOW 2: initialize schema (idempotent). Skip in read-only mode.
if !cfg.ReadOnly {
if err := withEmbeddedDolt(ctx, dbDSN, configureRetries, func(ctx context.Context, db *sql.DB) error {
return initSchemaOnDB(ctx, db)
}); err != nil {
return nil, fmt.Errorf("failed to initialize schema: %w", err)
}
}
// Open the store connection (fresh connector for subsequent work).
db, connStr, embeddedConnector, err = openEmbeddedConnection(dbDSN)
}
_ = conn.Close()
// Connect to dolt sql-server via MySQL protocol
db, connStr, err := openServerConnection(ctx, cfg)
if err != nil {
if accessLock != nil {
accessLock.Release()
}
return nil, err
}
// Test connection
// IMPORTANT: In embedded mode, do not use a caller-supplied ctx to open the first
// underlying connection. Many tests (and some call sites) pass contexts that are
// canceled shortly after New() returns; the embedded driver derives a session context
// from Connect(ctx) and reuses it across statements. We force the initial connection
// to be created with a non-canceling context to avoid poisoning the connection pool.
pingCtx := ctx
if embeddedConnector != nil {
pingCtx = context.Background()
}
if pingCtx == nil {
pingCtx = context.Background()
}
if err := db.PingContext(pingCtx); err != nil {
// Ensure we don't leak filesystem locks if embedded open fails after creating a connector.
_ = db.Close()
if embeddedConnector != nil {
_ = embeddedConnector.Close()
}
if accessLock != nil {
accessLock.Release()
}
return nil, fmt.Errorf("failed to ping Dolt database: %w", err)
}
store := &DoltStore{
db: db,
dbPath: absPath,
connStr: connStr,
committerName: cfg.CommitterName,
committerEmail: cfg.CommitterEmail,
remote: cfg.Remote,
branch: "main",
readOnly: cfg.ReadOnly,
db: db,
dbPath: absPath,
connStr: connStr,
embeddedConnector: embeddedConnector,
committerName: cfg.CommitterName,
committerEmail: cfg.CommitterEmail,
remote: cfg.Remote,
branch: "main",
readOnly: cfg.ReadOnly,
serverMode: cfg.ServerMode,
accessLock: accessLock,
}
// Initialize schema (idempotent)
if !cfg.ReadOnly {
// Schema initialization:
// - Embedded mode: already performed above as an explicit unit of work.
// - Server mode: still needs to initialize schema here (idempotent).
if cfg.ServerMode && !cfg.ReadOnly {
if err := store.initSchema(ctx); err != nil {
return nil, fmt.Errorf("failed to initialize schema: %w", err)
}
@ -220,6 +343,31 @@ func New(ctx context.Context, cfg *Config) (*DoltStore, error) {
return store, nil
}
// openEmbeddedConnection opens a connection using the embedded Dolt driver
func openEmbeddedConnection(dsn string) (*sql.DB, string, *embedded.Connector, error) {
openCfg, err := embedded.ParseDSN(dsn)
if err != nil {
return nil, "", nil, fmt.Errorf("failed to parse Dolt DSN: %w", err)
}
openCfg.BackOff = newEmbeddedOpenBackoff()
connector, err := embedded.NewConnector(openCfg)
if err != nil {
return nil, "", nil, fmt.Errorf("failed to create Dolt connector: %w", err)
}
db := sql.OpenDB(connector)
// Configure connection pool
// Dolt embedded mode is single-writer like SQLite
db.SetMaxOpenConns(1)
db.SetMaxIdleConns(1)
db.SetConnMaxLifetime(0)
// NOTE: connector must be closed by the caller to release filesystem locks.
// DoltStore.Close() will handle this.
return db, dsn, connector, nil
}
// openServerConnection opens a connection to a dolt sql-server via MySQL protocol
func openServerConnection(ctx context.Context, cfg *Config) (*sql.DB, string, error) {
// DSN format: user:password@tcp(host:port)/database?parseTime=true
@ -445,7 +593,21 @@ func (s *DoltStore) Close() error {
}
}
}
// For embedded mode, ensure the underlying engine is closed to release filesystem locks.
if s.embeddedConnector != nil {
cerr := doltutil.CloseWithTimeout("embeddedConnector", s.embeddedConnector.Close)
// Ignore context cancellation noise from Dolt shutdown plumbing.
if cerr != nil && !errors.Is(cerr, context.Canceled) {
err = errors.Join(err, cerr)
}
s.embeddedConnector = nil
}
s.db = nil
// Release advisory lock after db and connector are closed
if s.accessLock != nil {
s.accessLock.Release()
s.accessLock = nil
}
return err
}

View file

@ -1,3 +1,4 @@
//go:build cgo
package dolt
import (

View file

@ -1,3 +1,4 @@
//go:build cgo
package dolt
import (

View file

@ -1,3 +1,4 @@
//go:build cgo
package dolt
import (
@ -224,7 +225,8 @@ func (s *DoltStore) GetChangesSinceExport(ctx context.Context, fromCommit string
}
// If fromCommit equals HEAD, there are no changes.
// Note: This also avoids issues when querying dolt_diff with identical from/to refs.
// Note: This also avoids a nil pointer panic in the embedded Dolt driver
// when querying dolt_diff with identical from/to refs.
if fromCommit == currentCommit {
return &ExportChanges{Entries: nil}, nil
}

View file

@ -1,4 +1,4 @@
// Package doltutil provides shared utilities for Dolt operations.
// Package doltutil provides shared utilities for embedded Dolt operations.
// This package exists to avoid import cycles between dolt and dolt/migrations.
package doltutil
@ -8,9 +8,11 @@ import (
)
// CloseTimeout is the maximum time to wait for close operations.
// Embedded Dolt can hang indefinitely on close; this prevents commands from hanging.
const CloseTimeout = 5 * time.Second
// CloseWithTimeout runs a close function with a timeout to prevent indefinite hangs.
// This is needed because embedded Dolt's engine can hang during shutdown.
// Returns an error if the close times out or if the close function returns an error.
func CloseWithTimeout(name string, closeFn func() error) error {
done := make(chan error, 1)

View file

@ -27,11 +27,13 @@ type Options struct {
ReadOnly bool
LockTimeout time.Duration
// Dolt server connection options
// Dolt server mode options (federation)
ServerMode bool // Connect to dolt sql-server instead of embedded
ServerHost string // Server host (default: 127.0.0.1)
ServerPort int // Server port (default: 3307)
ServerUser string // MySQL user (default: root)
Database string // Database name for Dolt (default: beads)
ServerUser string // MySQL user (default: root)
Database string // Database name for Dolt server mode (default: beads)
OpenTimeout time.Duration // Advisory lock timeout for embedded dolt (0 = no lock)
}
// New creates a storage backend based on the backend type.
@ -60,9 +62,9 @@ func NewWithOptions(ctx context.Context, backend, path string, opts Options) (st
if factory, ok := backendRegistry[backend]; ok {
return factory(ctx, path, opts)
}
// Provide helpful error for unknown backends
// Provide helpful error for dolt on systems without CGO
if backend == configfile.BackendDolt {
return nil, fmt.Errorf("dolt backend is not registered; ensure the dolt storage package is imported")
return nil, fmt.Errorf("dolt backend requires CGO (not available on this build); use sqlite backend or install from pre-built binaries")
}
return nil, fmt.Errorf("unknown storage backend: %s (supported: sqlite, dolt)", backend)
}
@ -89,18 +91,21 @@ func NewFromConfigWithOptions(ctx context.Context, beadsDir string, opts Options
case configfile.BackendSQLite:
return NewWithOptions(ctx, backend, cfg.DatabasePath(beadsDir), opts)
case configfile.BackendDolt:
// Merge Dolt server config into options (config provides defaults, opts can override)
if opts.ServerHost == "" {
opts.ServerHost = cfg.GetDoltServerHost()
}
if opts.ServerPort == 0 {
opts.ServerPort = cfg.GetDoltServerPort()
}
if opts.ServerUser == "" {
opts.ServerUser = cfg.GetDoltServerUser()
}
if opts.Database == "" {
opts.Database = cfg.GetDoltDatabase()
// Merge Dolt server mode config into options (config provides defaults, opts can override)
if cfg.IsDoltServerMode() {
opts.ServerMode = true
if opts.ServerHost == "" {
opts.ServerHost = cfg.GetDoltServerHost()
}
if opts.ServerPort == 0 {
opts.ServerPort = cfg.GetDoltServerPort()
}
if opts.ServerUser == "" {
opts.ServerUser = cfg.GetDoltServerUser()
}
if opts.Database == "" {
opts.Database = cfg.GetDoltDatabase()
}
}
return NewWithOptions(ctx, backend, cfg.DatabasePath(beadsDir), opts)
default:

View file

@ -1,8 +1,14 @@
//go:build cgo
package factory
import (
"context"
"fmt"
"os"
"path/filepath"
"strings"
"github.com/steveyegge/beads/internal/config"
"github.com/steveyegge/beads/internal/configfile"
"github.com/steveyegge/beads/internal/storage"
"github.com/steveyegge/beads/internal/storage/dolt"
@ -10,17 +16,118 @@ import (
func init() {
RegisterBackend(configfile.BackendDolt, func(ctx context.Context, path string, opts Options) (storage.Storage, error) {
// Only bootstrap in embedded mode - server mode has database on server
if !opts.ServerMode {
if err := bootstrapEmbeddedDolt(ctx, path, opts); err != nil {
return nil, err
}
}
store, err := dolt.New(ctx, &dolt.Config{
Path: path,
Database: opts.Database,
ReadOnly: opts.ReadOnly,
ServerHost: opts.ServerHost,
ServerPort: opts.ServerPort,
ServerUser: opts.ServerUser,
Path: path,
Database: opts.Database,
ReadOnly: opts.ReadOnly,
OpenTimeout: opts.OpenTimeout,
ServerMode: opts.ServerMode,
ServerHost: opts.ServerHost,
ServerPort: opts.ServerPort,
ServerUser: opts.ServerUser,
})
if err != nil {
// If server mode failed with a connection error, fall back to embedded mode.
// This preserves the 2-tier architecture: server when available, embedded when not.
if opts.ServerMode && isServerConnectionError(err) {
fmt.Fprintf(os.Stderr, "Warning: Dolt server at %s:%d is not reachable, falling back to embedded mode\n",
opts.ServerHost, opts.ServerPort)
// Run bootstrap for embedded mode (JSONL import if needed)
if berr := bootstrapEmbeddedDolt(ctx, path, opts); berr != nil {
return nil, berr
}
return dolt.New(ctx, &dolt.Config{
Path: path,
Database: opts.Database,
ReadOnly: opts.ReadOnly,
OpenTimeout: opts.OpenTimeout,
ServerMode: false, // Fall back to embedded
})
}
return nil, err
}
return store, nil
})
}
// bootstrapEmbeddedDolt checks if a JSONL-to-Dolt bootstrap is needed and runs it if so.
func bootstrapEmbeddedDolt(ctx context.Context, path string, opts Options) error {
// Path is the dolt subdirectory, parent is .beads directory
beadsDir := filepath.Dir(path)
// In dolt-native mode, JSONL is export-only backup — never auto-import.
// If the dolt DB doesn't exist in this mode, that's an error, not a bootstrap opportunity.
// This prevents split-brain: without this guard, a wrong path (from B1) would silently
// create a new DB from stale JSONL, diverging from the real dolt-native data.
if config.GetSyncMode() == config.SyncModeDoltNative {
if !hasDoltSubdir(path) {
return fmt.Errorf("dolt database not found at %s (JSONL auto-import is disabled in dolt-native sync mode; run 'bd init --backend=dolt' to create a new database)", path)
}
return nil // Dolt exists, no bootstrap needed
}
bootstrapped, result, err := dolt.Bootstrap(ctx, dolt.BootstrapConfig{
BeadsDir: beadsDir,
DoltPath: path,
LockTimeout: opts.LockTimeout,
})
if err != nil {
return fmt.Errorf("bootstrap failed: %w", err)
}
if bootstrapped && result != nil {
fmt.Fprintf(os.Stderr, "Bootstrapping Dolt from JSONL...\n")
if len(result.ParseErrors) > 0 {
fmt.Fprintf(os.Stderr, " Skipped %d malformed lines (see above for details)\n", len(result.ParseErrors))
}
fmt.Fprintf(os.Stderr, " Imported %d issues", result.IssuesImported)
if result.IssuesSkipped > 0 {
fmt.Fprintf(os.Stderr, ", skipped %d duplicates", result.IssuesSkipped)
}
fmt.Fprintf(os.Stderr, "\n Dolt database ready\n")
}
return nil
}
// hasDoltSubdir checks if the given path contains any subdirectory with a .dolt directory inside.
func hasDoltSubdir(basePath string) bool {
entries, err := os.ReadDir(basePath)
if err != nil {
return false
}
for _, entry := range entries {
if !entry.IsDir() {
continue
}
doltDir := filepath.Join(basePath, entry.Name(), ".dolt")
if info, err := os.Stat(doltDir); err == nil && info.IsDir() {
return true
}
}
return false
}
// isServerConnectionError returns true if the error indicates the Dolt server
// is unreachable (connection refused, timeout, DNS failure, etc.).
func isServerConnectionError(err error) bool {
if err == nil {
return false
}
errLower := strings.ToLower(err.Error())
return strings.Contains(errLower, "connection refused") ||
strings.Contains(errLower, "no such host") ||
strings.Contains(errLower, "i/o timeout") ||
strings.Contains(errLower, "connection reset") ||
strings.Contains(errLower, "network is unreachable")
}

View file

@ -0,0 +1,77 @@
//go:build cgo
package factory
import (
"errors"
"testing"
)
func TestIsServerConnectionError(t *testing.T) {
tests := []struct {
name string
err error
want bool
}{
{
name: "nil error",
err: nil,
want: false,
},
{
name: "connection refused",
err: errors.New("dial tcp 127.0.0.1:3307: connect: connection refused"),
want: true,
},
{
name: "wrapped connection refused",
err: errors.New("failed to connect to Dolt server at 127.0.0.1:3307: dial tcp 127.0.0.1:3307: connect: connection refused"),
want: true,
},
{
name: "no such host",
err: errors.New("dial tcp: lookup badhost.invalid: no such host"),
want: true,
},
{
name: "i/o timeout",
err: errors.New("dial tcp 10.0.0.1:3307: i/o timeout"),
want: true,
},
{
name: "connection reset",
err: errors.New("read tcp 127.0.0.1:54321->127.0.0.1:3307: connection reset by peer"),
want: true,
},
{
name: "network unreachable",
err: errors.New("dial tcp 192.168.1.1:3307: connect: network is unreachable"),
want: true,
},
{
name: "auth error is not connection error",
err: errors.New("Error 1045: Access denied for user 'root'@'localhost'"),
want: false,
},
{
name: "schema error is not connection error",
err: errors.New("failed to create schema: table already exists"),
want: false,
},
{
name: "database error is not connection error",
err: errors.New("failed to create database: unknown error"),
want: false,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got := isServerConnectionError(tt.err)
if got != tt.want {
t.Errorf("isServerConnectionError(%v) = %v, want %v", tt.err, got, tt.want)
}
})
}
}

View file

@ -0,0 +1,46 @@
name: Test
on: [pull_request]
concurrency:
group: test-${{ github.event.pull_request.number || github.ref }}
cancel-in-progress: true
jobs:
test:
strategy:
matrix:
go-version: [1.24.x]
platform: [ubuntu-latest, macos-latest, windows-latest]
runs-on: ${{ matrix.platform }}
steps:
- name: Install Go
uses: actions/setup-go@v3
with:
go-version: ${{ matrix.go-version }}
- name: Checkout code
uses: actions/checkout@v3
- name: Install ICU4C (MacOS)
if: ${{ matrix.platform == 'macos-latest' }}
run: |
dir=$(brew --cellar icu4c)
dir="$dir"/$(ls "$dir")
echo CGO_CPPFLAGS=-I$dir/include >> $GITHUB_ENV
echo CGO_LDFLAGS=-L$dir/lib >> $GITHUB_ENV
- name: Install ICU4C (Windows)
if: ${{ matrix.platform == 'windows-latest' }}
uses: msys2/setup-msys2@v2
with:
path-type: inherit
msystem: UCRT64
pacboy: icu:p toolchain:p pkg-config:p
- name: Test
if: ${{ matrix.platform == 'macos-latest' }}
run: go test ./...
- name: Test
if: ${{ matrix.platform == 'windows-latest' }}
shell: msys2 {0}
run: |
go.exe test ./...
- name: Test
if: ${{ matrix.platform == 'ubuntu-latest' }}
run: go test -race ./...

View file

@ -0,0 +1,44 @@
# Compiled Object files, Static and Dynamic libs (Shared Objects)
*.o
*.a
*.so
# Compiled binaries
_example/_example
# Folders
_obj
_test
# Architecture specific extensions/prefixes
*.[568vq]
[568vq].out
*.cgo1.go
*.cgo2.c
_cgo_defun.c
_cgo_gotypes.go
_cgo_export.*
_testmain.go
*.exe
*.test
*.prof
# Tpc-h tbl files
benchmark/*tbl
vendor
Makefile.main
.ci/
_example/main
_example/*.exe
test-server
.idea/*
/.vscode/
# OSX Files
.DS_Store
*~

View file

@ -0,0 +1,202 @@
Apache License
Version 2.0, January 2004
http://www.apache.org/licenses/
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
1. Definitions.
"License" shall mean the terms and conditions for use, reproduction,
and distribution as defined by Sections 1 through 9 of this document.
"Licensor" shall mean the copyright owner or entity authorized by
the copyright owner that is granting the License.
"Legal Entity" shall mean the union of the acting entity and all
other entities that control, are controlled by, or are under common
control with that entity. For the purposes of this definition,
"control" means (i) the power, direct or indirect, to cause the
direction or management of such entity, whether by contract or
otherwise, or (ii) ownership of fifty percent (50%) or more of the
outstanding shares, or (iii) beneficial ownership of such entity.
"You" (or "Your") shall mean an individual or Legal Entity
exercising permissions granted by this License.
"Source" form shall mean the preferred form for making modifications,
including but not limited to software source code, documentation
source, and configuration files.
"Object" form shall mean any form resulting from mechanical
transformation or translation of a Source form, including but
not limited to compiled object code, generated documentation,
and conversions to other media types.
"Work" shall mean the work of authorship, whether in Source or
Object form, made available under the License, as indicated by a
copyright notice that is included in or attached to the work
(an example is provided in the Appendix below).
"Derivative Works" shall mean any work, whether in Source or Object
form, that is based on (or derived from) the Work and for which the
editorial revisions, annotations, elaborations, or other modifications
represent, as a whole, an original work of authorship. For the purposes
of this License, Derivative Works shall not include works that remain
separable from, or merely link (or bind by name) to the interfaces of,
the Work and Derivative Works thereof.
"Contribution" shall mean any work of authorship, including
the original version of the Work and any modifications or additions
to that Work or Derivative Works thereof, that is intentionally
submitted to Licensor for inclusion in the Work by the copyright owner
or by an individual or Legal Entity authorized to submit on behalf of
the copyright owner. For the purposes of this definition, "submitted"
means any form of electronic, verbal, or written communication sent
to the Licensor or its representatives, including but not limited to
communication on electronic mailing lists, source code control systems,
and issue tracking systems that are managed by, or on behalf of, the
Licensor for the purpose of discussing and improving the Work, but
excluding communication that is conspicuously marked or otherwise
designated in writing by the copyright owner as "Not a Contribution."
"Contributor" shall mean Licensor and any individual or Legal Entity
on behalf of whom a Contribution has been received by Licensor and
subsequently incorporated within the Work.
2. Grant of Copyright License. Subject to the terms and conditions of
this License, each Contributor hereby grants to You a perpetual,
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
copyright license to reproduce, prepare Derivative Works of,
publicly display, publicly perform, sublicense, and distribute the
Work and such Derivative Works in Source or Object form.
3. Grant of Patent License. Subject to the terms and conditions of
this License, each Contributor hereby grants to You a perpetual,
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
(except as stated in this section) patent license to make, have made,
use, offer to sell, sell, import, and otherwise transfer the Work,
where such license applies only to those patent claims licensable
by such Contributor that are necessarily infringed by their
Contribution(s) alone or by combination of their Contribution(s)
with the Work to which such Contribution(s) was submitted. If You
institute patent litigation against any entity (including a
cross-claim or counterclaim in a lawsuit) alleging that the Work
or a Contribution incorporated within the Work constitutes direct
or contributory patent infringement, then any patent licenses
granted to You under this License for that Work shall terminate
as of the date such litigation is filed.
4. Redistribution. You may reproduce and distribute copies of the
Work or Derivative Works thereof in any medium, with or without
modifications, and in Source or Object form, provided that You
meet the following conditions:
(a) You must give any other recipients of the Work or
Derivative Works a copy of this License; and
(b) You must cause any modified files to carry prominent notices
stating that You changed the files; and
(c) You must retain, in the Source form of any Derivative Works
that You distribute, all copyright, patent, trademark, and
attribution notices from the Source form of the Work,
excluding those notices that do not pertain to any part of
the Derivative Works; and
(d) If the Work includes a "NOTICE" text file as part of its
distribution, then any Derivative Works that You distribute must
include a readable copy of the attribution notices contained
within such NOTICE file, excluding those notices that do not
pertain to any part of the Derivative Works, in at least one
of the following places: within a NOTICE text file distributed
as part of the Derivative Works; within the Source form or
documentation, if provided along with the Derivative Works; or,
within a display generated by the Derivative Works, if and
wherever such third-party notices normally appear. The contents
of the NOTICE file are for informational purposes only and
do not modify the License. You may add Your own attribution
notices within Derivative Works that You distribute, alongside
or as an addendum to the NOTICE text from the Work, provided
that such additional attribution notices cannot be construed
as modifying the License.
You may add Your own copyright statement to Your modifications and
may provide additional or different license terms and conditions
for use, reproduction, or distribution of Your modifications, or
for any such Derivative Works as a whole, provided Your use,
reproduction, and distribution of the Work otherwise complies with
the conditions stated in this License.
5. Submission of Contributions. Unless You explicitly state otherwise,
any Contribution intentionally submitted for inclusion in the Work
by You to the Licensor shall be under the terms and conditions of
this License, without any additional terms or conditions.
Notwithstanding the above, nothing herein shall supersede or modify
the terms of any separate license agreement you may have executed
with Licensor regarding such Contributions.
6. Trademarks. This License does not grant permission to use the trade
names, trademarks, service marks, or product names of the Licensor,
except as required for reasonable and customary use in describing the
origin of the Work and reproducing the content of the NOTICE file.
7. Disclaimer of Warranty. Unless required by applicable law or
agreed to in writing, Licensor provides the Work (and each
Contributor provides its Contributions) on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
implied, including, without limitation, any warranties or conditions
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
PARTICULAR PURPOSE. You are solely responsible for determining the
appropriateness of using or redistributing the Work and assume any
risks associated with Your exercise of permissions under this License.
8. Limitation of Liability. In no event and under no legal theory,
whether in tort (including negligence), contract, or otherwise,
unless required by applicable law (such as deliberate and grossly
negligent acts) or agreed to in writing, shall any Contributor be
liable to You for damages, including any direct, indirect, special,
incidental, or consequential damages of any character arising as a
result of this License or out of the use or inability to use the
Work (including but not limited to damages for loss of goodwill,
work stoppage, computer failure or malfunction, or any and all
other commercial damages or losses), even if such Contributor
has been advised of the possibility of such damages.
9. Accepting Warranty or Additional Liability. While redistributing
the Work or Derivative Works thereof, You may choose to offer,
and charge a fee for, acceptance of support, warranty, indemnity,
or other liability obligations and/or rights consistent with this
License. However, in accepting such obligations, You may act only
on Your own behalf and on Your sole responsibility, not on behalf
of any other Contributor, and only if You agree to indemnify,
defend, and hold each Contributor harmless for any liability
incurred by, or claims asserted against, such Contributor by reason
of your accepting any such warranty or additional liability.
END OF TERMS AND CONDITIONS
APPENDIX: How to apply the Apache License to your work.
To apply the Apache License to your work, attach the following
boilerplate notice, with the fields enclosed by brackets "[]"
replaced with your own identifying information. (Don't include
the brackets!) The text should be enclosed in the appropriate
comment syntax for the file format. We also recommend that a
file or class name and description of purpose be included on the
same "printed page" as the copyright notice for easier
identification within third-party archives.
Copyright [yyyy] [name of copyright owner]
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.

View file

@ -0,0 +1,38 @@
# ICU Regular Expressions in Go
Minimal bindings to [ICU4C](https://github.com/unicode-org/icu)'s regex implementation, for use in Go.
This package is not intended to be a general purpose binding. It's primary purpose is to support [dolt](https://github.com/dolthub/dolt)'s need for ICU-compatible regexes in order to implement MySQL-compatible functionality.
# Use
```go
// Create a regex
regex := regex.CreateRegex(1024)
defer regex.Close()
// Set its pattern
regex.SetRegexString(context.TODO(), "[abc]+", regex.RegexFlags_None)
// Set its match string
regex.SetMatchString(context.TODO(), "123abcabcabcdef")
// Extract a matching substring; note start and occurence number are 1 indexed.
substr, ok, err := regex.Substring(context.TODO(), 1, 1)
assert.NoError(t, err)
assert.True(t, ok)
assert.Equals(t, substr, "abcabcabc")
```
# Building and Dependencies
This library, and consequently anything that depends on it, requires ICU4C to build and link against. This library does not ship a pre-compiled version of ICU4C and does not build ICU4C alongside itself as part of its Cgo binding. Consequently, building this library or anything that depends on it requires a C++ toolchain and a version of ICU4C installed.
For Windows, this library currently only supports MinGW. We are happy to accept changes to support other toolchains based on Go build tags, for example.
For Linux, a package like `libicu-dev` typically has the necessary library.
For Windows, with msys2, `pacman -S icu-devel` installs the necessary development libraries.
For macOS, `brew install icu4c` will install the necsesary library, but it is not on the default search path of the toolchain. Building with something like `CGO_CPPFLAGS=-I$(brew --cellar icu4c)/$(ls $(brew --cellar icu4c))/include CGO_LDFLAGS=-L$(brew --cellar icu4c)/$(ls $(brew --cellar icu4c))/lib` is potentially necessary.
There is some support for statically linking ICU4C, by building with the build tag `icu_static`. Currently this only changes the linker line for a Windows build. For a macOS or Linux build to link it statically, the build tag `icu_static` should still be used, but it should also be the case that the dynamic libraries are not installed.
When using a self-built dynamic library on macOS, the resulting binaries work best if `runConfigureICU` is run with `--enable-rpath`, so that the ICU4C dynamic libraries are discoverable by the built binary at their installed location.

View file

@ -0,0 +1,35 @@
#!/bin/bash
# Copyright 2023 Dolthub, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
set -eo pipefail
paths=`find . -maxdepth 1 -mindepth 1 \( -name gen -prune -o -type d -print -o -type f -name '*.go' -print \)`
goimports -w -local github.com/dolthub/go-icu-regex $paths
bad_files=$(find $paths -name '*.go' | while read f; do
if [[ $(awk '/import \(/{flag=1;next}/\)/{flag=0}flag' < $f | egrep -c '$^') -gt 2 ]]; then
echo $f
fi
done)
if [ "$bad_files" != "" ]; then
for f in $bad_files; do
awk '/import \(/{flag=1}/\)/{flag=0}flag&&!/^$/||!flag' < "$f" > "$f.bak"
mv "$f.bak" "$f"
done
goimports -w -local github.com/dolthub/go-icu-regex .
fi

View file

@ -0,0 +1,15 @@
module github.com/dolthub/go-icu-regex
go 1.24
require (
github.com/stretchr/testify v1.8.2
gopkg.in/src-d/go-errors.v1 v1.0.0
)
require (
github.com/davecgh/go-spew v1.1.1 // indirect
github.com/pkg/errors v0.9.1 // indirect
github.com/pmezard/go-difflib v1.0.0 // indirect
gopkg.in/yaml.v3 v3.0.1 // indirect
)

View file

@ -0,0 +1,21 @@
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
github.com/stretchr/testify v1.8.2 h1:+h33VjcLVPDHtOdpUCuF+7gSuG3yGIftsP1YvFihtJ8=
github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/src-d/go-errors.v1 v1.0.0 h1:cooGdZnCjYbeS1zb1s6pVAAimTdKceRrpn7aKOnNIfc=
gopkg.in/src-d/go-errors.v1 v1.0.0/go.mod h1:q1cBlomlw2FnDBDNGlnh6X0jPihy+QxZfMMNxPCbdYg=
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

View file

@ -0,0 +1,88 @@
// This code has been adapted from https://github.com/mysql/mysql-server/blob/ea7087d885006918ad54458e7aad215b1650312c/sql/regexp/regexp_engine.cc
// #cgo macos CFLAGS: -Ibin/darwin-aarch64/include
// #cgo macos CXXFLAGS: -Ibin/darwin-aarch64/include
#include "unicode/regex.h"
#include "cstring"
#ifdef __cplusplus
extern "C" {
#endif
void appendHead(URegularExpression* m_re, std::u16string& m_replace_buffer, int& m_replace_buffer_pos, UErrorCode* m_error_code, size_t size) {
if (size == 0) { return; }
int32_t text_length32 = 0;
auto text = uregex_getText(m_re, &text_length32, m_error_code);
if (*m_error_code != U_ZERO_ERROR) { return; }
if (m_replace_buffer.size() < size) { m_replace_buffer.resize(size); }
std::copy(text, text + size, &m_replace_buffer.at(0));
m_replace_buffer_pos = (int)size;
}
int tryToAppendReplacement(URegularExpression* m_re, std::u16string& m_replace_buffer, int& m_replace_buffer_pos, UErrorCode* m_error_code, UChar* replacement, int replacementLen) {
if (m_replace_buffer.empty()) { return 0; }
UChar* ptr = reinterpret_cast<UChar*>(&m_replace_buffer.at(0) + m_replace_buffer_pos);
int capacity = m_replace_buffer.size() - m_replace_buffer_pos;
return uregex_appendReplacement(m_re, replacement, replacementLen, &ptr, &capacity, m_error_code);
}
void appendReplacement(URegularExpression* m_re, std::u16string& m_replace_buffer, int& m_replace_buffer_pos, UErrorCode* m_error_code, UChar* replacement, int replacementLen) {
int replacement_size = tryToAppendReplacement(m_re, m_replace_buffer, m_replace_buffer_pos, m_error_code, replacement, replacementLen);
if (*m_error_code == U_BUFFER_OVERFLOW_ERROR) {
size_t required_buffer_size = m_replace_buffer_pos + replacement_size;
m_replace_buffer.resize(required_buffer_size);
*m_error_code = U_ZERO_ERROR;
tryToAppendReplacement(m_re, m_replace_buffer, m_replace_buffer_pos, m_error_code, replacement, replacementLen);
}
m_replace_buffer_pos += replacement_size;
}
int tryToAppendTail(URegularExpression* m_re, std::u16string& m_replace_buffer, int& m_replace_buffer_pos, UErrorCode* m_error_code) {
if (m_replace_buffer.empty()) { return 0; }
UChar* ptr = reinterpret_cast<UChar*>(&m_replace_buffer.at(0) + m_replace_buffer_pos);
int capacity = m_replace_buffer.size() - m_replace_buffer_pos;
return uregex_appendTail(m_re, &ptr, &capacity, m_error_code);
}
void appendTail(URegularExpression* m_re, std::u16string& m_replace_buffer, int& m_replace_buffer_pos, UErrorCode* m_error_code) {
int tail_size = tryToAppendTail(m_re, m_replace_buffer, m_replace_buffer_pos, m_error_code);
if (*m_error_code == U_BUFFER_OVERFLOW_ERROR) {
size_t required_buffer_size = m_replace_buffer_pos + tail_size;
m_replace_buffer.resize(required_buffer_size);
*m_error_code = U_ZERO_ERROR;
tryToAppendTail(m_re, m_replace_buffer, m_replace_buffer_pos, m_error_code);
}
m_replace_buffer_pos += tail_size;
}
UChar* replace(URegularExpression* regexp, UChar* replacement, int replacementLen, UChar* original, int originalSize, int start, int occurrence, int* returnSize) {
*returnSize = originalSize;
UErrorCode m_error_code = U_ZERO_ERROR;
std::u16string m_replace_buffer;
int m_replace_buffer_pos = 0;
bool found = uregex_find(regexp, start, &m_error_code);
int end_of_previous_match = 0;
for (int i = 1; i < occurrence && found; ++i) {
end_of_previous_match = uregex_end(regexp, 0, &m_error_code);
found = uregex_findNext(regexp, &m_error_code);
}
if (!found && U_SUCCESS(m_error_code)) { return original; }
m_replace_buffer.resize(originalSize);
appendHead(regexp, m_replace_buffer, m_replace_buffer_pos, &m_error_code, std::max(end_of_previous_match, start));
if (found) {
do {
appendReplacement(regexp, m_replace_buffer, m_replace_buffer_pos, &m_error_code, replacement, replacementLen);
} while (occurrence == 0 && uregex_findNext(regexp, &m_error_code));
}
appendTail(regexp, m_replace_buffer, m_replace_buffer_pos, &m_error_code);
m_replace_buffer.resize(m_replace_buffer_pos);
UChar* returnStr = static_cast<UChar*>(malloc(m_replace_buffer.size() * sizeof(UChar*)));
memcpy(returnStr, m_replace_buffer.data(), m_replace_buffer.size() * sizeof(UChar*));
*returnSize = m_replace_buffer.size();
return returnStr;
}
#ifdef __cplusplus
} // extern "C"
#endif

View file

@ -0,0 +1,12 @@
#include "unicode/regex.h"
#ifdef __cplusplus
extern "C" {
#endif
UChar* replace(URegularExpression* regexp, UChar* replacement, int replacementLen, UChar* original, int originalSize, int start, int occurrence, int* returnSize);
#ifdef __cplusplus
} // extern "C"
#endif

View file

@ -0,0 +1,182 @@
package icu
// #cgo CXXFLAGS: -std=c++17
// #cgo !windows LDFLAGS: -licui18n -licuuc -licudata
// #cgo icu_static CPPFLAGS: -DU_STATIC_IMPLEMENTATION
// #cgo windows,icu_static LDFLAGS: -lsicuin -lsicuuc -lsicudt
// #cgo windows,!icu_static LDFLAGS: -licuin -licuuc -licudt
// #include "unicode/uregex.h"
// #include <stdlib.h>
// #include "file.h"
import "C"
import (
"runtime"
"unicode/utf16"
"unsafe"
)
type URegularExpression struct {
ptr *C.URegularExpression
cleanup runtime.Cleanup
}
type UErrorCode C.UErrorCode
func Uregex_open(str *UCharStr, flags uint32, uerr *UErrorCode) *URegularExpression {
res := new(URegularExpression)
var ec C.UErrorCode
res.ptr = C.uregex_open(str.ptr, C.int32_t(str.len), C.uint32_t(flags), nil, &ec)
if uerr != nil {
*uerr = UErrorCode(ec)
}
res.cleanup = runtime.AddCleanup(res, func(ptr *C.URegularExpression) {
C.uregex_close(ptr)
}, res.ptr)
return res
}
func (re *URegularExpression) Free() {
re.cleanup.Stop()
C.uregex_close(re.ptr)
}
func Uregex_start(regex *URegularExpression, group int, uerr *UErrorCode) (idx int32) {
var ec C.UErrorCode
res := C.uregex_start(regex.ptr, C.int32_t(group), &ec)
if uerr != nil {
*uerr = UErrorCode(ec)
}
return int32(res)
}
func Uregex_end(regex *URegularExpression, group int, uerr *UErrorCode) (idx int32) {
var ec C.UErrorCode
res := C.uregex_end(regex.ptr, C.int32_t(group), &ec)
if uerr != nil {
*uerr = UErrorCode(ec)
}
return int32(res)
}
func Uregex_find(regex *URegularExpression, startIndex int, uerr *UErrorCode) (ok bool) {
var ec C.UErrorCode
res := C.uregex_find(regex.ptr, C.int32_t(startIndex), &ec)
if uerr != nil {
*uerr = UErrorCode(ec)
}
return res != 0
}
func Uregex_findNext(regex *URegularExpression, uerr *UErrorCode) (ok bool) {
var ec C.UErrorCode
res := C.uregex_findNext(regex.ptr, &ec)
if uerr != nil {
*uerr = UErrorCode(ec)
}
return res != 0
}
func Uregex_setText(regex *URegularExpression, str *UCharStr, uerr *UErrorCode) {
var ec C.UErrorCode
C.uregex_setText(regex.ptr, str.ptr, C.int32_t(str.len), &ec)
if uerr != nil {
*uerr = UErrorCode(ec)
}
}
func Replace(regex *URegularExpression, replacement string, original *UCharStr, start int, occurrence int) string {
var replacementStr UCharStr
replacementStr.SetString(replacement)
defer replacementStr.Free()
var res UCharStr
var resLen C.int
res.ptr = C.replace(regex.ptr, replacementStr.ptr, C.int(replacementStr.len), original.ptr, C.int(original.len), C.int(start), C.int(occurrence), &resLen)
res.len = int(resLen)
if res.ptr != original.ptr {
// replace will not allocate if there is no match, returning original instead.
// we return a non-owning reference here if that is the case, but need to take
// ownership of the storage if it is newly allocated for us.
res.cap = int(resLen)
}
defer res.Free()
return res.GetString()
}
// A small wrapper around an allocated blob of memory and a populated *UChar.
type UCharStr struct {
ptr *C.UChar
len int
// If cap != 0, this UCharStr owns the ptr and is responsible for deallocating it.
cap int
cleanup runtime.Cleanup
}
func (s *UCharStr) SetString(str string) {
uints := utf16.Encode([]rune(str))
sz := len(uints) + 1
s.alloc(sz)
var i int
for i = 0; i < len(uints); i++ {
*(*C.UChar)(unsafe.Pointer(uintptr(unsafe.Pointer(s.ptr)) + uintptr(i)*C.sizeof_UChar)) = C.UChar(uints[i])
}
*(*C.UChar)(unsafe.Pointer(uintptr(unsafe.Pointer(s.ptr)) + uintptr(i)*C.sizeof_UChar)) = C.UChar(0)
s.len = len(uints)
}
func (s *UCharStr) GetString() string {
codeunits := make([]uint16, s.len)
for i := 0; i < s.len; i++ {
codeunits[i] = uint16(*(*C.UChar)(unsafe.Pointer(uintptr(unsafe.Pointer(s.ptr)) + uintptr(i)*C.sizeof_UChar)))
}
return string(utf16.Decode(codeunits))
}
func (s *UCharStr) GetSubstring(start, end int) string {
return s.slice(start, end).GetString()
}
func (s *UCharStr) alloc(sz int) {
if sz < 64 {
sz = 64
}
if sz > s.cap {
s.Free()
s.cap = NextPow2(sz)
s.ptr = (*C.UChar)(C.malloc(C.size_t(s.cap * C.sizeof_UChar)))
s.cleanup = runtime.AddCleanup(s, func(ptr *C.UChar) {
C.free(unsafe.Pointer(ptr))
}, s.ptr)
}
}
func (s *UCharStr) Free() {
if s.cap > 0 {
s.cleanup.Stop()
C.free(unsafe.Pointer(s.ptr))
s.cap = 0
s.ptr = nil
s.len = 0
}
}
func (s *UCharStr) slice(start, end int) *UCharStr {
// slice never owns the storage and must not outlive the owning *UCharStr, including a SetString called on it.
var res UCharStr
res.len = end-start
res.ptr = (*C.UChar)(unsafe.Pointer(uintptr(unsafe.Pointer(s.ptr)) + 2 * uintptr(start)))
return &res
}
func NextPow2(i int) int {
ui := uint32(i)
ui--
ui |= ui >> 1
ui |= ui >> 2
ui |= ui >> 4
ui |= ui >> 8
ui |= ui >> 16
ui++
return int(ui)
}

View file

@ -0,0 +1,36 @@
package icu
import (
"testing"
"github.com/stretchr/testify/assert"
)
func TestNextPow2(t *testing.T) {
assert.Equal(t, 32, NextPow2(32))
assert.Equal(t, 64, NextPow2(33))
}
func TestUCharStr(t *testing.T) {
var s UCharStr
defer s.Free()
assert.Equal(t, 0, s.len)
assert.Equal(t, 0, s.cap)
assert.Equal(t, "", s.GetString())
s.SetString("hello, world!")
assert.Equal(t, 13, s.len)
assert.Equal(t, 64, s.cap)
assert.Equal(t, "hello, world!", s.GetString())
s.SetString("hello, world! this is a longer string and it requires a reallocation. we just keep adding words until it is bigger than 64 characters.")
assert.Equal(t, 134, s.len)
assert.Equal(t, 256, s.cap)
assert.Equal(t, "hello, world! this is a longer string and it requires a reallocation. we just keep adding words until it is bigger than 64 characters.", s.GetString())
s.SetString("hello, world!")
assert.Equal(t, 13, s.len)
assert.Equal(t, 256, s.cap)
assert.Equal(t, "hello, world!", s.GetString())
assert.Equal(t, "hello", s.GetSubstring(0,5))
assert.Equal(t, "ello", s.GetSubstring(1,5))
assert.Equal(t, "world", s.GetSubstring(7, 12))
}

View file

@ -0,0 +1,283 @@
//go:build !windows
// Copyright 2023 Dolthub, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package regex
import (
"context"
"fmt"
"gopkg.in/src-d/go-errors.v1"
"github.com/dolthub/go-icu-regex/internal/icu"
)
// Regex is an interface that wraps around the ICU library, exposing ICU's regular expression functionality. It is
// imperative that Regex is closed once it is finished.
type Regex interface {
// SetRegexString sets the string that will later be matched against. This must be called at least once before any other
// calls are made (except for Close).
SetRegexString(ctx context.Context, regexStr string, flags RegexFlags) error
// SetMatchString sets the string that we will either be matching against, or executing the replacements on. This
// must be called after SetRegexString, but before any other calls.
SetMatchString(ctx context.Context, matchStr string) error
// IndexOf returns the index of the previously-set regex matching the previously-set match string. Must call
// SetRegexString and SetMatchString before this function. `endIndex` determines whether the returned index is at
// the beginning or end of the match. `start` and `occurrence` start at 1, not 0. Returns 0 if the index was not found.
IndexOf(ctx context.Context, start int, occurrence int, endIndex bool) (int, error)
// Matches returns whether the previously-set regex matches the previously-set match string. Must call
// SetRegexString and SetMatchString before this function.
Matches(ctx context.Context, start int, occurrence int) (bool, error)
// Replace returns a new string with the replacement string occupying the matched portions of the match string,
// based on the regex. Position starts at 1, not 0. Must call SetRegexString and SetMatchString before this function.
Replace(ctx context.Context, replacementStr string, position int, occurrence int) (string, error)
// Substring returns the match of the previously-set match string, using the previously-set regex. Must call
// SetRegexString and SetMatchString before this function. `start` and `occurrence` start at 1, not 0.
Substring(ctx context.Context, start int, occurrence int) (string, bool, error)
// Close frees up the internal resources. This MUST be called, else a panic will occur at some non-deterministic time.
Close() error
}
var (
// ErrRegexNotYetSet is returned when attempting to use another function before the regex has been initialized.
ErrRegexNotYetSet = errors.NewKind("SetRegexString must be called before any other function")
// ErrMatchNotYetSet is returned when attempting to use another function before the match string has been set.
ErrMatchNotYetSet = errors.NewKind("SetMatchString must be called as there is nothing to match against")
// ErrInvalidRegex is returned when an invalid regex is given
ErrInvalidRegex = errors.NewKind("the given regular expression is invalid")
)
// RegexFlags are flags to define the behavior of the regular expression. Use OR (|) to combine flags. All flag values
// were taken directly from ICU.
type RegexFlags uint32
const (
// Enable case insensitive matching.
RegexFlags_None RegexFlags = 0
// Enable case insensitive matching.
RegexFlags_Case_Insensitive RegexFlags = 2
// Allow white space and comments within patterns.
RegexFlags_Comments RegexFlags = 4
// If set, '.' matches line terminators, otherwise '.' matching stops at line end.
RegexFlags_Dot_All RegexFlags = 32
// If set, treat the entire pattern as a literal string. Metacharacters or escape sequences in the input sequence
// will be given no special meaning.
//
// The flag RegexFlags_Case_Insensitive retains its impact on matching when used in conjunction with this flag. The
// other flags become superfluous.
RegexFlags_Literal RegexFlags = 16
// Control behavior of "$" and "^". If set, recognize line terminators within string, otherwise, match only at start
// and end of input string.
RegexFlags_Multiline RegexFlags = 8
// Unix-only line endings. When this mode is enabled, only '\n' is recognized as a line ending in the behavior
// of ., ^, and $.
RegexFlags_Unix_Lines RegexFlags = 1
// Unicode word boundaries. If set, \b uses the Unicode TR 29 definition of word boundaries. Warning: Unicode word
// boundaries are quite different from traditional regular expression word boundaries.
// See http://unicode.org/reports/tr29/#Word_Boundaries
RegexFlags_Unicode_Word RegexFlags = 256
// Error on Unrecognized backslash escapes. If set, fail with an error on patterns that contain backslash-escaped
// ASCII letters without a known special meaning. If this flag is not set, these escaped letters represent
// themselves.
RegexFlags_Error_On_Unknown_Escapes RegexFlags = 512
)
// CreateRegex creates a Regex. |stringBufferInBytes| is a hint to allocate string buffers
// for a certain size to avoid reallocation in the future, but is currently unused by the
// primary implementation.
func CreateRegex(stringBufferInBytes uint32) Regex {
return &privateRegex{}
}
// privateRegex is the private implementation of the Regex interface.
type privateRegex struct {
regexPtr *icu.URegularExpression
regexStr icu.UCharStr
matchStr icu.UCharStr
matchSet bool
}
var _ Regex = (*privateRegex)(nil)
// SetRegexString implements the interface Regex.
func (pr *privateRegex) SetRegexString(ctx context.Context, regexStr string, flags RegexFlags) (err error) {
if pr.regexPtr != nil {
pr.regexPtr.Free()
pr.regexPtr = nil
}
pr.regexStr.SetString(regexStr)
pr.matchSet = false
// Create the URegularExpression*
errorCode := icu.UErrorCode(0)
regex := icu.Uregex_open(&pr.regexStr, uint32(flags), &errorCode)
if errorCode > 0 {
return ErrInvalidRegex.New()
}
pr.regexPtr = regex
return nil
}
// SetMatchString implements the interface Regex.
func (pr *privateRegex) SetMatchString(ctx context.Context, matchStr string) (err error) {
// Check for the regex pointer first
if pr.regexPtr == nil {
return ErrRegexNotYetSet.New()
}
pr.matchStr.SetString(matchStr)
pr.matchSet = true
// Set the text on the URegularExpression*
errorCode := icu.UErrorCode(0)
icu.Uregex_setText(pr.regexPtr, &pr.matchStr, &errorCode)
if errorCode > 0 {
return fmt.Errorf("unexpected UErrorCode from uregex_setText: %d", errorCode)
}
return nil
}
// IndexOf implements the interface Regex.
func (pr *privateRegex) IndexOf(ctx context.Context, start int, occurrence int, endIndex bool) (int, error) {
// Check for the regex pointer first
if pr.regexPtr == nil {
return 0, ErrRegexNotYetSet.New()
}
// Check that the match string has been set
if !pr.matchSet {
return 0, ErrMatchNotYetSet.New()
}
// Look for a match
var errorCode icu.UErrorCode
ok := icu.Uregex_find(pr.regexPtr, start-1, &errorCode)
for i := 1; i < occurrence && ok; i++ {
ok = icu.Uregex_findNext(pr.regexPtr, &errorCode)
}
if !ok {
return 0, nil
}
// Get the index of the match
var index int
if endIndex {
index32 := icu.Uregex_end(pr.regexPtr, 0, &errorCode)
index = int(index32)
} else {
index32 := icu.Uregex_start(pr.regexPtr, 0, &errorCode)
index = int(index32)
}
if errorCode > 0 {
return 0, fmt.Errorf("unexpected UErrorCode from uregex_find/uregex_findNext: %d", errorCode)
}
return index + 1, nil
}
// Matches implements the interface Regex.
func (pr *privateRegex) Matches(ctx context.Context, start int, occurrence int) (ok bool, err error) {
// Check for the regex pointer first
if pr.regexPtr == nil {
return false, ErrRegexNotYetSet.New()
}
// Check that the match string has been set
if !pr.matchSet {
return false, ErrMatchNotYetSet.New()
}
// Return if we found a match
var errorCode icu.UErrorCode
ok = icu.Uregex_find(pr.regexPtr, start, &errorCode)
for i := 1; i < occurrence && ok; i++ {
ok = icu.Uregex_findNext(pr.regexPtr, &errorCode)
}
if errorCode > 0 {
return false, fmt.Errorf("unexpected UErrorCode from uregex_find/uregex_findNext: %d", errorCode)
}
return ok, err
}
// Replace implements the interface Regex.
func (pr *privateRegex) Replace(ctx context.Context, replacement string, start int, occurrence int) (replacedStr string, err error) {
// Check for the regex pointer first
if pr.regexPtr == nil {
return "", ErrRegexNotYetSet.New()
}
// Check that the match string has been set
if !pr.matchSet {
return "", ErrMatchNotYetSet.New()
}
return icu.Replace(pr.regexPtr, replacement, &pr.matchStr, start-1, occurrence), nil
}
// Substring implements the interface Regex.
func (pr *privateRegex) Substring(ctx context.Context, start int, occurrence int) (string, bool, error) {
// Check for the regex pointer first
if pr.regexPtr == nil {
return "", false, ErrRegexNotYetSet.New()
}
// Check that the match string has been set
if !pr.matchSet {
return "", false, ErrMatchNotYetSet.New()
}
// Look for a match
var errorCode icu.UErrorCode
ok := icu.Uregex_find(pr.regexPtr, start-1, &errorCode)
for i := 1; i < occurrence && ok; i++ {
ok = icu.Uregex_findNext(pr.regexPtr, &errorCode)
}
if !ok {
return "", false, nil
}
// Get the bounds of the match
idxStart := icu.Uregex_start(pr.regexPtr, 0, &errorCode)
idxEnd := icu.Uregex_end(pr.regexPtr, 0, &errorCode)
if errorCode > 0 {
return "", false, fmt.Errorf("unexpected UErrorCode from uregex_find/uregex_findNext: %d", errorCode)
}
return pr.matchStr.GetSubstring(int(idxStart), int(idxEnd)), true, nil
}
// Close implements the interface Regex.
func (pr *privateRegex) Close() (err error) {
if pr == nil {
return nil
}
if pr.regexPtr != nil {
pr.regexPtr.Free()
pr.regexPtr = nil
}
pr.matchStr.Free()
pr.regexStr.Free()
return nil
}

View file

@ -0,0 +1,203 @@
// Copyright 2023 Dolthub, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package regex
import (
"context"
"sync"
"testing"
"github.com/stretchr/testify/require"
)
func TestRegexpMatchLoop(t *testing.T) {
// Just test concurrency.
var wg sync.WaitGroup
wg.Add(16)
for i := 0; i < 16; i++ {
go func() {
defer wg.Done()
for i := 0; i < 2048; i++ {
TestRegexMatch(t)
}
}()
}
wg.Wait()
}
func TestRegexMatch(t *testing.T) {
ctx := context.Background()
regex := CreateRegex(512 * 1024)
require.NoError(t, regex.SetRegexString(ctx, `abc+.*this st`, RegexFlags_None))
err := regex.SetMatchString(ctx, "Find the abc in this string")
require.NoError(t, err)
ok, err := regex.Matches(ctx, 0, 0)
require.NoError(t, err)
require.True(t, ok)
err = regex.SetMatchString(ctx, "Find the abc in this here string")
require.NoError(t, err)
ok, err = regex.Matches(ctx, 0, 0)
require.NoError(t, err)
require.False(t, ok)
require.NoError(t, regex.Close())
regex = CreateRegex(11)
require.NoError(t, regex.SetRegexString(ctx, `[a-zA-Z0-9]{5} \w{4} aab`, RegexFlags_None))
err = regex.SetMatchString(ctx, "Words like aab don't exist")
require.NoError(t, err)
ok, err = regex.Matches(ctx, 0, 0)
require.NoError(t, err)
require.True(t, ok)
require.NoError(t, regex.SetRegexString(ctx, `^[aA]bcd[eE]$`, RegexFlags_None))
err = regex.SetMatchString(ctx, "abcde")
require.NoError(t, err)
ok, err = regex.Matches(ctx, 0, 0)
require.NoError(t, err)
require.True(t, ok)
err = regex.SetMatchString(ctx, "Abcde")
require.NoError(t, err)
ok, err = regex.Matches(ctx, 0, 0)
require.NoError(t, err)
require.True(t, ok)
err = regex.SetMatchString(ctx, "AbcdE")
require.NoError(t, err)
ok, err = regex.Matches(ctx, 0, 0)
require.NoError(t, err)
require.True(t, ok)
require.NoError(t, regex.Close())
regex = CreateRegex(128)
require.NoError(t, regex.SetRegexString(ctx, `^abcde$`, RegexFlags_None))
err = regex.SetMatchString(ctx, "abcde")
require.NoError(t, err)
ok, err = regex.Matches(ctx, 0, 0)
require.NoError(t, err)
require.True(t, ok)
err = regex.SetMatchString(ctx, "aBCDe")
require.NoError(t, err)
ok, err = regex.Matches(ctx, 0, 0)
require.NoError(t, err)
require.False(t, ok)
require.NoError(t, regex.Close())
}
func TestRegexReplace(t *testing.T) {
ctx := context.Background()
regex := CreateRegex(512 * 1024)
require.NoError(t, regex.SetRegexString(ctx, `[a-z]+`, RegexFlags_None))
err := regex.SetMatchString(ctx, "abc def ghi")
require.NoError(t, err)
replacedStr, err := regex.Replace(ctx, "X", 1, 2)
require.NoError(t, err)
require.Equal(t, "abc X ghi", replacedStr)
replacedStr, err = regex.Replace(ctx, "X", 1, 3)
require.NoError(t, err)
require.Equal(t, "abc def X", replacedStr)
replacedStr, err = regex.Replace(ctx, "X", 1, 0)
require.NoError(t, err)
require.Equal(t, "X X X", replacedStr)
require.NoError(t, regex.Close())
}
func TestRegexIndexOf(t *testing.T) {
ctx := context.Background()
regex := CreateRegex(1024)
require.NoError(t, regex.SetRegexString(ctx, `[a-j]+`, RegexFlags_None))
err := regex.SetMatchString(ctx, "abc def ghi")
require.NoError(t, err)
idx, err := regex.IndexOf(ctx, 1, 1, false)
require.NoError(t, err)
require.Equal(t, 1, idx)
idx, err = regex.IndexOf(ctx, 4, 1, false)
require.NoError(t, err)
require.Equal(t, 5, idx)
idx, err = regex.IndexOf(ctx, 8, 1, false)
require.NoError(t, err)
require.Equal(t, 9, idx)
idx, err = regex.IndexOf(ctx, 1, 2, false)
require.NoError(t, err)
require.Equal(t, 5, idx)
idx, err = regex.IndexOf(ctx, 1, 3, false)
require.NoError(t, err)
require.Equal(t, 9, idx)
idx, err = regex.IndexOf(ctx, 1, 4, false)
require.NoError(t, err)
require.Equal(t, 0, idx)
idx, err = regex.IndexOf(ctx, 1, 1, true)
require.NoError(t, err)
require.Equal(t, 4, idx)
idx, err = regex.IndexOf(ctx, 4, 1, true)
require.NoError(t, err)
require.Equal(t, 8, idx)
idx, err = regex.IndexOf(ctx, 8, 1, true)
require.NoError(t, err)
require.Equal(t, 12, idx)
idx, err = regex.IndexOf(ctx, 1, 2, true)
require.NoError(t, err)
require.Equal(t, 8, idx)
idx, err = regex.IndexOf(ctx, 1, 3, true)
require.NoError(t, err)
require.Equal(t, 12, idx)
idx, err = regex.IndexOf(ctx, 1, 4, true)
require.NoError(t, err)
require.Equal(t, 0, idx)
require.NoError(t, regex.SetMatchString(ctx, "klmno fghij abcde"))
idx, err = regex.IndexOf(ctx, 1, 1, false)
require.NoError(t, err)
require.Equal(t, 7, idx)
idx, err = regex.IndexOf(ctx, 1, 1, true)
require.NoError(t, err)
require.Equal(t, 12, idx)
require.NoError(t, regex.Close())
}
func TestRegexSubstring(t *testing.T) {
ctx := context.Background()
regex := CreateRegex(1024)
require.NoError(t, regex.SetRegexString(ctx, `[a-z]+`, RegexFlags_None))
err := regex.SetMatchString(ctx, "abc def ghi")
require.NoError(t, err)
substr, ok, err := regex.Substring(ctx, 1, 1)
require.NoError(t, err)
require.True(t, ok)
require.Equal(t, "abc", substr)
substr, ok, err = regex.Substring(ctx, 4, 1)
require.NoError(t, err)
require.True(t, ok)
require.Equal(t, "def", substr)
substr, ok, err = regex.Substring(ctx, 8, 1)
require.NoError(t, err)
require.True(t, ok)
require.Equal(t, "ghi", substr)
substr, ok, err = regex.Substring(ctx, 1, 2)
require.NoError(t, err)
require.True(t, ok)
require.Equal(t, "def", substr)
substr, ok, err = regex.Substring(ctx, 1, 3)
require.NoError(t, err)
require.True(t, ok)
require.Equal(t, "ghi", substr)
substr, ok, err = regex.Substring(ctx, 1, 4)
require.NoError(t, err)
require.False(t, ok)
require.Equal(t, "", substr)
require.NoError(t, regex.SetMatchString(ctx, "ghx dey abz"))
substr, ok, err = regex.Substring(ctx, 1, 1)
require.NoError(t, err)
require.True(t, ok)
require.Equal(t, "ghx", substr)
require.NoError(t, regex.Close())
}

View file

@ -0,0 +1,281 @@
//go:build windows
// Copyright 2023 Dolthub, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package regex
import (
"context"
"fmt"
"regexp"
"gopkg.in/src-d/go-errors.v1"
)
// Regex is an interface that wraps around the ICU library, exposing ICU's regular expression functionality. It is
// imperative that Regex is closed once it is finished.
type Regex interface {
// SetRegexString sets the string that will later be matched against. This must be called at least once before any other
// calls are made (except for Close).
SetRegexString(ctx context.Context, regexStr string, flags RegexFlags) error
// SetMatchString sets the string that we will either be matching against, or executing the replacements on. This
// must be called after SetRegexString, but before any other calls.
SetMatchString(ctx context.Context, matchStr string) error
// IndexOf returns the index of the previously-set regex matching the previously-set match string. Must call
// SetRegexString and SetMatchString before this function. `endIndex` determines whether the returned index is at
// the beginning or end of the match. `start` and `occurrence` start at 1, not 0. Returns 0 if the index was not found.
IndexOf(ctx context.Context, start int, occurrence int, endIndex bool) (int, error)
// Matches returns whether the previously-set regex matches the previously-set match string. Must call
// SetRegexString and SetMatchString before this function.
Matches(ctx context.Context, start int, occurrence int) (bool, error)
// Replace returns a new string with the replacement string occupying the matched portions of the match string,
// based on the regex. Position starts at 1, not 0. Must call SetRegexString and SetMatchString before this function.
Replace(ctx context.Context, replacementStr string, position int, occurrence int) (string, error)
// Substring returns the match of the previously-set match string, using the previously-set regex. Must call
// SetRegexString and SetMatchString before this function. `start` and `occurrence` start at 1, not 0.
Substring(ctx context.Context, start int, occurrence int) (string, bool, error)
// Close frees up the internal resources. This MUST be called, else a panic will occur at some non-deterministic time.
Close() error
}
var (
// ErrRegexNotYetSet is returned when attempting to use another function before the regex has been initialized.
ErrRegexNotYetSet = errors.NewKind("SetRegexString must be called before any other function")
// ErrMatchNotYetSet is returned when attempting to use another function before the match string has been set.
ErrMatchNotYetSet = errors.NewKind("SetMatchString must be called as there is nothing to match against")
// ErrInvalidRegex is returned when an invalid regex is given
ErrInvalidRegex = errors.NewKind("the given regular expression is invalid")
)
// RegexFlags are flags to define the behavior of the regular expression. Use OR (|) to combine flags. All flag values
// were taken directly from ICU.
type RegexFlags uint32
const (
// Enable case insensitive matching.
RegexFlags_None RegexFlags = 0
// Enable case insensitive matching.
RegexFlags_Case_Insensitive RegexFlags = 2
// Allow white space and comments within patterns.
RegexFlags_Comments RegexFlags = 4
// If set, '.' matches line terminators, otherwise '.' matching stops at line end.
RegexFlags_Dot_All RegexFlags = 32
// If set, treat the entire pattern as a literal string. Metacharacters or escape sequences in the input sequence
// will be given no special meaning.
//
// The flag RegexFlags_Case_Insensitive retains its impact on matching when used in conjunction with this flag. The
// other flags become superfluous.
RegexFlags_Literal RegexFlags = 16
// Control behavior of "$" and "^". If set, recognize line terminators within string, otherwise, match only at start
// and end of input string.
RegexFlags_Multiline RegexFlags = 8
// Unix-only line endings. When this mode is enabled, only '\n' is recognized as a line ending in the behavior
// of ., ^, and $.
RegexFlags_Unix_Lines RegexFlags = 1
// Unicode word boundaries. If set, \b uses the Unicode TR 29 definition of word boundaries. Warning: Unicode word
// boundaries are quite different from traditional regular expression word boundaries.
// See http://unicode.org/reports/tr29/#Word_Boundaries
RegexFlags_Unicode_Word RegexFlags = 256
// Error on Unrecognized backslash escapes. If set, fail with an error on patterns that contain backslash-escaped
// ASCII letters without a known special meaning. If this flag is not set, these escaped letters represent
// themselves.
RegexFlags_Error_On_Unknown_Escapes RegexFlags = 512
)
// CreateRegex creates a Regex. |stringBufferInBytes| is a hint to allocate string buffers
// for a certain size to avoid reallocation in the future, but is currently unused by the
// primary implementation.
func CreateRegex(stringBufferInBytes uint32) Regex {
return &privateRegex{}
}
// privateRegex is the private implementation of the Regex interface for Windows.
type privateRegex struct {
re *regexp.Regexp
str string
sset bool
done bool
start int
locs [][]int
}
var _ Regex = (*privateRegex)(nil)
// SetRegexString implements the interface Regex.
func (pr *privateRegex) SetRegexString(ctx context.Context, regexStr string, flags RegexFlags) (err error) {
_ = ctx
// i : RegexFlags_Case_Insensitive
// m : RegexFlags_Multiline
// s : RegexFlags_Dot_All
// RegexFlags_Unix_Lines
var flg = "(?"
if flags&RegexFlags_Case_Insensitive != 0 {
flg += "i"
}
if flags&RegexFlags_Multiline != 0 {
flg += "m"
}
if flags&RegexFlags_Dot_All != 0 {
flg += "s"
}
if len(flg) > 2 {
flg += ")"
} else {
flg = ""
}
if flags&RegexFlags_Literal != 0 {
regexStr = regexp.QuoteMeta(regexStr)
}
pr.done = false
pr.sset = false
pr.re, err = regexp.Compile(flg + regexStr)
if err != nil {
return ErrInvalidRegex.New()
}
return nil
}
// SetMatchString implements the interface Regex.
func (pr *privateRegex) SetMatchString(ctx context.Context, matchStr string) (err error) {
_ = ctx
if pr.re == nil {
return ErrRegexNotYetSet.New()
}
pr.done = false
pr.str = matchStr
pr.sset = true
return nil
}
func (pr *privateRegex) do(start int) error {
if start < 1 {
start = 1
}
if !pr.done || pr.start != start {
if pr.re == nil {
return ErrRegexNotYetSet.New()
}
if !pr.sset {
return ErrMatchNotYetSet.New()
}
pr.locs = pr.re.FindAllStringIndex(pr.str[start-1:], -1)
pr.start = start
pr.done = true
}
return nil
}
func (pr *privateRegex) location(occurrence int) []int {
occurrence--
if occurrence < 0 {
occurrence = 0
}
if len(pr.locs) < occurrence+1 {
return nil
}
return pr.locs[occurrence]
}
// IndexOf implements the interface Regex.
func (pr *privateRegex) IndexOf(ctx context.Context, start int, occurrence int, endIndex bool) (int, error) {
_ = ctx
err := pr.do(start)
if err != nil {
return 0, err
}
loc := pr.location(occurrence)
if loc == nil {
return 0, nil
}
pos := loc[0]
if endIndex {
pos = loc[1]
}
return pos + pr.start, nil
}
// Matches implements the interface Regex.
func (pr *privateRegex) Matches(ctx context.Context, start int, occurrence int) (bool, error) {
_ = ctx
err := pr.do(start + 1)
if err != nil {
return false, err
}
loc := pr.location(occurrence)
return loc != nil, nil
}
// Replace implements the interface Regex.
func (pr *privateRegex) Replace(ctx context.Context, replacement string, start int, occurrence int) (string, error) {
_ = ctx
err := pr.do(start)
if err != nil {
return "", err
}
var locs [][]int
if occurrence == 0 {
locs = pr.locs
} else {
loc := pr.location(occurrence)
if loc != nil {
locs = [][]int{loc}
}
}
offs := pr.start - 1
pos := offs
ret := []byte(pr.str[:pos])
for _, loc := range locs {
ret = fmt.Appendf(ret, "%s%s", pr.str[pos:loc[0]+offs], replacement)
pos = loc[1] + offs
}
ret = fmt.Append(ret, pr.str[pos:])
return string(ret), nil
}
// Substring implements the interface Regex.
func (pr *privateRegex) Substring(ctx context.Context, start int, occurrence int) (string, bool, error) {
_ = ctx
err := pr.do(start)
if err != nil {
return "", false, err
}
loc := pr.location(occurrence)
if loc == nil {
return "", false, nil
}
return pr.str[loc[0]+pr.start-1 : loc[1]+pr.start-1], true, nil
}
// Close implements the interface Regex.
func (pr *privateRegex) Close() (err error) {
pr.re = nil
pr.str = ""
pr.done = false
pr.locs = nil
return nil
}

View file

@ -7,6 +7,17 @@ SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
REPO_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"
SKIP_FILE="$REPO_ROOT/.test-skip"
# ICU4C is keg-only on macOS (Homebrew doesn't symlink it into /opt/homebrew).
# Dolt's go-icu-regex dependency needs these paths to compile and link.
if [[ "$(uname)" == "Darwin" ]]; then
ICU_PREFIX="$(brew --prefix icu4c 2>/dev/null || true)"
if [[ -n "$ICU_PREFIX" ]]; then
export CGO_CFLAGS="${CGO_CFLAGS:+$CGO_CFLAGS }-I${ICU_PREFIX}/include"
export CGO_CPPFLAGS="${CGO_CPPFLAGS:+$CGO_CPPFLAGS }-I${ICU_PREFIX}/include"
export CGO_LDFLAGS="${CGO_LDFLAGS:+$CGO_LDFLAGS }-L${ICU_PREFIX}/lib"
fi
fi
# Build skip pattern from .test-skip file
build_skip_pattern() {
if [[ ! -f "$SKIP_FILE" ]]; then